沃巴蒂
ff22ed3a8b
本次同步 09-12 至 09-28 的代码变更，主要内容： 新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务 功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema） 结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本 未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
114 lines
3.8 KiB
Python
114 lines
3.8 KiB
Python
# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
|
|
# SPDX-License-Identifier: MIT
|
|
from src.crawler.article import Article
|
|
|
|
|
|
class DummyMarkdownify:
|
|
"""A dummy markdownify replacement for patching if needed."""
|
|
|
|
@staticmethod
|
|
def markdownify(html):
|
|
return html
|
|
|
|
|
|
def test_to_markdown_includes_title(monkeypatch):
|
|
article = Article("Test Title", "<p>Hello <b>world</b>!</p>")
|
|
result = article.to_markdown(including_title=True)
|
|
assert result.startswith("# Test Title")
|
|
assert "Hello" in result
|
|
|
|
|
|
def test_to_markdown_excludes_title():
|
|
article = Article("Test Title", "<p>Hello <b>world</b>!</p>")
|
|
result = article.to_markdown(including_title=False)
|
|
assert not result.startswith("# Test Title")
|
|
assert "Hello" in result
|
|
|
|
|
|
def test_to_message_with_text_only():
|
|
article = Article("Test Title", "<p>Hello world!</p>")
|
|
article.url = "https://example.com/"
|
|
result = article.to_message()
|
|
assert isinstance(result, list)
|
|
assert any(item["type"] == "text" for item in result)
|
|
assert all("type" in item for item in result)
|
|
|
|
|
|
def test_to_message_with_image(monkeypatch):
|
|
html = '<p>Intro</p><img src="img/pic.png"/>'
|
|
article = Article("Title", html)
|
|
article.url = "https://host.com/path/"
|
|
# The markdownify library will convert <img> to markdown image syntax
|
|
result = article.to_message()
|
|
# Should have both text and image_url types
|
|
types = [item["type"] for item in result]
|
|
assert "image_url" in types
|
|
assert "text" in types
|
|
# Check that the image_url is correctly joined
|
|
image_items = [item for item in result if item["type"] == "image_url"]
|
|
assert image_items
|
|
assert image_items[0]["image_url"]["url"] == "https://host.com/path/img/pic.png"
|
|
|
|
|
|
def test_to_message_multiple_images():
|
|
html = '<p>Start</p><img src="a.png"/><p>Mid</p><img src="b.jpg"/>End'
|
|
article = Article("Title", html)
|
|
article.url = "http://x/"
|
|
result = article.to_message()
|
|
image_urls = [
|
|
item["image_url"]["url"] for item in result if item["type"] == "image_url"
|
|
]
|
|
assert "http://x/a.png" in image_urls
|
|
assert "http://x/b.jpg" in image_urls
|
|
text_items = [item for item in result if item["type"] == "text"]
|
|
assert any("Start" in item["text"] for item in text_items)
|
|
assert any("Mid" in item["text"] for item in text_items)
|
|
|
|
|
|
def test_to_message_handles_empty_html():
|
|
article = Article("Empty", "")
|
|
article.url = "http://test/"
|
|
result = article.to_message()
|
|
assert isinstance(result, list)
|
|
assert result[0]["type"] == "text"
|
|
|
|
|
|
def test_to_markdown_handles_none_content():
|
|
article = Article("Test Title", None)
|
|
result = article.to_markdown(including_title=True)
|
|
assert "# Test Title" in result
|
|
assert "No content available" in result
|
|
|
|
|
|
def test_to_markdown_handles_empty_string():
|
|
article = Article("Test Title", "")
|
|
result = article.to_markdown(including_title=True)
|
|
assert "# Test Title" in result
|
|
assert "No content available" in result
|
|
|
|
|
|
def test_to_markdown_handles_whitespace_only():
|
|
article = Article("Test Title", " \n \t ")
|
|
result = article.to_markdown(including_title=True)
|
|
assert "# Test Title" in result
|
|
assert "No content available" in result
|
|
|
|
|
|
def test_to_message_handles_none_content():
|
|
article = Article("Title", None)
|
|
article.url = "http://test/"
|
|
result = article.to_message()
|
|
assert isinstance(result, list)
|
|
assert len(result) > 0
|
|
assert result[0]["type"] == "text"
|
|
assert "No content available" in result[0]["text"]
|
|
|
|
|
|
def test_to_message_handles_whitespace_only_content():
|
|
article = Article("Title", " \n ")
|
|
article.url = "http://test/"
|
|
result = article.to_message()
|
|
assert isinstance(result, list)
|
|
assert result[0]["type"] == "text"
|
|
assert "No content available" in result[0]["text"]
|