figmar/deepdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(deepdata): 引入修改密码、文件预览组件与清洗算子增强，deepflow 改名为 deep-flow

Browse Source 
本次同步 09-12 至 09-28 的代码变更，主要内容：

新增功能
- 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService
  对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口
- 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、
  ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器
- 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator
- 新增 tools/minio-file-server MinIO 文件服务

功能增强
- 数据清洗：cleaning_task_service/scheduler/routes 重构，
  data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整
- 数据标注：annotation sync.py、project.py 接口与schema 更新
- 数据采集、算子市场、首页与菜单路由相关页面调整
- DataJuicer 算子接入（useOperatorOperations、dj_ops_schema）

结构调整
- deepflow/ 更名为 deep-flow/（419 个文件重命名）
- 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本

未纳入本次提交
- scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录）
- *.backup*、__pycache__ 产物
- zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物）
- DataX/（内嵌独立仓库，远程亦未收录其源码）
This commit is contained in:
沃巴蒂
2026-10-03 22:44:56 +08:00
parent cbb1048e18
commit ff22ed3a8b
1156 changed files with 129912 additions and 21716 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/ops/formatter/mineru_formatter/process.py
+104 -104
View File
@@ -1,105 +1,105 @@
#!/user/bin/python
# -*- coding: utf-8 -*-
"""
Description: MinerU PDF文本抽取
Create: 2025/10/29 17:24
"""
import asyncio
import glob
import os
import shutil
import time
import uuid
from pathlib import Path
from typing import Dict, Any
from deepdata.core.base_op import Mapper, FileExporter
from deepdata.sql_manager.persistence_atction import TaskInfoPersistence
from loguru import logger
from mineru.cli.common import aio_do_parse, read_fn
from mineru.cli.fast_api import get_infer_result
from pypdf import PdfReader
class MineruFormatter(Mapper):
"""基于外部API，抽取PDF中的文本"""
def __init__(self, *args, **kwargs):
super(MineruFormatter, self).__init__(*args, **kwargs)
self.server_url = kwargs.get("mineruApi", "http://deepdata-mineru:8000")
self.backend = "vlm-http-client"
self.output_dir = "/dataset/outputs"
self.max_retries = 3
self.target_type = kwargs.get("exportType", "md")
def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
start = time.time()
filename = sample[self.filename_key]
if not filename.lower().endswith((".png", ".jpeg", ".jpg", ".webp", ".gif", ".pdf")):
return sample
try:
sample[self.text_key] = asyncio.run(self.async_process_file(sample))
sample[self.target_type_key] = self.target_type
logger.info(
f"fileName: {filename}, method: MineruFormatter costs {(time.time() - start):6f} s")
except Exception as e:
logger.exception(f"fileName: {filename}, method: MineruFormatter causes error: {e}")
raise
return sample
async def async_process_file(self, sample):
filename = sample[self.filename_key]
filename_without_ext = os.path.splitext(filename)[0]
filepath = sample[self.filepath_key]
parse_dir = os.path.join(self.output_dir, filename_without_ext, "vlm")
pdf_bytes = read_fn(filepath)
total_page = len(PdfReader(filepath).pages)
content = ""
for page in range(0, total_page, 10):
logger.info(f"fileName: {filename}, total_page: {total_page}, page: {page}.")
for attempt in range(self.max_retries):
try:
await aio_do_parse(
output_dir=self.output_dir,
pdf_file_names=[filename_without_ext],
pdf_bytes_list=[pdf_bytes],
p_lang_list=["ch"],
backend=self.backend,
server_url=self.server_url,
start_page_id=page,
end_page_id=min(page + 9, total_page - 1),
)
break # 成功则跳出重试循环
except Exception as e:
logger.warning(
f"Extract {filename} [{page}-{page + 9}] failed (attempt {attempt + 1}/{self.max_retries}). "
f"Error: {e}. Retrying in 5s..."
)
if attempt < self.max_retries - 1:
await asyncio.sleep(5)
else:
logger.error(f"aio_do_parse failed after {self.max_retries} attempts.")
raise # 耗尽次数后抛出异常，交给上层 execute 处理
if os.path.exists(parse_dir):
content += get_infer_result(".md", filename_without_ext, parse_dir)
self.save_images(parse_dir, sample["dataset_id"], os.path.abspath(sample[self.export_path_key]) + "/images")
shutil.rmtree(parse_dir)
return content
def save_images(self, parse_dir, dataset_id, export_path):
Path(export_path).mkdir(parents=True, exist_ok=True)
images_dir = os.path.join(parse_dir, "images")
image_paths = glob.glob(os.path.join(glob.escape(images_dir), "*.jpg"))
for image_path in image_paths:
shutil.copy(image_path, export_path)
image_sample = {}
image = Path(image_path)
image_name = image.name
image_sample[self.filename_key] = image_name
image_sample[self.filetype_key] = "jpg"
image_sample[self.filesize_key] = image.stat().st_size
image_sample["dataset_id"] = dataset_id
image_sample[self.filepath_key] = export_path + "/" + image_name
#!/user/bin/python␍
# -*- coding: utf-8 -*-␍
␍
"""␍
Description: MinerU PDF文本抽取␍
Create: 2025/10/29 17:24␍
"""␍
import asyncio␍
import glob␍
import os␍
import shutil␍
import time␍
import uuid␍
from pathlib import Path␍
from typing import Dict, Any␍
␍
from deepdata.core.base_op import Mapper, FileExporter␍
from deepdata.sql_manager.persistence_atction import TaskInfoPersistence␍
from loguru import logger␍
from mineru.cli.common import aio_do_parse, read_fn␍
from mineru.cli.fast_api import get_infer_result␍
from pypdf import PdfReader␍
␍
␍
class MineruFormatter(Mapper):␍
"""基于外部API，抽取PDF中的文本"""␍
␍
def __init__(self, *args, **kwargs):␍
super(MineruFormatter, self).__init__(*args, **kwargs)␍
self.server_url = kwargs.get("mineruApi", "http://deepdata-mineru:8000")␍
self.backend = "vlm-http-client"␍
self.output_dir = "/dataset/outputs"␍
self.max_retries = 3␍
self.target_type = kwargs.get("exportType", "md")␍
␍
def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:␍
start = time.time()␍
filename = sample[self.filename_key]␍
if not filename.lower().endswith((".png", ".jpeg", ".jpg", ".webp", ".gif", ".pdf")):␍
return sample␍
try:␍
sample[self.text_key] = asyncio.run(self.async_process_file(sample))␍
sample[self.target_type_key] = self.target_type␍
logger.info(␍
f"fileName: {filename}, method: MineruFormatter costs {(time.time() - start):.6f} s")␍
except Exception as e:␍
logger.exception(f"fileName: {filename}, method: MineruFormatter causes error: {e}")␍
raise␍
return sample␍
␍
async def async_process_file(self, sample):␍
filename = sample[self.filename_key]␍
filename_without_ext = os.path.splitext(filename)[0]␍
filepath = sample[self.filepath_key]␍
parse_dir = os.path.join(self.output_dir, filename_without_ext, "vlm")␍
pdf_bytes = read_fn(filepath)␍
total_page = len(PdfReader(filepath).pages)␍
content = ""␍
for page in range(0, total_page, 10):␍
logger.info(f"fileName: {filename}, total_page: {total_page}, page: {page}.")␍
for attempt in range(self.max_retries):␍
try:␍
await aio_do_parse(␍
output_dir=self.output_dir,␍
pdf_file_names=[filename_without_ext],␍
pdf_bytes_list=[pdf_bytes],␍
p_lang_list=["ch"],␍
backend=self.backend,␍
server_url=self.server_url,␍
start_page_id=page,␍
end_page_id=min(page + 9, total_page - 1),␍
)␍
break # 成功则跳出重试循环␍
except Exception as e:␍
logger.warning(␍
f"Extract {filename} [{page}-{page + 9}] failed (attempt {attempt + 1}/{self.max_retries}). "␍
f"Error: {e}. Retrying in 5s..."␍
)␍
if attempt < self.max_retries - 1:␍
await asyncio.sleep(5)␍
else:␍
logger.error(f"aio_do_parse failed after {self.max_retries} attempts.")␍
raise # 耗尽次数后抛出异常，交给上层 execute 处理␍
if os.path.exists(parse_dir):␍
content += get_infer_result(".md", filename_without_ext, parse_dir)␍
self.save_images(parse_dir, sample["dataset_id"], os.path.abspath(sample[self.export_path_key]) + "/images")␍
shutil.rmtree(parse_dir)␍
return content␍
␍
def save_images(self, parse_dir, dataset_id, export_path):␍
Path(export_path).mkdir(parents=True, exist_ok=True)␍
␍
images_dir = os.path.join(parse_dir, "images")␍
image_paths = glob.glob(os.path.join(glob.escape(images_dir), "*.jpg"))␍
for image_path in image_paths:␍
shutil.copy(image_path, export_path)␍
image_sample = {}␍
image = Path(image_path)␍
image_name = image.name␍
image_sample[self.filename_key] = image_name␍
image_sample[self.filetype_key] = "jpg"␍
image_sample[self.filesize_key] = image.stat().st_size␍
image_sample["dataset_id"] = dataset_id␍
image_sample[self.filepath_key] = export_path + "/" + image_name␍
TaskInfoPersistence().update_file_result(image_sample, str(uuid.uuid4()))
runtime/ops/formatter/slide_formatter/process.py
+36 -36
View File
@@ -1,36 +1,36 @@
# -- encoding: utf-8 --
"""
Description: 医疗图片解析载入
Create: 2025/02/08 11:00
"""
import time
from typing import Dict, Any
from loguru import logger
from deepdata.core.base_op import Mapper
class SlideFormatter(Mapper):
def __init__(self, *args, **kwargs):
super(SlideFormatter, self).__init__(*args, **kwargs)
def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
'''
Read medical image and corresponding mask file, each as Image type in COntent value. Return Content.
'''
start = time.time()
file_type = sample[self.filetype_key]
types_openslide = ['svs', 'tif', 'dcm', 'vms', 'vmu',
'ndpi', 'scn', 'mrxs', 'tiff', 'svslide',
'bif', 'czi', 'sdpc']
if file_type not in types_openslide:
raise TypeError(f"Format not supported: {file_type}. Supported formats are: {', '.join(types_openslide)}.")
file_name = sample[self.filename_key]
logger.info(f"fileName: {file_name}, method: SlideFormatter costs {(time.time() - start):6f} s")
# Not really loading the slide, instead, use path as lazy loading.
return sample
# -- encoding: utf-8 --␍
␍
"""␍
Description: 医疗图片解析载入␍
Create: 2025/02/08 11:00␍
"""␍
␍
import time␍
from typing import Dict, Any␍
␍
from loguru import logger␍
␍
from deepdata.core.base_op import Mapper␍
␍
␍
class SlideFormatter(Mapper):␍
␍
def __init__(self, *args, **kwargs):␍
super(SlideFormatter, self).__init__(*args, **kwargs)␍
␍
def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:␍
'''␍
Read medical image and corresponding mask file, each as Image type in COntent value. Return Content.␍
'''␍
start = time.time()␍
file_type = sample[self.filetype_key]␍
types_openslide = ['svs', 'tif', 'dcm', 'vms', 'vmu',␍
'ndpi', 'scn', 'mrxs', 'tiff', 'svslide',␍
'bif', 'czi', 'sdpc']␍
if file_type not in types_openslide:␍
raise TypeError(f"Format not supported: {file_type}. Supported formats are: {', '.join(types_openslide)}.")␍
␍
file_name = sample[self.filename_key]␍
logger.info(f"fileName: {file_name}, method: SlideFormatter costs {(time.time() - start):.6f} s")␍
# Not really loading the slide, instead, use path as lazy loading.␍
return sample␍