feat(deepdata): 引入修改密码、文件预览组件与清洗算子增强,deepflow 改名为 deep-flow

本次同步 09-12 至 09-28 的代码变更,主要内容:

新增功能
- 修改密码:后端新增 ChangePasswordRequest DTO 与UserController/UserService
  对应接口,前端新增 ChangePasswordDialog 并在 Header 接入入口
- 文件预览组件重构:file-preview 目录拆分出 VideoPreview(支持 blobUrl 直传)、
  ImagePreview、AudioPreview,index.tsx 统一调度各类型预览器
- 新增 my_ops 算子模块:markitdown_parser、mineru_pdf_parser、text_quality_evaluator
- 新增 tools/minio-file-server MinIO 文件服务

功能增强
- 数据清洗:cleaning_task_service/scheduler/routes 重构,
  data_juicer_executor 与 wrapper 增强,OperatorTable/LogsTable/TaskDetail 同步调整
- 数据标注:annotation sync.py、project.py 接口与schema 更新
- 数据采集、算子市场、首页与菜单路由相关页面调整
- DataJuicer 算子接入(useOperatorOperations、dj_ops_schema)

结构调整
- deepflow/ 更名为 deep-flow/(419 个文件重命名)
- 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本

未纳入本次提交
- scripts-backup/、runtime/python-executor-backup/、components_backup/(备份目录)
- *.backup*、__pycache__ 产物
- zh_core_web_sm-3.8.0-py3-none-any.whl(46MB 构建产物)
- DataX/(内嵌独立仓库,远程亦未收录其源码)
This commit is contained in:
沃巴蒂
2026-10-03 22:44:56 +08:00
parent cbb1048e18
commit ff22ed3a8b
1156 changed files with 129912 additions and 21716 deletions
+85
View File
@@ -0,0 +1,85 @@
from markitdown import MarkItDown
from openpyxl import load_workbook
from io import BytesIO
from PIL import Image
import base64
from zipfile import ZipFile
def extract_xlsx_all_images_base64(xlsx_path: str, max_width=1200):
img_list = []
wb = load_workbook(xlsx_path)
# 1.先读取单元格内嵌图片
for sheet_name in wb.sheetnames:
ws = wb[sheet_name]
for img in ws._images:
img_bytes = img.ref
pil_img = Image.open(BytesIO(img_bytes))
if pil_img.width > max_width:
ratio = max_width / pil_img.width
new_size = (max_width, int(pil_img.height * ratio))
pil_img = pil_img.resize(new_size, Image.Resampling.LANCZOS)
buf = BytesIO()
pil_img.save(buf, format="PNG", optimize=True)
b64_str = base64.b64encode(buf.getvalue()).decode("utf-8")
uri = f"data:image/png;base64,{b64_str}"
img_list.append({"sheet": sheet_name, "data_uri": uri})
wb.close()
# 2.解析xlsx压缩包内drawing.xml,提取浮动形状图片
with ZipFile(xlsx_path, "r") as zf:
file_list = zf.namelist()
# 遍历所有drawing,匹配media图片
drawing_files = [f for f in file_list if f.startswith("xl/drawings/drawing") and f.endswith(".xml")]
for drawing_path in drawing_files:
# 解析drawing.xml获取图片rId
import xml.etree.ElementTree as ET
ns_a = {"a": "http://schemas.openxmlformats.org/drawingml/2006/main"}
ns_xdr = {"xdr": "http://schemas.openxmlformats.org/drawingml/2006/spreadsheetDrawing"}
xml_data = zf.read(drawing_path)
root = ET.fromstring(xml_data)
for blip in root.findall(".//a:blip", ns_a):
r_id = blip.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}embed")
if not r_id:
continue
# 读取drawing的关系文件
rel_path = drawing_path.replace(".xml", ".rels")
if rel_path not in file_list:
continue
rel_xml = zf.read(rel_path)
rel_root = ET.fromstring(rel_xml)
ns_rel = {"r": "http://schemas.openxmlformats.org/package/2006/relationships"}
for rel in rel_root.findall(".//r:Relationship", ns_rel):
if rel.get("Id") == r_id:
target = rel.get("Target")
media_path = f"xl/drawings/{target}"
if media_path in file_list:
media_bytes = zf.read(media_path)
pil_img = Image.open(BytesIO(media_bytes))
if pil_img.width > max_width:
ratio = max_width / pil_img.width
new_size = (max_width, int(pil_img.height * ratio))
pil_img = pil_img.resize(new_size, Image.Resampling.LANCZOS)
buf = BytesIO()
pil_img.save(buf, format="PNG", optimize=True)
b64_str = base64.b64encode(buf.getvalue()).decode("utf-8")
uri = f"data:image/png;base64,{b64_str}"
# 缺陷:无法精确归属到sheet名称,统一标记为浮动图片
img_list.append({"sheet": "FloatingShape", "data_uri": uri})
return img_list
if __name__ == "__main__":
xlsx_file = "test_xlsx.xlsx"
out_md = "xlsx_out.md"
md = MarkItDown()
res = md.convert(source=xlsx_file)
md_text = res.text_content
images = extract_xlsx_all_images_base64(xlsx_file)
print(f"一共提取图片:{len(images)}")
if len(images) > 0:
md_text += "\n\n## 提取的内嵌图片\n"
for item in images:
md_text += f"\n【Sheet:{item['sheet']}】\n![]({item['data_uri']})\n"
with open(out_md, "w", encoding="utf-8") as f:
f.write(md_text)
print(f"✅转换完成,输出文件:{out_md}")