feat(deepdata): 引入修改密码、文件预览组件与清洗算子增强，deepflow 改名为 deep-flow

本次同步 09-12 至 09-28 的代码变更，主要内容：新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema）结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
2026-10-03 22:44:56 +08:00
parent cbb1048e18
commit ff22ed3a8b
1156 changed files with 129912 additions and 21716 deletions
@@ -1,48 +1,48 @@
-#!/user/bin/python
-# -*- coding: utf-8 -*-
-
-"""
-Description: 邮件地址匿名化
-Create: 2025/01/15
-"""
-from loguru import logger
-import re
-import time
-from typing import Dict, Any
-
-from email_validator import validate_email, EmailNotValidError
-
-
-from deepdata.core.base_op import Mapper
-
-
-class EmailNumberCleaner(Mapper):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.front_email_pattern = r'(?<=[^0-9a-zA-Z\!\#\$\%\&\'\*\+\-\/\=\?\^\_\`\{\|\}\~\-])'
-        self.back_email_pattern = r'(?=[^0-9a-zA-Z\!\#\$\%\&\'\*\+\-\/\=\?\^\_\`\{\|\}\~\-])'
-        self.email_pattern = r'([a-zA-Z\d.\-+_]+\s?@\s?[a-zA-Z\d.\-+_]+\.[a-zA-Z0-9]{2,6})'
-
-    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
-        start = time.time()
-        self.read_file_first(sample)
-        sample[self.text_key] = self._email_number_filter(sample[self.text_key])
-        logger.info(f"fileName: {sample[self.filename_key]}, method: EmailCleaner costs {time.time() - start:6f} s")
-        return sample
-
-    def _email_number_filter(self, input_data: str):
-        """ 邮箱匿名化"""
-        mixed_data = ''.join(['龥', input_data, '龥'])
-        paired_emails = re.compile(self.front_email_pattern + self.email_pattern + self.back_email_pattern).findall(
-            mixed_data)
-        if paired_emails:
-            for email in paired_emails:
-                try:
-                    # 验证电子邮件地址
-                    validate_email(email, check_deliverability=False)
-                    mixed_data = re.compile(self.front_email_pattern + re.escape(email) + self.back_email_pattern).sub(
-                        "<email>", mixed_data, count=1)
-                except EmailNotValidError as err:
-                    # 日志打印该电子邮件地址无效（不显示具体电子邮件地址）
-                    logger.error(f"email is abnormal email form: {err}")
-        return mixed_data[1:-1]
+#!/user/bin/python␍
+# -*- coding: utf-8 -*-␍
+␍
+"""␍
+Description: 邮件地址匿名化␍
+Create: 2025/01/15␍
+"""␍
+from loguru import logger␍
+import re␍
+import time␍
+from typing import Dict, Any␍
+␍
+from email_validator import validate_email, EmailNotValidError␍
+␍
+␍
+from deepdata.core.base_op import Mapper␍
+␍
+␍
+class EmailNumberCleaner(Mapper):␍
+    def __init__(self, *args, **kwargs):␍
+        super().__init__(*args, **kwargs)␍
+        self.front_email_pattern = r'(?<=[^0-9a-zA-Z\!\#\$\%\&\'\*\+\-\/\=\?\^\_\`\{\|\}\~\-])'␍
+        self.back_email_pattern = r'(?=[^0-9a-zA-Z\!\#\$\%\&\'\*\+\-\/\=\?\^\_\`\{\|\}\~\-])'␍
+        self.email_pattern = r'([a-zA-Z\d.\-+_]+\s?@\s?[a-zA-Z\d.\-+_]+\.[a-zA-Z0-9]{2,6})'␍
+␍
+    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:␍
+        start = time.time()␍
+        self.read_file_first(sample)␍
+        sample[self.text_key] = self._email_number_filter(sample[self.text_key])␍
+        logger.info(f"fileName: {sample[self.filename_key]}, method: EmailCleaner costs {time.time() - start:.6f} s")␍
+        return sample␍
+␍
+    def _email_number_filter(self, input_data: str):␍
+        """ 邮箱匿名化"""␍
+        mixed_data = ''.join(['龥', input_data, '龥'])␍
+        paired_emails = re.compile(self.front_email_pattern + self.email_pattern + self.back_email_pattern).findall(␍
+            mixed_data)␍
+        if paired_emails:␍
+            for email in paired_emails:␍
+                try:␍
+                    # 验证电子邮件地址␍
+                    validate_email(email, check_deliverability=False)␍
+                    mixed_data = re.compile(self.front_email_pattern + re.escape(email) + self.back_email_pattern).sub(␍
+                        "<email>", mixed_data, count=1)␍
+                except EmailNotValidError as err:␍
+                    # 日志打印该电子邮件地址无效（不显示具体电子邮件地址）␍
+                    logger.error(f"email is abnormal email form: {err}")␍
+        return mixed_data[1:-1]␍