feat(deepdata): 引入修改密码、文件预览组件与清洗算子增强，deepflow 改名为 deep-flow

本次同步 09-12 至 09-28 的代码变更，主要内容：新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema）结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
2026-10-03 22:44:56 +08:00
parent cbb1048e18
commit ff22ed3a8b
1156 changed files with 129912 additions and 21716 deletions
@@ -1,52 +1,79 @@
-import presidio_analyzer as analyzer
-import presidio_anonymizer as anonymizer
-import spacy
-
-from deepdata.core.base_op import Mapper
-
-from .custom_entities import id_recognizer, phone_recognizer, zipcode_recognizer, url_recognizer
-
-
-class PiiDetector(Mapper):
-    custom_ops = True
-
-    def __init__(self, *args, **kwargs):
-        super(PiiDetector, self).__init__(*args, **kwargs)
-        self.support_language = kwargs.get("support_language", "zh")
-
-        self.nlp_engine = None
-        self.text_analyzer = None
-        self.anom = None
-
-        self.init_model(*args, **kwargs)
-
-    def init_model(self, *args, **kwargs):
-        spacy.load("zh_core_web_sm")
-        provider = analyzer.nlp_engine.NlpEngineProvider(
-            nlp_configuration={
-                "nlp_engine_name": "spacy",
-                "models": [
-                    {"lang_code": "zh", "model_name": "zh_core_web_sm"}
-                ]
-            }
-        )
-
-        # 创建NLP Engine
-        self.nlp_engine = provider.create_engine()
-
-        #  初始化AnalyzerEngine
-        self.text_analyzer = analyzer.AnalyzerEngine(nlp_engine=self.nlp_engine, supported_languages=["zh"])
-        self.text_analyzer.registry.load_predefined_recognizers()
-        for recognizer in [id_recognizer, phone_recognizer, zipcode_recognizer, url_recognizer]:
-            self.text_analyzer.registry.add_recognizer(recognizer)
-
-        # 初始化AnonymizerEngine
-        self.anom = anonymizer.AnonymizerEngine()
-
-    def execute(self, sample):
-        self.read_file_first(sample)
-        text = sample.get('text')
-        analyzer_results = self.text_analyzer.analyze(text=text, language=self.support_language)
-        res = self.anom.anonymize(text=text, analyzer_results=analyzer_results)
-        sample['text'] = res.text
-        return sample
+import time
+from typing import Dict, Any
+
+import presidio_analyzer as analyzer
+import presidio_anonymizer as anonymizer
+import spacy
+from loguru import logger
+
+from deepdata.core.base_op import Mapper
+
+from .custom_entities import id_recognizer, phone_recognizer, zipcode_recognizer, url_recognizer
+
+
+class PiiDetector(Mapper):
+    custom_ops = True
+
+    def __init__(self, *args, **kwargs):
+        super(PiiDetector, self).__init__(*args, **kwargs)
+        self.support_language = kwargs.get("support_language", "zh")
+
+        self.nlp_engine = None
+        self.text_analyzer = None
+        self.anom = None
+
+        self.init_model(*args, **kwargs)
+
+    def init_model(self, *args, **kwargs):
+        try:
+            spacy.load("zh_core_web_sm")
+        except OSError:
+            logger.warning(
+                "zh_core_web_sm model not found. Please run: python -m spacy download zh_core_web_sm"
+            )
+            raise
+
+        provider = analyzer.nlp_engine.NlpEngineProvider(
+            nlp_configuration={
+                "nlp_engine_name": "spacy",
+                "models": [
+                    {"lang_code": "zh", "model_name": "zh_core_web_sm"}
+                ]
+            }
+        )
+
+        # 创建NLP Engine
+        self.nlp_engine = provider.create_engine()
+
+        #  初始化AnalyzerEngine
+        self.text_analyzer = analyzer.AnalyzerEngine(nlp_engine=self.nlp_engine, supported_languages=["zh"])
+        self.text_analyzer.registry.load_predefined_recognizers()
+        for recognizer in [id_recognizer, phone_recognizer, zipcode_recognizer, url_recognizer]:
+            self.text_analyzer.registry.add_recognizer(recognizer)
+
+        # 初始化AnonymizerEngine
+        self.anom = anonymizer.AnonymizerEngine()
+
+    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
+        start = time.time()
+        self.read_file_first(sample)
+        text = sample.get(self.text_key, "")
+        if not text:
+            logger.warning(f"fileName: {sample.get(self.filename_key, 'unknown')}, text is empty, skip PII detection.")
+            return sample
+
+        try:
+            analyzer_results = self.text_analyzer.analyze(text=text, language=self.support_language)
+            res = self.anom.anonymize(text=text, analyzer_results=analyzer_results)
+            sample[self.text_key] = res.text
+        except Exception as e:
+            logger.error(
+                f"fileName: {sample.get(self.filename_key, 'unknown')}, "
+                f"PII detection failed: {e}. Returning original text."
+            )
+
+        logger.info(
+            f"fileName: {sample.get(self.filename_key, 'unknown')}, "
+            f"method: PiiDetector costs {time.time() - start:.6f} s"
+        )
+        return sample