deepdata: core product code (backend, frontend, runtime, deepflow module)

2026-08-30 19:23:38 +08:00
commit bd3db30be9
1405 changed files with 271727 additions and 0 deletions
@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+
+import sys
+from pathlib import Path
+from deepdata.core.base_op import OPERATORS
+from deepdata.common.utils.custom_importer import CustomImporter
+
+
+def _configure_importer():
+    base_path = Path(__file__).resolve().parent
+    sys.meta_path.append(CustomImporter(base_path))
+
+
+_configure_importer()
+
+
+def _import_operators():
+    from . import slide_formatter
+    from . import mineru_formatter
+
+
+_import_operators()
@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from deepdata.core.base_op import OPERATORS
+
+OPERATORS.register_module(module_name='MineruFormatter',
+                          module_path="ops.formatter.mineru_formatter.process")
@@ -0,0 +1,34 @@
+name: 'MinerU PDF文本抽取'
+name_en: 'MinerU PDF Text Extraction'
+description: '基于MinerU API，抽取PDF中的文本。'
+description_en: 'Extracts text from PDF files based on MinerU API.'
+language: 'python'
+vendor: 'huawei'
+raw_id: 'MineruFormatter'
+version: '1.0.0'
+types:
+  - 'collect'
+modal: 'text'
+effect:
+  before: ''
+  after: ''
+inputs: 'text'
+outputs: 'text'
+settings:
+  mineruApi:
+    name: 'Mineru Api地址'
+    description: '指定mineru服务的api地址，默认为本地同一集群内地址。'
+    type: 'input'
+    defaultVal: 'http://deepdata-mineru:8000'
+    required: false
+  exportType:
+    name: '导出类型'
+    description: '指定清洗结果文件类型。若指定为md且后续存在其他清洗算子，可能导致文件格式错乱。'
+    type: 'select'
+    defaultVal: 'markdown'
+    required: false
+    options:
+      - label: 'markdown'
+        value: 'md'
+      - label: 'txt'
+        value: 'txt'
@@ -0,0 +1,105 @@
+#!/user/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Description: MinerU PDF文本抽取
+Create: 2025/10/29 17:24
+"""
+import asyncio
+import glob
+import os
+import shutil
+import time
+import uuid
+from pathlib import Path
+from typing import Dict, Any
+
+from deepdata.core.base_op import Mapper, FileExporter
+from deepdata.sql_manager.persistence_atction import TaskInfoPersistence
+from loguru import logger
+from mineru.cli.common import aio_do_parse, read_fn
+from mineru.cli.fast_api import get_infer_result
+from pypdf import PdfReader
+
+
+class MineruFormatter(Mapper):
+    """基于外部API，抽取PDF中的文本"""
+
+    def __init__(self, *args, **kwargs):
+        super(MineruFormatter, self).__init__(*args, **kwargs)
+        self.server_url = kwargs.get("mineruApi", "http://deepdata-mineru:8000")
+        self.backend = "vlm-http-client"
+        self.output_dir = "/dataset/outputs"
+        self.max_retries = 3
+        self.target_type = kwargs.get("exportType", "md")
+
+    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
+        start = time.time()
+        filename = sample[self.filename_key]
+        if not filename.lower().endswith((".png", ".jpeg", ".jpg", ".webp", ".gif", ".pdf")):
+            return sample
+        try:
+            sample[self.text_key] = asyncio.run(self.async_process_file(sample))
+            sample[self.target_type_key] = self.target_type
+            logger.info(
+                f"fileName: {filename}, method: MineruFormatter costs {(time.time() - start):6f} s")
+        except Exception as e:
+            logger.exception(f"fileName: {filename}, method: MineruFormatter causes error: {e}")
+            raise
+        return sample
+
+    async def async_process_file(self, sample):
+        filename = sample[self.filename_key]
+        filename_without_ext = os.path.splitext(filename)[0]
+        filepath = sample[self.filepath_key]
+        parse_dir = os.path.join(self.output_dir, filename_without_ext, "vlm")
+        pdf_bytes = read_fn(filepath)
+        total_page = len(PdfReader(filepath).pages)
+        content = ""
+        for page in range(0, total_page, 10):
+            logger.info(f"fileName: {filename}, total_page: {total_page}, page: {page}.")
+            for attempt in range(self.max_retries):
+                try:
+                    await aio_do_parse(
+                        output_dir=self.output_dir,
+                        pdf_file_names=[filename_without_ext],
+                        pdf_bytes_list=[pdf_bytes],
+                        p_lang_list=["ch"],
+                        backend=self.backend,
+                        server_url=self.server_url,
+                        start_page_id=page,
+                        end_page_id=min(page + 9, total_page - 1),
+                    )
+                    break  # 成功则跳出重试循环
+                except Exception as e:
+                    logger.warning(
+                        f"Extract {filename} [{page}-{page + 9}] failed (attempt {attempt + 1}/{self.max_retries}). "
+                        f"Error: {e}. Retrying in 5s..."
+                    )
+                    if attempt < self.max_retries - 1:
+                        await asyncio.sleep(5)
+                    else:
+                        logger.error(f"aio_do_parse failed after {self.max_retries} attempts.")
+                        raise  # 耗尽次数后抛出异常，交给上层 execute 处理
+            if os.path.exists(parse_dir):
+                content += get_infer_result(".md", filename_without_ext, parse_dir)
+                self.save_images(parse_dir, sample["dataset_id"], os.path.abspath(sample[self.export_path_key]) + "/images")
+                shutil.rmtree(parse_dir)
+        return content
+
+    def save_images(self, parse_dir, dataset_id, export_path):
+        Path(export_path).mkdir(parents=True, exist_ok=True)
+
+        images_dir = os.path.join(parse_dir, "images")
+        image_paths = glob.glob(os.path.join(glob.escape(images_dir), "*.jpg"))
+        for image_path in image_paths:
+            shutil.copy(image_path, export_path)
+            image_sample = {}
+            image = Path(image_path)
+            image_name = image.name
+            image_sample[self.filename_key] = image_name
+            image_sample[self.filetype_key] = "jpg"
+            image_sample[self.filesize_key] = image.stat().st_size
+            image_sample["dataset_id"] = dataset_id
+            image_sample[self.filepath_key] = export_path + "/" + image_name
+            TaskInfoPersistence().update_file_result(image_sample, str(uuid.uuid4()))
@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from deepdata.core.base_op import OPERATORS
+
+OPERATORS.register_module(module_name='SlideFormatter',
+                          module_path="ops.formatter.slide_formatter.process")
@@ -0,0 +1,16 @@
+name: '病理图片内容抽取'
+name_en: 'Pathology Image Content Extraction'
+description: '解析病理图片。'
+description_en: 'Analyze pathological images.'
+language: 'python'
+vendor: 'huawei'
+raw_id: 'SlideFormatter'
+version: '1.0.0'
+types:
+  - 'collect'
+modal: 'image'
+effect:
+  before: ''
+  after: ''
+inputs: 'image'
+outputs: 'image'
@@ -0,0 +1,36 @@
+# -- encoding: utf-8 --
+
+"""
+Description: 医疗图片解析载入
+Create: 2025/02/08 11:00
+"""
+
+import time
+from typing import Dict, Any
+
+from loguru import logger
+
+from deepdata.core.base_op import Mapper
+
+
+class SlideFormatter(Mapper):
+
+    def __init__(self, *args, **kwargs):
+        super(SlideFormatter, self).__init__(*args, **kwargs)
+
+    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
+        '''
+        Read medical image and corresponding mask file, each as Image type in COntent value. Return Content.
+        '''
+        start = time.time()
+        file_type = sample[self.filetype_key]
+        types_openslide = ['svs', 'tif', 'dcm', 'vms', 'vmu',
+                           'ndpi', 'scn', 'mrxs', 'tiff', 'svslide',
+                           'bif', 'czi', 'sdpc']
+        if file_type not in types_openslide:
+            raise TypeError(f"Format not supported: {file_type}. Supported formats are: {', '.join(types_openslide)}.")
+
+        file_name = sample[self.filename_key]
+        logger.info(f"fileName: {file_name}, method: SlideFormatter costs {(time.time() - start):6f} s")
+        # Not really loading the slide, instead, use path as lazy loading.
+        return sample