deepdata: core product code (backend, frontend, runtime, deepflow module)

2026-08-30 19:23:38 +08:00
commit bd3db30be9
1405 changed files with 271727 additions and 0 deletions
@@ -0,0 +1,15 @@
+"""
+Operator File Parsers
+算子文件解析器
+"""
+from .abstract_parser import AbstractParser
+from .tar_parser import TarParser
+from .zip_parser import ZipParser
+from .parser_holder import ParserHolder
+
+__all__ = [
+    "AbstractParser",
+    "TarParser",
+    "ZipParser",
+    "ParserHolder",
+]
@@ -0,0 +1,129 @@
+"""
+Abstract Parser
+抽象解析器基类
+"""
+import json
+import yaml
+from abc import ABC, abstractmethod
+from typing import Dict, Any, Optional
+
+from app.module.operator.schema import OperatorDto, OperatorReleaseDto
+from app.module.operator.constants import CATEGORY_MAP, CATEGORY_OTHER_VENDOR_ID, CATEGORY_CUSTOMIZED_ID, \
+    CATEGORY_CLEANING_ID
+from app.module.operator.exceptions import FieldNotFoundError
+
+
+class AbstractParser(ABC):
+    """算子文件解析器抽象基类"""
+
+    @abstractmethod
+    def parse_yaml_from_archive(
+        self,
+        archive_path: str,
+        entry_path: str,
+        file_name: Optional[str] = None,
+        file_size: Optional[int] = None
+    ) -> OperatorDto:
+        """
+        从压缩包内读取指定路径的 yaml 文件并解析为 OperatorDto
+
+        Args:
+            archive_path: 压缩包路径（zip 或 tar）
+            entry_path: 压缩包内部的文件路径，例如 "config/app.yaml"
+
+        Returns:
+            解析后的 OperatorDto
+        """
+        pass
+
+    @abstractmethod
+    def extract_to(self, archive_path: str, target_dir: str) -> None:
+        """
+        将压缩包解压到目标目录（保持相对路径）
+
+        Args:
+            archive_path: 压缩包路径
+            target_dir: 目标目录
+        """
+        pass
+
+    def parse_yaml(
+        self,
+        yaml_content: str,
+        file_name: Optional[str] = None,
+        file_size: Optional[int] = None
+    ) -> OperatorDto:
+        """解析 YAML 内容为 OperatorDto"""
+        content: Dict[str, Any] = yaml.safe_load(yaml_content)
+
+        operator = OperatorDto(
+            id=self._to_string(content.get("raw_id")),
+            name=self._to_string(content.get("name")),
+            description=self._to_string(content.get("description")),
+            version=self._to_string(content.get("version")),
+            inputs=self._to_json(content.get("inputs")),
+            outputs=self._to_json(content.get("outputs")),
+            runtime=self._to_json(content.get("runtime")),
+            settings=self._to_json(content.get("settings")),
+            metrics=self._to_json(content.get("metrics")),
+            file_name=file_name,
+            file_size=file_size,
+        )
+
+        # Handle changelog
+        changelog = content.get("release")
+        if isinstance(changelog, list):
+            operator_release = OperatorReleaseDto(
+                id=operator.id,
+                version=operator.version,
+                changelog=changelog
+            )
+        else:
+            operator_release = OperatorReleaseDto(
+                id=operator.id,
+                version=operator.version,
+                changelog=[]
+            )
+        operator.releases = [operator_release]
+
+        # Build categories
+        categories = []
+        types = content.get("types")
+        if isinstance(types, list):
+            for t in types:
+                if self._to_lower(t) in CATEGORY_MAP:
+                    categories.append(CATEGORY_MAP[self._to_lower(t)])
+        if len(categories) == 0:
+            categories.append(CATEGORY_CLEANING_ID)
+
+        categories.extend([
+            CATEGORY_MAP.get(self._to_lower(content.get("language")), ""),
+            CATEGORY_MAP.get(self._to_lower(content.get("modal")), ""),
+            CATEGORY_MAP.get(self._to_lower(content.get("vendor")), CATEGORY_OTHER_VENDOR_ID),
+            CATEGORY_CUSTOMIZED_ID,
+        ])
+
+        operator.categories = categories
+
+        return operator
+
+    def _to_string(self, obj: Any) -> str:
+        """转换为字符串"""
+        if obj is None:
+            raise FieldNotFoundError("field")
+        return str(obj)
+
+    def _to_lower(self, obj: Any) -> str:
+        """转换为小写字符串"""
+        if obj is None:
+            raise FieldNotFoundError("field")
+        return str(obj).lower()
+
+    def _to_json(self, obj: Any) -> Optional[str]:
+        """转换为 JSON 字符串"""
+        if obj is None:
+            return None
+        try:
+            return json.dumps(obj).strip('"').strip("'")
+        except (TypeError, ValueError) as e:
+            raise ValueError(f"Failed to serialize to JSON: {e}")
@@ -0,0 +1,59 @@
+"""
+Parser Holder
+解析器持有者，根据文件类型选择合适的解析器
+"""
+import os
+from typing import Dict, Type, Optional
+
+from app.module.operator.parsers.abstract_parser import AbstractParser
+from app.module.operator.parsers.tar_parser import TarParser
+from app.module.operator.parsers.zip_parser import ZipParser
+from app.module.operator.schema import OperatorDto
+
+
+class ParserHolder:
+    """解析器持有者，根据文件类型选择解析器"""
+
+    def __init__(self):
+        self._parsers: Dict[str, AbstractParser] = {
+            "tar": TarParser(),
+            "gz": TarParser(),
+            "tgz": TarParser(),
+            "zip": ZipParser(),
+        }
+
+    def get_parser(self, file_path: str) -> AbstractParser:
+        """根据文件扩展名获取解析器"""
+        _, ext = os.path.splitext(file_path)
+        file_type = ext.lstrip('.').lower()
+
+        if file_type not in self._parsers:
+            raise ValueError(f"Unsupported file type: {file_type}")
+
+        return self._parsers[file_type]
+
+    def parse_yaml_from_archive(
+        self,
+        file_type: str,
+        archive_path: str,
+        entry_path: str,
+        file_name: Optional[str] = None,
+        file_size: Optional[int] = None
+    ) -> OperatorDto:
+        """从压缩包解析 YAML"""
+        if file_type not in self._parsers:
+            raise ValueError(f"Unsupported file type: {file_type}")
+
+        return self._parsers[file_type].parse_yaml_from_archive(
+            archive_path,
+            entry_path,
+            file_name,
+            file_size
+        )
+
+    def extract_to(self, file_type: str, archive_path: str, target_dir: str) -> None:
+        """解压文件到目标目录"""
+        if file_type not in self._parsers:
+            raise ValueError(f"Unsupported file type: {file_type}")
+
+        self._parsers[file_type].extract_to(archive_path, target_dir)
@@ -0,0 +1,47 @@
+"""
+Tar File Parser
+TAR 文件解析器
+"""
+import tarfile
+import os
+from typing import Optional
+
+from app.module.operator.parsers.abstract_parser import AbstractParser
+from app.module.operator.schema import OperatorDto
+
+
+class TarParser(AbstractParser):
+    """TAR 压缩包解析器"""
+
+    def parse_yaml_from_archive(
+        self,
+        archive_path: str,
+        entry_path: str,
+        file_name: Optional[str] = None,
+        file_size: Optional[int] = None
+    ) -> OperatorDto:
+        """从 TAR 文件中解析 YAML"""
+        try:
+            with tarfile.open(archive_path, 'r:*') as tar:
+                for member in tar.getmembers():
+                    if member.name == entry_path or member.name.endswith(f"/{entry_path}"):
+                        file = tar.extractfile(member)
+                        if file:
+                            content = file.read().decode('utf-8')
+                            return self.parse_yaml(content, file_name, file_size)
+            raise FileNotFoundError(f"File '{entry_path}' not found in archive")
+        except (tarfile.TarError, EOFError) as e:
+            raise ValueError(f"Failed to parse TAR file: {e}")
+
+    def extract_to(self, archive_path: str, target_dir: str) -> None:
+        """解压 TAR 文件到目标目录"""
+        try:
+            os.makedirs(target_dir, exist_ok=True)
+            with tarfile.open(archive_path, 'r:*') as tar:
+                # Safety check: prevent path traversal
+                for member in tar.getmembers():
+                    if os.path.isabs(member.name) or ".." in member.name.split("/"):
+                        raise ValueError(f"Unsafe path in archive: {member.name}")
+                tar.extractall(target_dir)
+        except (tarfile.TarError, EOFError) as e:
+            raise ValueError(f"Failed to extract TAR file: {e}")
@@ -0,0 +1,46 @@
+"""
+Zip File Parser
+ZIP 文件解析器
+"""
+import zipfile
+import os
+from typing import Optional
+
+from app.module.operator.parsers.abstract_parser import AbstractParser
+from app.module.operator.schema import OperatorDto
+
+
+class ZipParser(AbstractParser):
+    """ZIP 压缩包解析器"""
+
+    def parse_yaml_from_archive(
+        self,
+        archive_path: str,
+        entry_path: str,
+        file_name: Optional[str] = None,
+        file_size: Optional[int] = None
+    ) -> OperatorDto:
+        """从 ZIP 文件中解析 YAML"""
+        try:
+            with zipfile.ZipFile(archive_path, 'r') as zf:
+                for name in zf.namelist():
+                    if name == entry_path or name.endswith(f"/{entry_path}"):
+                        with zf.open(name) as file:
+                            content = file.read().decode('utf-8')
+                            return self.parse_yaml(content, file_name, file_size)
+            raise FileNotFoundError(f"File '{entry_path}' not found in archive")
+        except (zipfile.BadZipFile, zipfile.LargeZipFile) as e:
+            raise ValueError(f"Failed to parse ZIP file: {e}")
+
+    def extract_to(self, archive_path: str, target_dir: str) -> None:
+        """解压 ZIP 文件到目标目录"""
+        try:
+            os.makedirs(target_dir, exist_ok=True)
+            with zipfile.ZipFile(archive_path, 'r') as zf:
+                # Safety check: prevent path traversal
+                for name in zf.namelist():
+                    if os.path.isabs(name) or ".." in name.split("/"):
+                        raise ValueError(f"Unsafe path in archive: {name}")
+                zf.extractall(target_dir)
+        except (zipfile.BadZipFile, zipfile.LargeZipFile) as e:
+            raise ValueError(f"Failed to extract ZIP file: {e}")