deepdata: core product code (backend, frontend, runtime, deepflow module)

2026-08-30 19:23:38 +08:00
commit bd3db30be9
1405 changed files with 271727 additions and 0 deletions
@@ -0,0 +1,21 @@
+"""
+Shared Module Init
+共享模块初始化
+"""
+from .file_service import FileService
+from .file_models import (
+    ChunkUploadPreRequestDto,
+    ChunkUploadRequestDto,
+    FileUploadResult,
+)
+from .chunks_saver import ChunksSaver
+from .chunk_upload_repository import ChunkUploadRepository
+
+__all__ = [
+    "FileService",
+    "ChunkUploadPreRequestDto",
+    "ChunkUploadRequestDto",
+    "FileUploadResult",
+    "ChunksSaver",
+    "ChunkUploadRepository",
+]
@@ -0,0 +1,95 @@
+"""
+Chunk Upload Repository
+分片上传数据访问层
+"""
+from typing import Optional, List
+
+from sqlalchemy import select, update, delete
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db.models.chunk_upload import ChunkUploadPreRequest
+from app.core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class ChunkUploadRepository:
+    """分片上传数据访问层"""
+
+    async def find_by_id(
+        self,
+        req_id: str,
+        db: AsyncSession
+    ) -> Optional[ChunkUploadPreRequest]:
+        """根据ID查询"""
+        result = await db.execute(
+            select(ChunkUploadPreRequest).where(ChunkUploadPreRequest.id == req_id)
+        )
+        return result.scalar_one_or_none()
+
+    async def find_by_service_id(
+        self,
+        service_id: str,
+        db: AsyncSession
+    ) -> List[ChunkUploadPreRequest]:
+        """根据服务ID查询"""
+        result = await db.execute(
+            select(ChunkUploadPreRequest).where(
+                ChunkUploadPreRequest.service_id == service_id
+            )
+        )
+        return result.scalars().all()
+
+    async def find_all(self, db: AsyncSession) -> List[ChunkUploadPreRequest]:
+        """查询所有"""
+        result = await db.execute(select(ChunkUploadPreRequest))
+        return result.scalars().all()
+
+    async def insert(
+        self,
+        request: ChunkUploadPreRequest,
+        db: AsyncSession
+    ) -> None:
+        """插入"""
+        db.add(request)
+
+    async def update(
+        self,
+        request: ChunkUploadPreRequest,
+        db: AsyncSession
+    ) -> int:
+        """更新"""
+        from datetime import datetime, timezone
+        result = await db.execute(
+            update(ChunkUploadPreRequest)
+            .where(ChunkUploadPreRequest.id == request.id)
+            .values(
+                uploaded_file_num=request.uploaded_file_num,
+                timeout=request.timeout,
+            )
+        )
+        return result.rowcount
+
+    async def delete_by_id(
+        self,
+        req_id: str,
+        db: AsyncSession
+    ) -> int:
+        """根据ID删除"""
+        result = await db.execute(
+            delete(ChunkUploadPreRequest).where(ChunkUploadPreRequest.id == req_id)
+        )
+        return result.rowcount
+
+    async def delete_by_service_id(
+        self,
+        service_id: str,
+        db: AsyncSession
+    ) -> int:
+        """根据服务ID删除"""
+        result = await db.execute(
+            delete(ChunkUploadPreRequest).where(
+                ChunkUploadPreRequest.service_id == service_id
+            )
+        )
+        return result.rowcount
@@ -0,0 +1,166 @@
+"""
+Chunks Saver
+分片保存器，用于处理文件分片上传
+"""
+import os
+from pathlib import Path
+from typing import Optional
+from datetime import datetime, timezone
+
+from fastapi import UploadFile
+
+from app.core.logging import get_logger
+from app.module.shared.file_models import ChunkUploadRequestDto
+
+logger = get_logger(__name__)
+
+
+class ChunksSaver:
+    """分片保存器"""
+
+    TEMP_DIR_NAME_FORMAT = "req_%s_chunks"
+
+    @staticmethod
+    def save(
+        file_upload_request: ChunkUploadRequestDto,
+        pre_upload_req_id: str,
+        upload_path: str,
+        file_content: bytes
+    ) -> Optional[Path]:
+        """
+        保存分片
+
+        Args:
+            file_upload_request: 上传分片的请求
+            pre_upload_req_id: 预上传请求ID
+            upload_path: 上传基础路径
+            file_content: 文件内容（字节）
+
+        Returns:
+            保存后的文件路径，如果不是最后一个分片则返回None
+        """
+        start_time = datetime.now(timezone.utc)
+
+        temp_dir = Path(upload_path) / (
+            ChunksSaver.TEMP_DIR_NAME_FORMAT % pre_upload_req_id
+        )
+        temp_dir.mkdir(parents=True, exist_ok=True)
+
+        temp_file = temp_dir / str(file_upload_request.file_no)
+
+        ChunksSaver._append_to_target_file(temp_file, file_content)
+
+        if file_upload_request.total_chunk_num != file_upload_request.chunk_no:
+            elapsed = (datetime.now(timezone.utc) - start_time).total_seconds()
+            logger.debug(f"save chunk {file_upload_request.chunk_no} cost {elapsed}s")
+            return None
+
+        final_file = Path(upload_path) / file_upload_request.file_name
+
+        # 防止路径穿越：规范化后校验仍在 upload_path 下
+        resolved_final = final_file.resolve()
+        resolved_base = Path(upload_path).resolve()
+        if not str(resolved_final).startswith(str(resolved_base) + os.sep):
+            logger.error(
+                f"Path traversal attempt in chunk save: file_name={file_upload_request.file_name}, "
+                f"resolved_path={resolved_final}"
+            )
+            raise ValueError("Path traversal detected in file name")
+
+        try:
+            temp_file.rename(final_file)
+        except OSError as e:
+            logger.error(
+                f"failed to mv file: {temp_file.name}, req id: {pre_upload_req_id}, error: {e}"
+            )
+            raise ValueError("failed to move file to target dir") from e
+
+        elapsed = (datetime.now(timezone.utc) - start_time).total_seconds()
+        logger.debug(f"save chunk {file_upload_request.chunk_no} cost {elapsed}s")
+
+        return final_file
+
+    @staticmethod
+    def save_file(
+        file_upload_request: ChunkUploadRequestDto,
+        upload_path: str,
+        file_content: bytes
+    ) -> Path:
+        """
+        保存文件（不分片）
+
+        Args:
+            file_upload_request: 上传请求
+            upload_path: 上传路径
+            file_content: 文件内容（字节）
+
+        Returns:
+            保存后的文件路径
+        """
+        target_file = Path(upload_path) / file_upload_request.file_name
+
+        # 防止路径穿越：规范化后校验仍在 upload_path 下
+        resolved_target = target_file.resolve()
+        resolved_base = Path(upload_path).resolve()
+        if not str(resolved_target).startswith(str(resolved_base) + os.sep):
+            logger.error(
+                f"Path traversal attempt in save_file: file_name={file_upload_request.file_name}, "
+                f"resolved_path={resolved_target}"
+            )
+            raise ValueError("Path traversal detected in file name")
+
+        logger.info(f"file path {target_file}, file size {len(file_content)}")
+
+        try:
+            target_file.parent.mkdir(parents=True, exist_ok=True)
+            target_file.write_bytes(file_content)
+        except OSError as e:
+            logger.error(f"failed to save file: {target_file}, error: {e}")
+            raise ValueError("failed to save file") from e
+
+        return target_file
+
+    @staticmethod
+    def delete_folder(folder_path: str) -> None:
+        """
+        删除指定路径下的所有文件
+
+        Args:
+            folder_path: 文件夹路径
+        """
+        folder = Path(folder_path)
+
+        if not folder.exists():
+            logger.info(f"folder {folder_path} does not exist")
+            return
+
+        try:
+            for item in folder.glob("*"):
+                if item.is_file():
+                    item.unlink()
+                elif item.is_dir():
+                    for sub_item in item.glob("*"):
+                        if sub_item.is_file():
+                            sub_item.unlink()
+                        elif sub_item.is_dir():
+                            ChunksSaver.delete_folder(str(sub_item))
+                    item.rmdir()
+        except OSError as e:
+            logger.error(f"failed to delete folder: {folder_path}, error: {e}")
+            raise ValueError("failed to delete folder") from e
+
+    @staticmethod
+    def _append_to_target_file(target_file: Path, content: bytes) -> None:
+        """
+        追加内容到目标文件末尾
+
+        Args:
+            target_file: 目标文件
+            content: 要追加的内容
+        """
+        try:
+            with open(target_file, "ab") as f:
+                f.write(content)
+        except OSError as e:
+            logger.error(f"failed to append to file: {target_file}, error: {e}")
+            raise ValueError("failed to append content to file") from e
@@ -0,0 +1,3 @@
+"""
+公共模块
+"""
@@ -0,0 +1,115 @@
+from pathlib import Path
+from typing import List, Union, Optional
+
+from langchain_community.document_loaders import (
+    TextLoader,
+    JSONLoader,
+    CSVLoader,
+    UnstructuredMarkdownLoader,
+    UnstructuredFileLoader,
+    UnstructuredExcelLoader,
+    PyPDFLoader,
+    Docx2txtLoader, BSHTMLLoader
+)
+from langchain_core.documents import Document
+
+from app.core.logging import get_logger
+
+log = get_logger(__name__)
+
+class UniversalDocLoader:
+    """
+    通用泛文本文档加载类
+    支持格式：TXT/JSON/CSV/Markdown/Word(.docx)/PPT(.pptx)/PDF/Excel(.xlsx,.xls)/各种文本文件
+    """
+    # 格式-加载器映射（轻量优先）
+    SUPPORTED_FORMATS = {
+        # 纯文本类
+        ".txt": TextLoader,
+        ".json": JSONLoader,
+        ".csv": CSVLoader,
+        ".md": UnstructuredMarkdownLoader,
+        ".markdown": UnstructuredMarkdownLoader,
+        ".log": TextLoader,
+        ".xml": TextLoader,
+        ".html": BSHTMLLoader,
+        ".htm": BSHTMLLoader,
+        ".yaml": TextLoader,
+        ".yml": TextLoader,
+        # 办公文档类
+        ".docx": Docx2txtLoader,
+        ".doc": Docx2txtLoader,
+        ".pptx": UnstructuredFileLoader,
+        ".ppt": UnstructuredFileLoader,
+        ".xlsx": UnstructuredExcelLoader,
+        ".xls": UnstructuredExcelLoader,
+        # PDF 类
+        ".pdf": PyPDFLoader
+    }
+
+    def __init__(self, file_path: Union[str, Path]):
+        self.file_path = Path(file_path).resolve()
+        self.file_suffix = self.file_path.suffix.lower()
+        log.info(f"初始化文档加载器: {self.file_path} (格式: {self.file_suffix})")
+        self._validate_file()
+
+    def _validate_file(self) -> None:
+        """验证文件存在性和格式支持性"""
+        if not self.file_path.exists():
+            raise FileNotFoundError(f"文件不存在: {self.file_path}")
+        if self.file_suffix not in self.SUPPORTED_FORMATS:
+            raise ValueError(
+                f"不支持的格式: {self.file_suffix} | 支持格式: {list(self.SUPPORTED_FORMATS.keys())}"
+            )
+
+    def load(
+        self,
+        file_format: Optional[str] = None,
+        **loader_kwargs
+    ) -> List[Document]:
+        """
+        加载文档并返回 LangChain Document 列表
+        :param file_format: 手动指定格式（如 ".pdf"），默认自动识别
+        :param loader_kwargs: 传递给具体加载器的参数（如 JSONLoader 的 jq_schema）
+        :return: List[Document]
+        """
+        # 确定目标格式
+        target_format = file_format.lower() if file_format else self.file_suffix
+        loader_cls = self.SUPPORTED_FORMATS[target_format]
+
+        # 加载器默认参数优化
+        loader_kwargs = self._set_default_kwargs(loader_cls, loader_kwargs)
+
+        # 初始化并加载
+        loader = loader_cls(str(self.file_path), **loader_kwargs)
+        return loader.load()
+
+    @staticmethod
+    def _set_default_kwargs(loader_cls, kwargs: dict) -> dict:
+        """为不同加载器设置默认参数，简化调用"""
+        if loader_cls == JSONLoader and "jq_schema" not in kwargs:
+            kwargs.setdefault("jq_schema", ".")
+            kwargs.setdefault("text_content", False)
+        if loader_cls == CSVLoader and "csv_args" not in kwargs:
+            kwargs["csv_args"] = {"delimiter": ","}
+        if loader_cls == UnstructuredExcelLoader:
+            # Excel 文件使用 "single" 模式，避免生成过多无意义的元素
+            # "elements" 模式会为每个单元格生成单独的文档，可能导致大量空内容
+            if "mode" not in kwargs:
+                kwargs.setdefault("mode", "single")
+            # 确保加载表格结构
+            kwargs.setdefault("include_header", True)
+        if loader_cls == UnstructuredFileLoader and "mode" not in kwargs:
+            kwargs.setdefault("mode", "elements")
+        return kwargs
+
+
+# 文档加载器便捷函数
+def load_documents(
+    file_path: Union[str, Path],
+    file_format: Optional[str] = None,
+    **loader_kwargs
+) -> List[Document]:
+    """快速加载文档的便捷函数"""
+    loader = UniversalDocLoader(file_path)
+    return loader.load(file_format=file_format, **loader_kwargs)
@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+from typing import List, Optional
+from uuid import uuid4
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.logging import get_logger
+from app.db.models.base_entity import LineageNode, LineageEdge
+
+log = get_logger(__name__)
+
+
+class LineageService:
+    """血缘服务（Python 版）"""
+
+    def __init__(self, db: AsyncSession):
+        self.db = db
+
+    async def generate_graph(
+        self,
+        from_node: Optional[LineageNode],
+        edge: Optional[LineageEdge],
+        to_node: Optional[LineageNode]
+    ) -> None:
+        """
+        生成血缘图
+
+        Args:
+            from_node: 源节点
+            edge: 边
+            to_node: 目的节点
+        """
+        # 1. 如果 from_node 为空，直接返回不做任何处理
+        if from_node is None:
+            return
+
+        # 2. 如果 from_node 有值，检查 from_node 是否存在
+        existing_from = await self.get_node_by_id(from_node.id) if from_node.id else None
+
+        if existing_from is None:
+            # from_node 不存在，创建 from_node，记录 from_graph_id
+            from_graph_id = str(uuid4())
+            if not from_node.id:
+                from_node.id = str(uuid4())
+            from_node.graph_id = from_graph_id
+            self.db.add(from_node)
+            await self.db.flush()
+        else:
+            # from_node 存在，记录 from_graph_id
+            from_graph_id = existing_from.graph_id
+
+        # 处理 edge 和 to_node
+        await self._generate_graph(edge, to_node, from_graph_id)
+
+    async def _generate_graph(
+        self,
+        edge: Optional[LineageEdge],
+        to_node: Optional[LineageNode],
+        graph_id: Optional[str]
+    ) -> None:
+        # 无 edge，直接返回
+        if edge is None:
+            return
+
+        # 有 edge，处理 edge，检查是否有 to_node
+        await self._handle_lineage_edge(graph_id, edge)
+        if to_node is None:
+            return
+
+        # 有 to_node，检查 to_node 是否存在
+        existing_to = await self.get_node_by_id(to_node.id) if to_node.id else None
+
+        if existing_to is None:
+            # to_node 不存在，创建 to_node 后结束
+            if not to_node.id:
+                to_node.id = str(uuid4())
+            to_node.graph_id = graph_id
+            self.db.add(to_node)
+            await self.db.flush()
+        else:
+            # to_node 存在，将 from_node 所在的图并入 to_node 所在的图后结束
+            await self._merge_graph(graph_id, existing_to.graph_id)
+
+    async def _handle_lineage_edge(self, graph_id: Optional[str], edge: LineageEdge) -> None:
+        stmt = select(LineageEdge).where(
+            LineageEdge.graph_id == graph_id,
+            LineageEdge.from_node_id == edge.from_node_id,
+            LineageEdge.to_node_id == edge.to_node_id
+        )
+        result = await self.db.execute(stmt)
+        existing = result.scalar_one_or_none()
+
+        if existing is None:
+            if not edge.id:
+                edge.id = str(uuid4())
+            edge.graph_id = graph_id
+            self.db.add(edge)
+            await self.db.flush()
+        else:
+            edge.id = existing.id
+            if edge.graph_id is None:
+                edge.graph_id = existing.graph_id
+            # 覆盖更新现有记录
+            existing.graph_id = edge.graph_id
+            existing.process_id = edge.process_id
+            existing.edge_type = edge.edge_type
+            existing.name = edge.name
+            existing.description = edge.description
+            existing.edge_metadata = edge.edge_metadata
+            existing.from_node_id = edge.from_node_id
+            existing.to_node_id = edge.to_node_id
+            await self.db.flush()
+
+    async def _merge_graph(self, from_graph_id: Optional[str], to_graph_id: Optional[str]) -> None:
+        if not from_graph_id or not to_graph_id or from_graph_id == to_graph_id:
+            return
+
+        from_nodes = await self.get_nodes_by_graph_id(from_graph_id)
+        to_nodes = await self.get_nodes_by_graph_id(to_graph_id)
+
+        # choose smaller graph as source, larger as target
+        source_graph_id = from_graph_id if len(from_nodes) <= len(to_nodes) else to_graph_id
+        target_graph_id = to_graph_id if source_graph_id == from_graph_id else from_graph_id
+        source_nodes = from_nodes if source_graph_id == from_graph_id else to_nodes
+
+        # update nodes' graph_id
+        for node in source_nodes:
+            if node is None:
+                continue
+            node.graph_id = target_graph_id
+            self.db.add(node)
+
+        # update edges' graph_id (edges belonging to the source graph)
+        edges = await self.get_edges_by_graph_id(source_graph_id)
+        for edge in edges:
+            if edge is None:
+                continue
+            edge.graph_id = target_graph_id
+            self.db.add(edge)
+
+        await self.db.flush()
+
+    async def get_nodes_by_graph_id(self, graph_id: str) -> List[LineageNode]:
+        """从图ID获取图的节点列表"""
+        stmt = select(LineageNode).where(LineageNode.graph_id == graph_id)
+        result = await self.db.execute(stmt)
+        return list(result.scalars().all())
+
+    async def get_edges_by_graph_id(self, graph_id: str) -> List[LineageEdge]:
+        """从图ID获取图的边列表"""
+        stmt = select(LineageEdge).where(LineageEdge.graph_id == graph_id)
+        result = await self.db.execute(stmt)
+        return list(result.scalars().all())
+
+    async def get_node_by_id(self, node_id: str) -> Optional[LineageNode]:
+        """从节点ID获取节点"""
+        return await self.db.get(LineageNode, node_id)
@@ -0,0 +1,169 @@
+import os
+from typing import List, Optional, Tuple
+
+from langchain_core.documents import Document
+from langchain_text_splitters import (
+    RecursiveCharacterTextSplitter,
+    MarkdownHeaderTextSplitter
+)
+
+
+class DocumentSplitter:
+    """
+    文档分割器类 - 增强版，优先通过元数据识别文档类型
+    核心特性：
+    1. 优先从metadata的source字段（文件扩展名）识别Markdown
+    2. 元数据缺失时，通过内容特征降级检测
+    3. 支持CJK（中日韩）语言优化
+    """
+
+    def __init__(
+        self,
+        chunk_size: int = 2000,
+        chunk_overlap: int = 200,
+        is_cjk_language: bool = True,
+        markdown_headers: Optional[List[Tuple[str, str]]] = None
+    ):
+        """
+        初始化文档分割器
+
+        Args:
+            chunk_size: 每个文本块的最大长度（默认2000字符）
+            chunk_overlap: 文本块之间的重叠长度（默认200字符）
+            is_cjk_language: 是否处理中日韩等无词边界语言（默认True）
+            markdown_headers: Markdown标题分割规则（默认：#/##/###/####）
+        """
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.is_cjk_language = is_cjk_language
+
+        # 默认Markdown标题分割规则
+        self.markdown_headers = markdown_headers or [
+            ("#", "header_1"),
+            ("##", "header_2"),
+            ("###", "header_3"),
+            ("####", "header_4"),
+        ]
+
+        # 初始化基础文本分割器
+        self.text_splitter = self._create_text_splitter()
+
+    def _create_text_splitter(self) -> RecursiveCharacterTextSplitter:
+        """创建递归字符分割器（内部方法）"""
+        # 优化后的CJK分隔符列表（修复语法错误，调整优先级）
+        if self.is_cjk_language:
+            separators = [
+                "\n\n", "\n",  # 段落/换行（最高优先级）
+                "。", ".",  # 句号（中文/英文）
+                "！", "!",  # 感叹号（中文/英文）
+                "？", "?",  # 问号（中文/英文）
+                "；", ";",  # 分号（中文/英文）
+                "，", ",",  # 逗号（中文/英文）
+                "、",  # 顿号（中文）
+                "：", ":",  # 冒号（中文/英文）
+                " ",  # 空格
+                "\u200b", "",  # 零宽空格/兜底
+            ]
+        else:
+            separators = ["\n\n", "\n", " ", ".", "!", "?", ";", ":", ",", ""]
+
+        return RecursiveCharacterTextSplitter(
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap,
+            separators=separators,
+            length_function=len,
+            is_separator_regex=False
+        )
+
+    @staticmethod
+    def _is_markdown(doc: Document) -> bool:
+        """
+        优先从元数据判断是否为Markdown
+        规则：检查metadata中的source字段扩展名是否为.md/.markdown/.mdx等
+        """
+        # 获取source字段（忽略大小写）
+        source = doc.metadata.get("source", "").lower()
+        if not source:
+            return False
+
+        # 获取文件扩展名
+        ext = os.path.splitext(source)[-1].lower()
+        # Markdown常见扩展名列表
+        md_ext = [".md", ".markdown", ".mdx", ".mkd", ".mkdown"]
+        return ext in md_ext
+
+    def split(self, documents: List[Document], is_markdown: bool = False) -> List[Document]:
+        """
+        核心分割方法
+
+        Args:
+            documents: 待分割的Document列表
+            is_markdown: 是否为Markdown文档（默认False）
+
+        Returns:
+            分割后的Document列表
+        """
+        if not documents:
+            return []
+
+        # Markdown文档处理：先按标题分割，再按字符分割
+        if is_markdown:
+            # 初始化Markdown标题分割器
+            md_splitter = MarkdownHeaderTextSplitter(
+                headers_to_split_on=self.markdown_headers,
+                strip_headers=True,
+                return_each_line=False
+            )
+
+            # 按标题分割并继承元数据
+            md_chunks = []
+            for doc in documents:
+                chunks = md_splitter.split_text(doc.page_content)
+                for chunk in chunks:
+                    chunk.metadata.update(doc.metadata)
+                md_chunks.extend(chunks)
+
+            # 对标题分割后的内容进行字符分割
+            final_chunks = self.text_splitter.split_documents(md_chunks)
+
+        # 普通文本直接分割
+        else:
+            final_chunks = self.text_splitter.split_documents(documents)
+
+        return final_chunks
+
+    # 核心自动分割方法（元数据优先）
+    @classmethod
+    def auto_split(
+        cls,
+        documents: List[Document],
+        chunk_size: int = 2000,
+        chunk_overlap: int = 200
+    ) -> List[Document]:
+        """
+        极简快捷方法：自动识别文档类型并分割（元数据优先）
+        仅需传入3个参数，无需初始化类实例
+
+        Args:
+            documents: 待分割的Document列表
+            chunk_size: 每个文本块的最大长度（默认2000字符）
+            chunk_overlap: 文本块之间的重叠长度（默认200字符）
+
+        Returns:
+            分割后的Document列表
+        """
+        if not documents:
+            return []
+
+        # 初始化分割器实例（使用CJK默认优化）
+        splitter = cls(
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+            is_cjk_language=True
+        )
+
+        # 自动检测文档类型（元数据优先）
+        is_md = splitter._is_markdown(documents[0])
+
+        # 根据检测结果选择分割方式
+        return splitter.split(documents, is_markdown=is_md)
@@ -0,0 +1,38 @@
+"""
+File Models
+文件相关模型定义
+"""
+from pathlib import Path
+from typing import Optional
+from pydantic import BaseModel, Field
+from datetime import datetime
+
+
+class ChunkUploadPreRequestDto(BaseModel):
+    """分片上传预请求DTO"""
+    id: str = Field(..., description="请求ID")
+    total_file_num: int = Field(..., description="总文件数", ge=1)
+    uploaded_file_num: Optional[int] = Field(None, description="已上传文件数", ge=0)
+    upload_path: str = Field(..., description="文件路径")
+    timeout: Optional[datetime] = Field(None, description="上传请求超时时间")
+    service_id: Optional[str] = Field(None, description="上传请求所属服务ID")
+    check_info: Optional[str] = Field(None, description="业务信息")
+
+
+class ChunkUploadRequestDto(BaseModel):
+    """分片上传请求DTO"""
+    req_id: str = Field(..., description="预上传返回的ID")
+    file_no: int = Field(1, description="文件编号", ge=1)
+    file_name: str = Field(..., description="文件名称")
+    total_chunk_num: int = Field(1, description="总分块数量", ge=1)
+    chunk_no: int = Field(1, description="当前分块编号", ge=1)
+    file_size: Optional[int] = Field(None, description="文件大小", ge=0)
+    check_sum_hex: Optional[str] = Field(None, description="文件校验和（十六进制字符串）")
+
+
+class FileUploadResult(BaseModel):
+    """文件上传结果"""
+    is_all_files_uploaded: bool = Field(..., description="是否所有文件已上传")
+    check_info: Optional[str] = Field(None, description="业务上传信息")
+    saved_file_path: Optional[str] = Field(None, description="保存的文件路径")
+    file_name: str = Field(..., description="文件名称")
@@ -0,0 +1,183 @@
+"""
+File Service
+文件服务，处理文件上传、分片上传等功能
+"""
+import os
+import uuid
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Optional
+
+from app.core.logging import get_logger
+from app.db.models.chunk_upload import ChunkUploadPreRequest
+from app.module.shared.chunk_upload_repository import ChunkUploadRepository
+from app.module.shared.chunks_saver import ChunksSaver
+from app.module.shared.file_models import (
+    ChunkUploadRequestDto,
+    FileUploadResult,
+)
+
+logger = get_logger(__name__)
+
+
+class FileService:
+    """文件服务"""
+
+    DEFAULT_TIMEOUT_SECONDS = 120
+
+    def __init__(
+        self,
+        chunk_upload_repo: ChunkUploadRepository,
+    ):
+        self.chunk_upload_repo = chunk_upload_repo
+
+    async def pre_upload(
+        self,
+        upload_path: str,
+        service_id: str,
+        db_session,
+        check_info: Optional[str] = None
+    ) -> str:
+        """
+        预上传
+
+        Args:
+            upload_path: 上传路径
+            service_id: 服务ID
+            check_info: 业务信息
+
+        Returns:
+            预上传请求ID
+        """
+        req_id = str(uuid.uuid4())
+        timeout = datetime.utcnow().replace(
+            microsecond=0
+        ) + timedelta(seconds=self.DEFAULT_TIMEOUT_SECONDS)
+
+        pre_request = ChunkUploadPreRequest(
+            id=req_id,
+            total_file_num=1,
+            uploaded_file_num=0,
+            upload_path=upload_path,
+            timeout=timeout,
+            service_id=service_id,
+            check_info=check_info,
+        )
+
+        await self.chunk_upload_repo.insert(pre_request, db_session)
+        return req_id
+
+    async def chunk_upload(
+        self,
+        upload_request: ChunkUploadRequestDto,
+        upload_path: str,
+        file_content: bytes,
+        db_session,
+    ) -> FileUploadResult:
+        """
+        分片上传
+
+        Args:
+            upload_request: 上传请求
+            upload_path: 上传路径
+            file_content: 文件内容
+            db_session: 数据库会话
+
+        Returns:
+            上传结果
+        """
+        upload_request.file_size = len(file_content)
+
+        pre_request = await self.chunk_upload_repo.find_by_id(
+            upload_request.req_id, db_session
+        )
+
+        if pre_request is None:
+            logger.error(f"pre-upload request not found: {upload_request.req_id}")
+            raise ValueError("Pre-upload request not found")
+
+        if pre_request.is_upload_complete():
+            logger.error(f"upload already complete: {upload_request.req_id}")
+            raise ValueError("Upload already complete")
+
+        if pre_request.is_request_timeout():
+            logger.error(f"upload request timeout: {upload_request.req_id}")
+            raise ValueError("Upload request timeout")
+
+        saved_file_path = None
+
+        if upload_request.total_chunk_num > 1:
+            saved_file_path = await self._upload_chunk(
+                upload_request, pre_request, upload_path, file_content
+            )
+        else:
+            saved_file_path = await self._upload_file(
+                upload_request, pre_request, upload_path, file_content
+            )
+
+        update_count = await self.chunk_upload_repo.update(pre_request, db_session)
+
+        if update_count == 0:
+            logger.error(f"failed to update pre-request: {upload_request.req_id}")
+            raise ValueError("Failed to update pre-upload request")
+
+        is_finish = pre_request.uploaded_file_num == pre_request.total_file_num
+
+        if is_finish:
+            temp_dir = os.path.join(
+                upload_path,
+                ChunksSaver.TEMP_DIR_NAME_FORMAT % pre_request.id
+            )
+            try:
+                ChunksSaver.delete_folder(temp_dir)
+            except Exception as e:
+                logger.warning(f"failed to delete temp dir: {temp_dir}, error: {e}")
+
+            await self.chunk_upload_repo.delete_by_id(pre_request.id, db_session)
+
+        return FileUploadResult(
+            is_all_files_uploaded=is_finish,
+            check_info=pre_request.check_info,
+            saved_file_path=str(saved_file_path) if saved_file_path else None,
+            file_name=upload_request.file_name,
+        )
+
+    async def _upload_file(
+        self,
+        upload_request: ChunkUploadRequestDto,
+        pre_request: ChunkUploadPreRequest,
+        upload_path: str,
+        file_content: bytes
+    ) -> Path:
+        """上传单文件"""
+        saved_file = ChunksSaver.save_file(
+            upload_request, upload_path, file_content
+        )
+
+        pre_request.timeout = datetime.utcnow().replace(
+            microsecond=0
+        ) + timedelta(seconds=self.DEFAULT_TIMEOUT_SECONDS)
+        pre_request.increment_uploaded_file_num()
+
+        return saved_file
+
+    async def _upload_chunk(
+        self,
+        upload_request: ChunkUploadRequestDto,
+        pre_request: ChunkUploadPreRequest,
+        upload_path: str,
+        file_content: bytes
+    ) -> Optional[Path]:
+        """上传分片"""
+        saved_file = ChunksSaver.save(
+            upload_request, pre_request.id, upload_path, file_content
+        )
+
+        if saved_file is not None:
+            pre_request.increment_uploaded_file_num()
+            return saved_file
+
+        pre_request.timeout = datetime.utcnow().replace(
+            microsecond=0
+        ) + timedelta(seconds=self.DEFAULT_TIMEOUT_SECONDS)
+        return None
@@ -0,0 +1,7 @@
+# app/core/llm/__init__.py
+"""
+LangChain 模型工厂：统一创建 Chat、Embedding 及健康检查，便于各模块复用。
+"""
+from .factory import LLMFactory
+
+__all__ = ["LLMFactory"]
@@ -0,0 +1,124 @@
+# app/module/shared/llm/factory.py
+"""
+LangChain 模型工厂：基于 OpenAI 兼容接口封装 Chat / Embedding 的创建、健康检查与同步调用。
+- DashScope compatible‑mode：重写embedding逻辑，规避tiktoken token数组输入问题
+- 其他OpenAI兼容服务商：完全使用原生 OpenAIEmbeddings / ChatOpenAI
+"""
+from typing import Literal, List, Optional
+
+import httpx
+from langchain_core.language_models import BaseChatModel
+from langchain_core.embeddings import Embeddings
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+from pydantic import SecretStr
+
+
+class DashScopeOpenAIEmbeddings(OpenAIEmbeddings, Embeddings):
+    """
+    仅用于阿里云DashScope compatible‑mode
+    1. 绕过原生tiktoken逻辑，直接传入原始字符串，不传入token数字数组
+    2. 适配新版openai SDK返回Pydantic Embedding对象，使用属性 .embedding
+    """
+
+    def _get_len_safe_embeddings(self, texts: List[str], *, engine: str, **kwargs) -> List[List[float]]:
+        batch_size = self.chunk_size
+        embeddings: List[List[float]] = []
+        for i in range(0, len(texts), batch_size):
+            batch = texts[i:i + batch_size]
+            resp = self.client.create(
+                input=batch,
+                model=self.model,
+                encoding_format="float",
+            )
+            embeddings.extend([data.embedding for data in resp.data])
+        return embeddings
+
+
+class LLMFactory:
+    @staticmethod
+    def _is_dashscope_compatible(base_url: Optional[str]) -> bool:
+        if not base_url:
+            return False
+        return "dashscope.aliyuncs.com/compatible-mode" in base_url
+
+    @staticmethod
+    def create_chat(
+        model_name: str,
+        base_url: str,
+        api_key: str | None = None,
+    ) -> BaseChatModel:
+        """创建对话模型，兼容 OpenAI 及任意 base_url 的 OpenAI 兼容服务。"""
+        # DashScope容器环境关闭SSL校验；其余服务商使用默认httpx客户端（开启证书校验）
+        if LLMFactory._is_dashscope_compatible(base_url):
+            http_client = httpx.Client(verify=False)
+        else:
+            http_client = None
+
+        return ChatOpenAI(
+            model=model_name,
+            base_url=base_url or None,
+            api_key=SecretStr(api_key or ""),
+            http_client=http_client,
+        )
+
+    @staticmethod
+    def create_embedding(
+        model_name: str,
+        base_url: str,
+        api_key: str | None = None,
+    ) -> Embeddings:
+        is_dashscope = LLMFactory._is_dashscope_compatible(base_url)
+
+        # DashScope才关闭SSL校验，其他服务商走默认SSL校验
+        if is_dashscope:
+            http_client = httpx.Client(verify=False)
+        else:
+            http_client = None
+
+        kwargs = {
+            "model": model_name,
+            "base_url": base_url or None,
+            "api_key": SecretStr(api_key or ""),
+            "http_client": http_client,
+        }
+
+        # 仅DashScope注入encoding_format="float"，其他服务商不强制注入，避免参数不兼容
+        if is_dashscope:
+            import inspect
+            sig = inspect.signature(OpenAIEmbeddings.__init__)
+            if "encoding_format" in sig.parameters:
+                kwargs["encoding_format"] = "float"
+            return DashScopeOpenAIEmbeddings(**kwargs)
+
+        # 其余所有OpenAI兼容模型，走原生实现
+        return OpenAIEmbeddings(**kwargs)
+
+    @staticmethod
+    def check_health(
+        model_name: str,
+        base_url: str,
+        api_key: str | None,
+        model_type: Literal["CHAT", "EMBEDDING"] | str,
+    ) -> None:
+        """对配置做一次最小化调用进行健康检查，失败则抛出。"""
+        if model_type == "CHAT":
+            model = LLMFactory.create_chat(model_name, base_url, api_key)
+            model.invoke("hello")
+        else:
+            model = LLMFactory.create_embedding(model_name, base_url, api_key)
+            model.embed_query("text")
+
+    @staticmethod
+    def get_embedding_dimension(
+        model_name: str,
+        base_url: str,
+        api_key: str | None = None,
+    ) -> int:
+        """创建 Embedding 模型并返回向量维度。"""
+        emb = LLMFactory.create_embedding(model_name, base_url, api_key)
+        return len(emb.embed_query("text"))
+
+    @staticmethod
+    def invoke_sync(chat_model: BaseChatModel, prompt: str) -> str:
+        """同步调用对话模型并返回 content，供 run_in_executor 等场景使用。"""
+        return chat_model.invoke(prompt).content
@@ -0,0 +1,57 @@
+# 代码内容与原文件基本一致，确保它是通用的
+import httpx
+from typing import Optional
+from app.core.logging import get_logger
+
+logger = get_logger(__name__)
+
+# xuhuanqing
+class RuntimeClient:
+    """HTTP client for communicating with deepdata-runtime service"""
+    
+    def __init__(self, base_url: str = "http://deepdata-runtime:8081"):
+        self.base_url = base_url
+        self.client = httpx.AsyncClient(timeout=60.0)
+
+    async def submit_task(self, task_id: str, retry_count: int = 0) -> bool:
+        """Submit task to runtime executor (通用接口)"""
+        try:
+            url = f"{self.base_url}/api/task/{task_id}/submit"
+            # 注意：虽然 json payload 目前是 {retry_count}，
+            # 但 Runtime 接收的是一个通用的 JSON，未来可以扩展增加 executor_type 等字段
+            response = await self.client.post(url, json={"retry_count": retry_count})
+            response.raise_for_status()
+            logger.info(f"Task {task_id} submitted to Runtime successfully.")
+            return True
+        except httpx.HTTPError as e:
+            logger.error(f"Failed to submit task {task_id} to Runtime: {e}")
+            return False
+        except Exception as e:
+            logger.error(f"Unexpected error submitting task {task_id}: {e}")
+            return False
+
+    async def stop_task(self, task_id: str) -> bool:
+        """Stop running task"""
+        try:
+            url = f"{self.base_url}/api/task/{task_id}/stop"
+            response = await self.client.post(url)
+            response.raise_for_status()
+            return True
+        except Exception as e:
+            logger.error(f"Failed to stop task {task_id}: {e}")
+            return False
+
+    async def get_task_status(self, task_id: str) -> Optional[dict]:
+        """Get task execution status from runtime"""
+        try:
+            url = f"{self.base_url}/api/task/{task_id}/status"
+            response = await self.client.get(url)
+            response.raise_for_status()
+            return response.json()
+        except Exception as e:
+            logger.error(f"Failed to get task status {task_id}: {e}")
+            return None
+
+    async def close(self):
+        """Close HTTP client"""
+        await self.client.aclose()
@@ -0,0 +1,3 @@
+from .scheduler import Scheduler
+
+__all__ = ["Scheduler"]
@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+from typing import Any, Callable, Optional
+
+import pytz
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from apscheduler.triggers.cron import CronTrigger
+from apscheduler.triggers.interval import IntervalTrigger
+
+from app.core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class Scheduler:
+    def __init__(self, name: str = "scheduler", timezone: Optional[str] = None) -> None:
+        self._name = name
+        self._scheduler: Optional[AsyncIOScheduler] = None
+        # 使用指定的时区，如果没有指定则使用本地时区
+        self._timezone = pytz.timezone(timezone) if timezone else None
+
+    def start(self) -> AsyncIOScheduler:
+        if self._scheduler is None:
+            # 配置调度器使用本地时区
+            from tzlocal import get_localzone
+            local_tz = get_localzone()
+            self._scheduler = AsyncIOScheduler(timezone=local_tz)
+            self._scheduler.start()
+            logger.info(f"{self._name} started with timezone: {local_tz}")
+        return self._scheduler
+
+    def shutdown(self) -> None:
+        if self._scheduler is not None:
+            self._scheduler.shutdown(wait=False)
+            self._scheduler = None
+            logger.info(f"{self._name} stopped")
+
+    def add_cron_job(
+        self,
+        job_id: str,
+        cron_expression: str,
+        func: Callable[..., Any],
+        args: Optional[list[Any]] = None,
+        kwargs: Optional[dict[str, Any]] = None,
+        **job_kwargs: Any,
+    ) -> None:
+        scheduler = self._get_scheduler()
+        # 使用调度器的时区创建 CronTrigger
+        trigger = CronTrigger.from_crontab(cron_expression, timezone=scheduler.timezone)
+        scheduler.add_job(
+            func,
+            trigger=trigger,
+            args=args or [],
+            kwargs=kwargs or {},
+            id=job_id,
+            replace_existing=job_kwargs.pop("replace_existing", True),
+            max_instances=job_kwargs.pop("max_instances", 1),
+            coalesce=job_kwargs.pop("coalesce", True),
+            misfire_grace_time=job_kwargs.pop("misfire_grace_time", 60),
+            **job_kwargs,
+        )
+
+    def add_interval_job(
+        self,
+        job_id: str,
+        seconds: int,
+        func: Callable[..., Any],
+        args: Optional[list[Any]] = None,
+        kwargs: Optional[dict[str, Any]] = None,
+        **job_kwargs: Any,
+    ) -> None:
+        scheduler = self._get_scheduler()
+        trigger = IntervalTrigger(seconds=seconds, timezone=scheduler.timezone)
+        scheduler.add_job(
+            func,
+            trigger=trigger,
+            args=args or [],
+            kwargs=kwargs or {},
+            id=job_id,
+            replace_existing=job_kwargs.pop("replace_existing", True),
+            max_instances=job_kwargs.pop("max_instances", 1),
+            coalesce=job_kwargs.pop("coalesce", True),
+            misfire_grace_time=job_kwargs.pop("misfire_grace_time", 60),
+            **job_kwargs,
+        )
+
+    def remove_job(self, job_id: str) -> None:
+        if self._scheduler is None:
+            return
+        if self._scheduler.get_job(job_id):
+            self._scheduler.remove_job(job_id)
+
+    def has_job(self, job_id: str) -> bool:
+        if self._scheduler is None:
+            return False
+        return self._scheduler.get_job(job_id) is not None
+
+    def reschedule_job(
+        self,
+        job_id: str,
+        cron_expression: str,
+        func: Callable[..., Any],
+        args: Optional[list[Any]] = None,
+        kwargs: Optional[dict[str, Any]] = None,
+        **job_kwargs: Any,
+    ) -> None:
+        self.remove_job(job_id)
+        self.add_cron_job(
+            job_id=job_id,
+            cron_expression=cron_expression,
+            func=func,
+            args=args,
+            kwargs=kwargs,
+            **job_kwargs,
+        )
+
+    @staticmethod
+    def validate_cron_expression(cron_expression: str) -> None:
+        CronTrigger.from_crontab(cron_expression)
+
+    def _get_scheduler(self) -> AsyncIOScheduler:
+        if self._scheduler is None:
+            raise RuntimeError(f"{self._name} not initialized")
+        return self._scheduler
@@ -0,0 +1,19 @@
+from .common import (
+    BaseResponseModel,
+    StandardResponse,
+    PaginatedData,
+    TaskStatus
+)
+from .lineage import (
+    NodeType,
+    EdgeType
+)
+
+__all__ = [
+    "BaseResponseModel",
+    "StandardResponse",
+    "PaginatedData",
+    "TaskStatus",
+    "NodeType",
+    "EdgeType"
+]
@@ -0,0 +1,67 @@
+"""
+通用响应模型
+"""
+from typing import Generic, TypeVar, List
+from pydantic import BaseModel, Field
+from enum import Enum
+
+# 定义泛型类型变量
+T = TypeVar('T')
+
+# 定义一个将 snake_case 转换为 camelCase 的函数
+def to_camel(string: str) -> str:
+    """将 snake_case 字符串转换为 camelCase"""
+    components = string.split('_')
+    # 首字母小写，其余单词首字母大写
+    return components[0] + ''.join(x.title() for x in components[1:])
+
+class BaseResponseModel(BaseModel):
+    """基础响应模型，启用别名生成器"""
+
+    class Config:
+        populate_by_name = True
+        alias_generator = to_camel
+
+class StandardResponse(BaseResponseModel, Generic[T]):
+    """
+    标准API响应格式
+
+    所有API端点（包括错误响应）都应返回此格式
+    """
+    code: str = Field(..., description="业务状态码（字符串）。成功使用 '0'，错误使用 '{module}.{sequence}' 格式（如 'rag.001'）")
+    message: str = Field(..., description="响应消息")
+    data: T = Field(default=None, description="响应数据")
+
+    class Config:
+        populate_by_name = True
+        alias_generator = to_camel
+
+
+class ResponseCode(str, Enum):
+    """通用响应码枚举"""
+
+    # 成功响应
+    SUCCESS = "0"  # 操作成功
+
+    # 通用错误
+    BAD_REQUEST = "common.400"  # 错误的请求
+    UNAUTHORIZED = "common.401"  # 未授权
+    FORBIDDEN = "common.403"  # 禁止访问
+    NOT_FOUND = "common.404"  # 资源未找到
+    VALIDATION_ERROR = "common.422"  # 验证错误
+    INTERNAL_ERROR = "common.500"  # 服务器内部错误
+    SERVICE_UNAVAILABLE = "common.503"  # 服务不可用
+
+class PaginatedData(BaseResponseModel, Generic[T]):
+    """分页数据容器"""
+    page: int = Field(..., description="当前页码（从1开始）")
+    size: int = Field(..., description="页大小")
+    total_elements: int = Field(..., description="总条数")
+    total_pages: int = Field(..., description="总页数")
+    content: List[T] = Field(..., description="当前页数据")
+
+class TaskStatus(Enum):
+    PENDING = "PENDING"
+    RUNNING = "RUNNING"
+    COMPLETED = "COMPLETED"
+    FAILED = "FAILED"
@@ -0,0 +1,14 @@
+from enum import Enum
+
+class NodeType(str, Enum):
+    DATASOURCE = "DATASOURCE"
+    DATASET = "DATASET"
+    KNOWLEDGE_BASE = "KNOWLEDGE_BASE"
+    MODEL = "MODEL"
+
+class EdgeType(str, Enum):
+    DATA_COLLECTION = "DATA_COLLECTION"
+    DATA_CLEANING = "DATA_CLEANING"
+    DATA_LABELING = "DATA_LABELING"
+    DATA_SYNTHESIS = "DATA_SYNTHESIS"
+    DATA_RATIO = "DATA_RATIO"
@@ -0,0 +1,87 @@
+from openai import OpenAI
+
+
+def call_openai_style_model(base_url, api_key, model_name, prompt, **kwargs):
+    client = OpenAI(
+        base_url=base_url,
+        api_key=api_key
+    )
+
+    response = client.chat.completions.create(
+        model=model_name,
+        messages=[{"role": "user", "content": prompt}],
+        **kwargs
+    )
+    return response.choices[0].message.content
+
+
+def extract_json_substring(raw: str) -> str:
+    """从 LLM 的原始回答中提取最可能的 JSON 字符串片段。
+
+    处理思路：
+    - 原始回答可能是：说明文字 + JSON + 说明文字，甚至带有 Markdown 代码块。
+    - 优先在文本中查找第一个 '{' 或 '[' 作为 JSON 起始；
+    - 再从后向前找最后一个 '}' 或 ']' 作为结束；
+    - 如果找不到合适的边界，就退回原始字符串。
+    - 部分模型可能会在回复中加入 `<think>...</think>` 内部思考内容，应在解析前先去除。
+    - 也有模型会在 JSON 前后增加如 <reasoning>...</reasoning>、<analysis>...</analysis> 等标签，本方法会一并去除。
+    该方法不会保证截取的一定是合法 JSON，但能显著提高 json.loads 的成功率。
+    """
+    if not raw:
+        return raw
+
+    try:
+        import re
+
+        # 1. 先把所有完整的思考标签块整体去掉：<think>...</think> 等
+        thought_tags = [
+            "think",
+            "thinking",
+            "analysis",
+            "reasoning",
+            "reflection",
+            "inner_thoughts",
+        ]
+        for tag in thought_tags:
+            pattern = rf"<{tag}>[\s\S]*?</{tag}>"
+            raw = re.sub(pattern, "", raw, flags=re.IGNORECASE)
+
+        # 2. 再做一次“截取最后一个 </think>（或其它思考标签结束）之后的内容”的兜底
+        #    这样就算标签不成对或嵌套异常，也能保留尾部真正的回答
+        last_pos = -1
+        for tag in thought_tags:
+            # 匹配类似 </think> 或 </THINK>
+            m = list(re.finditer(rf"</{tag}>", raw, flags=re.IGNORECASE))
+            if m:
+                last_pos = max(last_pos, m[-1].end())
+        if last_pos != -1 and last_pos < len(raw):
+            raw = raw[last_pos:]
+
+    except Exception:
+        # 正则异常时不影响后续逻辑，继续使用当前文本
+        pass
+
+    start = None
+    end = None
+
+    # 查找第一个 JSON 起始符号
+    for i, ch in enumerate(raw):
+        if ch in "[{":
+            start = i
+            break
+
+    # 查找最后一个 JSON 结束符号
+    for i in range(len(raw) - 1, -1, -1):
+        if raw[i] in "]}":
+            end = i + 1  # 切片是左闭右开
+            break
+
+    if start is not None and end is not None and start < end:
+        return raw[start:end].strip()
+
+    # 兜底：去掉常见 Markdown 包裹（```json ... ```）
+    stripped = raw.strip()
+    if stripped.startswith("```"):
+        # 去掉首尾 ``` 标记
+        stripped = stripped.strip("`")
+    return stripped
@@ -0,0 +1,159 @@
+"""
+资源感知并发计算工具
+
+根据系统 CPU 和内存资源动态计算最优并发数
+适配全闪存储等高性能存储场景
+"""
+import os
+import logging
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# 尝试导入 psutil，失败则使用 os.cpu_count
+try:
+    import psutil
+    HAS_PSUTIL = True
+except ImportError:
+    HAS_PSUTIL = False
+    logger.warning("psutil not available, falling back to os.cpu_count() for CPU detection")
+
+
+def get_cpu_count(logical: bool = True) -> int:
+    """获取 CPU 核心数
+    
+    Args:
+        logical: 是否返回逻辑核心数（超线程），False返回物理核心数
+        
+    Returns:
+        CPU 核心数，无法检测时返回默认值 4
+    """
+    if HAS_PSUTIL:
+        count = psutil.cpu_count(logical=logical)
+        if count:
+            return count
+    
+    # 回退到 os.cpu_count()
+    count = os.cpu_count()
+    return count if count else 4
+
+
+def get_available_memory_gb() -> float:
+    """获取可用内存（GB）
+    
+    Returns:
+        可用内存大小（GB），无法检测时返回默认值 8.0
+    """
+    if HAS_PSUTIL:
+        mem = psutil.virtual_memory()
+        return mem.available / (1024 ** 3)
+    
+    # 无法检测，返回保守默认值
+    logger.warning("psutil not available, using default memory estimate")
+    return 8.0
+
+
+def calculate_optimal_concurrent(
+    min_concurrent: int = 8,
+    max_concurrent: int = 128,
+    cpu_factor: float = 4.0,
+    memory_per_task_mb: int = 32,
+    memory_reserve_ratio: float = 0.2,
+) -> int:
+    """根据系统资源计算最优并发数
+    
+    计算逻辑：
+    1. 基于 CPU 核心数：并发数 = CPU核心数 × cpu_factor
+    2. 基于内存限制：可用内存 × (1-reserve_ratio) ÷ memory_per_task
+    3. 最终取两者最小值，并限制在 [min_concurrent, max_concurrent] 范围内
+    
+    Args:
+        min_concurrent: 并发下限
+        max_concurrent: 并发上限
+        cpu_factor: CPU核心系数（全闪存储建议4.0）
+        memory_per_task_mb: 每任务预估内存占用（MB）
+        memory_reserve_ratio: 内存安全保留比例
+        
+    Returns:
+        计算得出的最优并发数
+    """
+    # 获取 CPU 核心数
+    cpu_count = get_cpu_count(logical=True)
+    
+    # 基于 CPU 计算并发数
+    cpu_based_concurrent = int(cpu_count * cpu_factor)
+    
+    # 获取可用内存
+    available_memory_gb = get_available_memory_gb()
+    
+    # 计算可用于并发任务的内存（扣除保留部分）
+    usable_memory_gb = available_memory_gb * (1 - memory_reserve_ratio)
+    usable_memory_mb = usable_memory_gb * 1024
+    
+    # 基于内存计算最大并发数
+    memory_based_concurrent = int(usable_memory_mb / memory_per_task_mb)
+    
+    # 取两者最小值（避免内存耗尽）
+    calculated_concurrent = min(cpu_based_concurrent, memory_based_concurrent)
+    
+    # 限制在范围内
+    final_concurrent = max(min_concurrent, min(max_concurrent, calculated_concurrent))
+    
+    # 记录计算过程
+    logger.info(
+        f"Concurrency calculation: "
+        f"CPU cores={cpu_count}, CPU-based={cpu_based_concurrent}, "
+        f"Available memory={available_memory_gb:.1f}GB, Memory-based={memory_based_concurrent}, "
+        f"Final concurrent={final_concurrent} (range=[{min_concurrent}, {max_concurrent}])"
+    )
+    
+    return final_concurrent
+
+
+def get_concurrent_for_ratio_copy(settings) -> int:
+    """获取配比文件复制的并发数
+    
+    Args:
+        settings: 配置实例
+        
+    Returns:
+        并发数（动态计算或固定值）
+    """
+    if not settings.ratio_copy_dynamic_concurrent:
+        return settings.ratio_copy_fixed_concurrent
+    
+    return calculate_optimal_concurrent(
+        min_concurrent=settings.ratio_copy_min_concurrent,
+        max_concurrent=settings.ratio_copy_max_concurrent,
+        cpu_factor=settings.ratio_copy_cpu_factor,
+        memory_per_task_mb=settings.ratio_copy_memory_per_task_mb,
+        memory_reserve_ratio=settings.ratio_copy_memory_reserve_ratio,
+    )
+
+
+def get_system_resource_info() -> dict:
+    """获取系统资源信息（用于日志和调试）
+    
+    Returns:
+        系统资源信息字典
+    """
+    cpu_logical = get_cpu_count(logical=True)
+    cpu_physical = get_cpu_count(logical=False) if HAS_PSUTIL else cpu_logical
+    
+    if HAS_PSUTIL:
+        mem = psutil.virtual_memory()
+        return {
+            "cpu_logical_cores": cpu_logical,
+            "cpu_physical_cores": cpu_physical,
+            "memory_total_gb": round(mem.total / (1024 ** 3), 2),
+            "memory_available_gb": round(mem.available / (1024 ** 3), 2),
+            "memory_used_percent": round(mem.percent, 1),
+            "psutil_available": True,
+        }
+    
+    return {
+        "cpu_logical_cores": cpu_logical,
+        "cpu_physical_cores": cpu_physical,
+        "memory_available_gb": get_available_memory_gb(),
+        "psutil_available": False,
+    }
@@ -0,0 +1,113 @@
+import json
+
+from enum import Enum
+from jsonschema import validate
+
+class ItemTypes(Enum):
+    QA = "QA"
+    COT = "COT"
+
+
+class StructuredFileItemHandler:
+    def __init__(self):
+        pass
+
+    def get_item_type(self) -> ItemTypes:
+        pass
+
+    def validate_json(self, data):
+        pass
+
+    def get_items_from_file(self, file_path: str) -> list[dict]:
+        file_type = file_path.split(".")[-1].upper()
+        items = []
+        if file_type == "JSON":
+            with open(file_path, "r", encoding="utf-8") as f:
+                data = json.load(f)
+                if not self.validate_json(data):
+                    return items
+                items = data
+        elif file_type == "JSONL":
+            with open(file_path, "r", encoding="utf-8") as f:
+                for line in f:
+                    data = json.loads(line)
+                    if not self.validate_json(data):
+                        continue
+                    items.append(data)
+        return items
+
+class QAItemHandler(StructuredFileItemHandler):
+    def __init__(self):
+        self.schema_alpaca = {
+            "type": "object",
+            "properties": {
+                "instruction": {"type": "string"},
+                "input": {"type": "string"},
+                "output": {"type": "string"}
+            },
+            "required": ["instruction", "output"],
+        }
+        self.schema_alpaca_list = {
+            "type": "array",
+            "items": self.schema_alpaca,
+        }
+        super().__init__()
+
+    def get_item_type(self):
+        return ItemTypes.QA
+
+    def validate_json(self, data):
+        try:
+            validate(instance=data, schema=self.schema_alpaca)
+            return True
+        except Exception as e:
+            try:
+                validate(instance=data, schema=self.schema_alpaca_list)
+                return True
+            except Exception as e:
+                return False
+
+
+class COTItemHandler(StructuredFileItemHandler):
+    def __init__(self):
+        self.schema = {
+            "type": "object",
+            "properties": {
+                "instruction": {"type": "string"},
+                "output": {"type": "string"},
+                "chain_of_thought": {"type": "string"}
+            },
+            "required": ["question", "instruction", "output"],
+        }
+        self.schema_list = {
+            "type": "array",
+            "items": self.schema,
+        }
+        super().__init__()
+
+    def get_item_type(self):
+        return ItemTypes.COT
+
+    def validate_json(self, data):
+        try:
+            validate(instance=data, schema=self.schema)
+            return True
+        except Exception as e:
+            try:
+                validate(instance=data, schema=self.schema_list)
+                return True
+            except Exception as e:
+                return False
+
+
+class StructuredFileHandlerFactory:
+    def __init__(self):
+        self.handlers: list[StructuredFileItemHandler] = []
+        self.handlers.append(QAItemHandler())
+        self.handlers.append(COTItemHandler())
+
+    def get_handler(self, item_type: str) -> StructuredFileItemHandler:
+        for handler in self.handlers:
+            if handler.get_item_type().value == item_type:
+                return handler
+        raise ValueError(f"Unsupported item type: {item_type}")