deepdata: core product code (backend, frontend, runtime, deepflow module)

2026-08-30 19:23:38 +08:00
commit bd3db30be9
1405 changed files with 271727 additions and 0 deletions
@@ -0,0 +1,16 @@
+"""
+公共工具模块
+
+提供文本清理、元数据构建、批量处理等工具类。
+"""
+from .text_cleaner import TextCleaner
+from .metadata_builder import MetadataBuilder
+from .batch_processor import BatchProcessor
+from .file_utils import get_file_path
+
+__all__ = [
+    "TextCleaner",
+    "MetadataBuilder",
+    "BatchProcessor",
+    "get_file_path",
+]
@@ -0,0 +1,94 @@
+"""
+批量处理工具
+
+提供分批处理文档分块的工具方法。
+"""
+import uuid
+import logging
+from typing import List
+from app.module.rag.infra.vectorstore import chunks_to_documents
+from app.module.rag.infra.document.types import DocumentChunk
+
+
+logger = logging.getLogger(__name__)
+
+
+class BatchProcessor:
+    """批量处理工具类"""
+    
+    DEFAULT_BATCH_SIZE: int = 20
+    
+    @staticmethod
+    async def store_in_batches(
+        vectorstore,
+        chunks: List[DocumentChunk],
+        batch_size: int = DEFAULT_BATCH_SIZE
+    ) -> int:
+        """分批存储分块到向量数据库
+        
+        Args:
+            vectorstore: 向量存储实例
+            chunks: 分块列表
+            batch_size: 批次大小
+            
+        Returns:
+            成功存储的分块数量
+        """
+        if not chunks:
+            return 0
+        
+        batch_size = batch_size or BatchProcessor.DEFAULT_BATCH_SIZE
+        total_chunks = len(chunks)
+        stored_count = 0
+        
+        for batch_start in range(0, total_chunks, batch_size):
+            batch_end = min(batch_start + batch_size, total_chunks)
+            batch_chunks = chunks[batch_start:batch_end]
+            
+            await BatchProcessor._store_single_batch(
+                vectorstore,
+                batch_chunks,
+                batch_start,
+                batch_end,
+                total_chunks
+            )
+            stored_count += len(batch_chunks)
+        
+        logger.info("批量存储完成，总数量: %d", stored_count)
+        return stored_count
+    
+    @staticmethod
+    async def _store_single_batch(
+        vectorstore,
+        batch_chunks: List[DocumentChunk],
+        batch_start: int,
+        batch_end: int,
+        total_chunks: int,
+    ) -> None:
+        """存储单个批次
+        
+        Args:
+            vectorstore: 向量存储实例
+            batch_chunks: 批次分块列表
+            batch_start: 批次起始索引
+            batch_end: 批次结束索引
+            total_chunks: 总分块数
+        """
+        logger.info("处理分块批次 %d-%d / %d", batch_start + 1, batch_end, total_chunks)
+        
+        ids = [str(uuid.uuid4()) for _ in batch_chunks]
+        documents, doc_ids = chunks_to_documents(batch_chunks, ids=ids)
+        
+        try:
+            # 使用异步方法避免阻塞事件循环
+            await vectorstore.aadd_documents(documents=documents, ids=doc_ids)
+            logger.info("批次 %d-%d 存储成功", batch_start + 1, batch_end)
+        except Exception as e:
+            logger.error(
+                "批次 %d-%d 存储失败: %s\n第一个文档内容: %.200s",
+                batch_start + 1,
+                batch_end,
+                str(e),
+                documents[0].page_content if documents else "N/A"
+            )
+            raise
@@ -0,0 +1,25 @@
+"""
+文件工具类
+
+提供文件路径提取等通用工具方法。
+"""
+from pathlib import Path
+from typing import Optional
+
+
+def get_file_path(rag_file) -> Optional[str]:
+    """获取 RAG 文件的绝对路径
+    
+    Args:
+        rag_file: RAG 文件实体（需要有 file_metadata 属性）
+        
+    Returns:
+        文件绝对路径，不存在返回 None
+    """
+    if not rag_file.file_metadata:
+        return None
+    
+    file_path = rag_file.file_metadata.get("file_path")
+    if file_path:
+        return str(Path(file_path).absolute())
+    return None
@@ -0,0 +1,42 @@
+"""
+元数据构建工具
+
+提供构建和管理分块元数据的工具方法。
+"""
+from typing import Dict, Any
+from app.db.models.knowledge_gen import KnowledgeBase, RagFile
+
+
+class MetadataBuilder:
+    """元数据构建工具类"""
+    
+    @staticmethod
+    def build_chunk_metadata(rag_file: RagFile, knowledge_base: KnowledgeBase) -> Dict[str, Any]:
+        """构建分块基础元数据
+        
+        Args:
+            rag_file: RAG 文件实体
+            knowledge_base: 知识库实体
+            
+        Returns:
+            元数据字典
+        """
+        return {
+            "rag_file_id": str(rag_file.id),
+            "original_file_id": str(rag_file.file_id),
+            "knowledge_base_id": str(knowledge_base.id),
+            "file_name": rag_file.file_name,
+            "knowledge_base_name": knowledge_base.name,
+        }
+    
+    @staticmethod
+    def add_to_chunks(chunks: list, base_metadata: Dict[str, Any]) -> None:
+        """为基础元数据添加到分块列表
+        
+        Args:
+            chunks: 分块列表
+            base_metadata: 基础元数据
+        """
+        for chunk in chunks:
+            if hasattr(chunk, 'metadata') and isinstance(chunk.metadata, dict):
+                chunk.metadata.update(base_metadata)
@@ -0,0 +1,67 @@
+"""
+文本清理工具
+
+提供文本清理和标准化的工具方法。
+"""
+import re
+import logging
+from typing import Optional
+
+
+logger = logging.getLogger(__name__)
+
+
+class TextCleaner:
+    """文本清理工具类"""
+    
+    @staticmethod
+    def clean(text: Optional[str]) -> str:
+        """清理文本
+        
+        Args:
+            text: 原始文本
+            
+        Returns:
+            清理后的文本
+        """
+        if not text:
+            return ""
+        
+        text = TextCleaner._remove_control_characters(text)
+        text = TextCleaner._normalize_whitespace(text)
+        text = TextCleaner._remove_empty_lines(text)
+        
+        return text.strip()
+    
+    @staticmethod
+    def _remove_control_characters(text: str) -> str:
+        """移除控制字符"""
+        control_chars = re.compile(r'[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f-\x9f]')
+        return control_chars.sub('', text)
+    
+    @staticmethod
+    def _normalize_whitespace(text: str) -> str:
+        """标准化空白字符"""
+        text = re.sub(r'[ \t]+', ' ', text)
+        text = re.sub(r'\n\s*\n\s*\n+', '\n\n', text)
+        return text
+    
+    @staticmethod
+    def _remove_empty_lines(text: str) -> str:
+        """移除空行"""
+        lines = text.split('\n')
+        return '\n'.join(line for line in lines if line.strip())
+    
+    @staticmethod
+    def has_printable_content(text: str) -> bool:
+        """检查是否包含可打印内容
+        
+        Args:
+            text: 文本
+            
+        Returns:
+            是否包含可打印内容
+        """
+        if not text:
+            return False
+        return any(c.isprintable() and not c.isspace() for c in text)