deepdata: core product code (backend, frontend, runtime, deepflow module)
This commit is contained in:
figmar
@@ -0,0 +1,42 @@
|
||||
"""
|
||||
文档处理模块
|
||||
|
||||
提供文档加载、分块和处理管道功能。
|
||||
|
||||
使用示例:
|
||||
from app.module.rag.infra.document import (
|
||||
ingest_file_to_chunks,
|
||||
SplitOptions,
|
||||
DocumentChunk,
|
||||
)
|
||||
|
||||
chunks = await ingest_file_to_chunks(
|
||||
"/path/to/doc.pdf",
|
||||
chunk_size=500,
|
||||
overlap_size=50,
|
||||
)
|
||||
"""
|
||||
from app.module.rag.infra.document.processor import (
|
||||
SplitOptions,
|
||||
default_split_options,
|
||||
ingest_file_to_chunks,
|
||||
load_and_split,
|
||||
)
|
||||
from app.module.rag.infra.document.types import (
|
||||
DocumentChunk,
|
||||
ParsedDocument,
|
||||
langchain_documents_to_parsed,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# 处理管道入口
|
||||
"load_and_split",
|
||||
"ingest_file_to_chunks",
|
||||
# 选项
|
||||
"SplitOptions",
|
||||
"default_split_options",
|
||||
# 类型
|
||||
"DocumentChunk",
|
||||
"ParsedDocument",
|
||||
"langchain_documents_to_parsed",
|
||||
]
|
||||
@@ -0,0 +1,26 @@
|
||||
"""
|
||||
文档加载器
|
||||
|
||||
封装 UniversalDocLoader，提供统一的文档加载接口。
|
||||
"""
|
||||
import asyncio
|
||||
from typing import List
|
||||
|
||||
from langchain_core.documents import Document
|
||||
|
||||
from app.module.shared.common.document_loaders import UniversalDocLoader
|
||||
|
||||
|
||||
async def load_document(file_path: str) -> List[Document]:
|
||||
"""加载文档（异步封装）
|
||||
|
||||
使用 UniversalDocLoader 加载文档，支持多种格式。
|
||||
|
||||
Args:
|
||||
file_path: 文件绝对路径
|
||||
|
||||
Returns:
|
||||
LangChain Document 列表
|
||||
"""
|
||||
loader = UniversalDocLoader(file_path)
|
||||
return await asyncio.to_thread(loader.load)
|
||||
@@ -0,0 +1,125 @@
|
||||
"""
|
||||
文档处理管道
|
||||
|
||||
提供统一的文档加载和分块入口，合并原有的 pipeline.py 和 options.py。
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.module.rag.infra.document.loader import load_document
|
||||
from app.module.rag.infra.document.splitter import DocumentSplitterFactory
|
||||
from app.module.rag.infra.document.types import (
|
||||
DocumentChunk,
|
||||
ParsedDocument,
|
||||
langchain_documents_to_parsed,
|
||||
)
|
||||
from app.module.rag.schema.enums import ProcessType
|
||||
|
||||
|
||||
@dataclass
|
||||
class SplitOptions:
|
||||
"""文档分片选项
|
||||
|
||||
Attributes:
|
||||
process_type: 分片策略
|
||||
chunk_size: 块大小（字符）
|
||||
overlap_size: 块间重叠
|
||||
delimiter: 仅 CUSTOM_SEPARATOR_CHUNK 时有效
|
||||
"""
|
||||
process_type: ProcessType = ProcessType.DEFAULT_CHUNK
|
||||
chunk_size: int = 500
|
||||
overlap_size: int = 50
|
||||
delimiter: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""转换为字典"""
|
||||
return {
|
||||
"process_type": self.process_type,
|
||||
"chunk_size": self.chunk_size,
|
||||
"overlap_size": self.overlap_size,
|
||||
"delimiter": self.delimiter,
|
||||
}
|
||||
|
||||
|
||||
def default_split_options() -> SplitOptions:
|
||||
"""默认分片选项：递归分块 500/50"""
|
||||
return SplitOptions()
|
||||
|
||||
|
||||
async def load_and_split(
|
||||
file_path: str,
|
||||
split_options: Optional[SplitOptions] = None,
|
||||
**chunk_metadata: Any,
|
||||
) -> List[DocumentChunk]:
|
||||
"""加载文档并分块
|
||||
|
||||
使用 UniversalDocLoader 加载文档，然后按指定策略分块。
|
||||
|
||||
Args:
|
||||
file_path: 文件绝对路径
|
||||
split_options: 分片选项，None 表示使用默认
|
||||
**chunk_metadata: 写入每个 chunk.metadata 的额外字段
|
||||
|
||||
Returns:
|
||||
分块列表
|
||||
"""
|
||||
documents = await load_document(file_path)
|
||||
|
||||
parser_metadata = {}
|
||||
for key in ["original_file_id", "rag_file_id", "file_name"]:
|
||||
if key in chunk_metadata:
|
||||
parser_metadata[key] = chunk_metadata[key]
|
||||
|
||||
parsed = langchain_documents_to_parsed(documents, file_path, **parser_metadata)
|
||||
|
||||
options = split_options or default_split_options()
|
||||
|
||||
base_chunk_metadata = {
|
||||
"file_name": parsed.metadata.get("file_name", ""),
|
||||
"file_extension": parsed.metadata.get("file_extension", ""),
|
||||
"absolute_directory_path": parsed.metadata.get("absolute_directory_path", ""),
|
||||
"original_file_id": parsed.metadata.get("original_file_id", ""),
|
||||
"rag_file_id": parsed.metadata.get("rag_file_id", ""),
|
||||
}
|
||||
base_chunk_metadata.update(chunk_metadata)
|
||||
|
||||
splitter = DocumentSplitterFactory.create_splitter(
|
||||
options.process_type,
|
||||
chunk_size=options.chunk_size,
|
||||
overlap_size=options.overlap_size,
|
||||
delimiter=options.delimiter,
|
||||
)
|
||||
|
||||
return await splitter.split(parsed.text, **base_chunk_metadata)
|
||||
|
||||
|
||||
async def ingest_file_to_chunks(
|
||||
file_path: str,
|
||||
process_type: ProcessType = ProcessType.DEFAULT_CHUNK,
|
||||
chunk_size: int = 500,
|
||||
overlap_size: int = 50,
|
||||
delimiter: Optional[str] = None,
|
||||
**chunk_metadata: Any,
|
||||
) -> List[DocumentChunk]:
|
||||
"""从本地文件加载文档并分块（便捷入口）
|
||||
|
||||
可被 ETL、URL 抓取、S3 等场景复用。
|
||||
|
||||
Args:
|
||||
file_path: 文件绝对路径
|
||||
process_type: 分块策略
|
||||
chunk_size: 块大小
|
||||
overlap_size: 重叠大小
|
||||
delimiter: 自定义分隔符
|
||||
**chunk_metadata: 写入每个 chunk.metadata 的额外字段
|
||||
|
||||
Returns:
|
||||
分块列表
|
||||
"""
|
||||
split_options = SplitOptions(
|
||||
process_type=process_type,
|
||||
chunk_size=chunk_size,
|
||||
overlap_size=overlap_size,
|
||||
delimiter=delimiter,
|
||||
)
|
||||
return await load_and_split(file_path, split_options=split_options, **chunk_metadata)
|
||||
@@ -0,0 +1,148 @@
|
||||
"""
|
||||
文档分块器
|
||||
|
||||
包含 DocumentSplitter 基类、工厂和 LangChain 实现。
|
||||
根据 ProcessType 创建对应的分块策略。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from langchain_text_splitters import (
|
||||
CharacterTextSplitter,
|
||||
RecursiveCharacterTextSplitter,
|
||||
)
|
||||
|
||||
from app.module.rag.infra.document.types import DocumentChunk
|
||||
from app.module.rag.schema.enums import ProcessType
|
||||
|
||||
|
||||
# 各 ProcessType 对应的分隔符配置（优先保持较大语义块）
|
||||
SEPARATORS_BY_PROCESS_TYPE = {
|
||||
ProcessType.PARAGRAPH_CHUNK: ["\n\n", "\n", " ", ""],
|
||||
ProcessType.SENTENCE_CHUNK: ["\n\n", "\n", "。", "！", "？", ". ", "! ", "? ", " ", ""],
|
||||
ProcessType.DEFAULT_CHUNK: ["\n\n", "\n", " ", ""],
|
||||
ProcessType.CUSTOM_SEPARATOR_CHUNK: None,
|
||||
}
|
||||
|
||||
|
||||
class DocumentSplitter(ABC):
|
||||
"""文档分块器基类
|
||||
|
||||
所有具体的分块器都需要继承此类并实现 split 方法。
|
||||
"""
|
||||
|
||||
def __init__(self, chunk_size: int = 500, overlap_size: int = 50):
|
||||
"""初始化分块器
|
||||
|
||||
Args:
|
||||
chunk_size: 分块大小
|
||||
overlap_size: 重叠大小
|
||||
"""
|
||||
self.chunk_size = chunk_size
|
||||
self.overlap_size = overlap_size
|
||||
|
||||
@abstractmethod
|
||||
async def split(self, text: str, **metadata: Any) -> List[DocumentChunk]:
|
||||
"""分割文档
|
||||
|
||||
Args:
|
||||
text: 文档文本
|
||||
**metadata: 额外的元数据
|
||||
|
||||
Returns:
|
||||
分块列表
|
||||
"""
|
||||
pass
|
||||
|
||||
def _create_chunk(self, text: str, chunk_index: int, **metadata: Any) -> DocumentChunk:
|
||||
"""创建文档分块"""
|
||||
chunk_metadata = {"chunk_index": chunk_index, **metadata}
|
||||
return DocumentChunk(text=text, metadata=chunk_metadata)
|
||||
|
||||
|
||||
class LangChainDocumentSplitter(DocumentSplitter):
|
||||
"""基于 LangChain 的分块器实现
|
||||
|
||||
根据 ProcessType 选择 RecursiveCharacterTextSplitter 或 CharacterTextSplitter。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
process_type: ProcessType,
|
||||
chunk_size: int = 500,
|
||||
overlap_size: int = 50,
|
||||
delimiter: Optional[str] = None,
|
||||
):
|
||||
super().__init__(chunk_size=chunk_size, overlap_size=overlap_size)
|
||||
self._process_type = process_type
|
||||
self._delimiter = delimiter or "\n\n"
|
||||
self._splitter = self._create_splitter()
|
||||
|
||||
def _create_splitter(self) -> RecursiveCharacterTextSplitter | CharacterTextSplitter:
|
||||
"""创建 LangChain 分块器实例"""
|
||||
if self._process_type == ProcessType.LENGTH_CHUNK:
|
||||
return CharacterTextSplitter(
|
||||
chunk_size=self.chunk_size,
|
||||
chunk_overlap=self.overlap_size,
|
||||
length_function=len,
|
||||
)
|
||||
|
||||
separators = SEPARATORS_BY_PROCESS_TYPE.get(self._process_type)
|
||||
if self._process_type == ProcessType.CUSTOM_SEPARATOR_CHUNK:
|
||||
separators = [self._delimiter, "\n", " ", ""]
|
||||
if separators is None:
|
||||
separators = ["\n\n", "\n", " ", ""]
|
||||
|
||||
return RecursiveCharacterTextSplitter(
|
||||
chunk_size=self.chunk_size,
|
||||
chunk_overlap=self.overlap_size,
|
||||
separators=separators,
|
||||
length_function=len,
|
||||
)
|
||||
|
||||
async def split(self, text: str, **metadata: Any) -> List[DocumentChunk]:
|
||||
"""分割文档（异步）"""
|
||||
if not text or not text.strip():
|
||||
return []
|
||||
|
||||
texts = await asyncio.to_thread(self._splitter.split_text, text)
|
||||
return [
|
||||
DocumentChunk(text=t, metadata={**metadata, "chunk_index": i})
|
||||
for i, t in enumerate(texts)
|
||||
]
|
||||
|
||||
|
||||
class DocumentSplitterFactory:
|
||||
"""文档分块器工厂
|
||||
|
||||
根据处理类型创建对应的分块器实例。
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def create_splitter(
|
||||
cls,
|
||||
process_type: ProcessType,
|
||||
chunk_size: int = 500,
|
||||
overlap_size: int = 50,
|
||||
delimiter: Optional[str] = None,
|
||||
) -> DocumentSplitter:
|
||||
"""创建分块器
|
||||
|
||||
Args:
|
||||
process_type: 处理类型
|
||||
chunk_size: 分块大小
|
||||
overlap_size: 重叠大小
|
||||
delimiter: 自定义分隔符（仅用于 CUSTOM_SEPARATOR_CHUNK）
|
||||
|
||||
Returns:
|
||||
DocumentSplitter 实例
|
||||
"""
|
||||
return LangChainDocumentSplitter(
|
||||
process_type=process_type,
|
||||
chunk_size=chunk_size,
|
||||
overlap_size=overlap_size,
|
||||
delimiter=delimiter,
|
||||
)
|
||||
@@ -0,0 +1,116 @@
|
||||
"""
|
||||
文档处理类型定义
|
||||
|
||||
包含 ParsedDocument（解析后文档）和 DocumentChunk（文档分块）。
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, Any, List, Optional
|
||||
from pathlib import Path
|
||||
|
||||
from langchain_core.documents import Document
|
||||
|
||||
|
||||
@dataclass
|
||||
class DocumentChunk:
|
||||
"""文档分块
|
||||
|
||||
包含分块的文本内容和元数据。
|
||||
|
||||
Attributes:
|
||||
text: 分块文本内容
|
||||
metadata: 分块元数据（包含文件信息、分块索引等）
|
||||
"""
|
||||
text: str
|
||||
metadata: dict
|
||||
|
||||
def __repr__(self):
|
||||
return f"<DocumentChunk(text_length={len(self.text)}, metadata_keys={list(self.metadata.keys())})>"
|
||||
|
||||
|
||||
class ParsedDocument:
|
||||
"""解析后的文档
|
||||
|
||||
包含文档的完整文本内容和元数据，由文档加载器生成，供分块器使用。
|
||||
|
||||
Attributes:
|
||||
text: 文档完整文本内容
|
||||
metadata: 文档元数据（如文件名、扩展名、路径等）
|
||||
file_name: 文件名
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text: str,
|
||||
metadata: Dict[str, Any],
|
||||
file_name: str
|
||||
):
|
||||
"""初始化解析后的文档
|
||||
|
||||
Args:
|
||||
text: 文档文本内容
|
||||
metadata: 文档元数据（如作者、创建时间等）
|
||||
file_name: 文件名
|
||||
"""
|
||||
self.text = text
|
||||
self.metadata = metadata
|
||||
self.file_name = file_name
|
||||
|
||||
def __repr__(self):
|
||||
return f"<ParsedDocument(file_name={self.file_name}, text_length={len(self.text)})>"
|
||||
|
||||
|
||||
def langchain_documents_to_parsed(
|
||||
documents: List[Document],
|
||||
file_path: str,
|
||||
file_name: Optional[str] = None,
|
||||
**extra_metadata: Any,
|
||||
) -> ParsedDocument:
|
||||
"""将 LangChain Document 列表转换为 ParsedDocument
|
||||
|
||||
多页/多段结果合并为一个文档，用于后续分块处理。
|
||||
|
||||
Args:
|
||||
documents: LangChain 加载器返回的 Document 列表
|
||||
file_path: 源文件路径
|
||||
file_name: 文件名，若提供则优先使用
|
||||
**extra_metadata: 额外的元数据字段（会合并到返回的 metadata 中）
|
||||
|
||||
Returns:
|
||||
ParsedDocument: 合并后的文档对象
|
||||
"""
|
||||
path = Path(file_path)
|
||||
name = file_name or path.name
|
||||
|
||||
if not documents:
|
||||
base_metadata = {
|
||||
"file_name": name,
|
||||
"file_extension": path.suffix.lower(),
|
||||
"file_size": path.stat().st_size if path.exists() else 0,
|
||||
}
|
||||
base_metadata.update(extra_metadata)
|
||||
return ParsedDocument(
|
||||
text="",
|
||||
metadata=base_metadata,
|
||||
file_name=name,
|
||||
)
|
||||
|
||||
texts = [d.page_content for d in documents if d.page_content]
|
||||
merged_text = "\n\n".join(texts)
|
||||
|
||||
meta: Dict[str, Any] = {
|
||||
"file_name": name,
|
||||
"file_extension": path.suffix.lower(),
|
||||
"file_size": path.stat().st_size if path.exists() else 0,
|
||||
"absolute_directory_path": str(path.parent),
|
||||
"file_path": str(path),
|
||||
}
|
||||
|
||||
meta.update(extra_metadata)
|
||||
|
||||
if documents and isinstance(documents[0].metadata, dict):
|
||||
first_meta = documents[0].metadata
|
||||
for k, v in first_meta.items():
|
||||
if k not in meta and v is not None:
|
||||
meta[k] = v
|
||||
|
||||
return ParsedDocument(text=merged_text, metadata=meta, file_name=name)
|
||||
Reference in New Issue
Block a user