deepdata: core product code (backend, frontend, runtime, deepflow module)
This commit is contained in:
figmar
@@ -0,0 +1,58 @@
|
||||
"""
|
||||
RAG 模块 Schema 导出
|
||||
|
||||
集中导出所有数据模型、请求和响应 DTO
|
||||
"""
|
||||
from .enums import ProcessType
|
||||
from app.db.models.knowledge_gen import RagType, FileStatus
|
||||
from .types import RagChunk
|
||||
from app.db.models.knowledge_gen import KnowledgeBase, RagFile
|
||||
from .request import (
|
||||
KnowledgeBaseCreateReq,
|
||||
KnowledgeBaseUpdateReq,
|
||||
KnowledgeBaseQueryReq,
|
||||
AddFilesReq,
|
||||
DeleteFilesReq,
|
||||
RagFileReq,
|
||||
RetrieveReq,
|
||||
FileInfo,
|
||||
PagingQuery,
|
||||
QueryRequest,
|
||||
)
|
||||
from .response import (
|
||||
ModelConfig,
|
||||
KnowledgeBaseResp,
|
||||
RagFileResp,
|
||||
RagChunkResp,
|
||||
SearchResult,
|
||||
PagedResponse,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Enums
|
||||
"RagType",
|
||||
"ProcessType",
|
||||
"FileStatus",
|
||||
# Entities
|
||||
"KnowledgeBase",
|
||||
"RagFile",
|
||||
"RagChunk",
|
||||
# Requests
|
||||
"KnowledgeBaseCreateReq",
|
||||
"KnowledgeBaseUpdateReq",
|
||||
"KnowledgeBaseQueryReq",
|
||||
"AddFilesReq",
|
||||
"DeleteFilesReq",
|
||||
"RagFileReq",
|
||||
"RetrieveReq",
|
||||
"FileInfo",
|
||||
"PagingQuery",
|
||||
"QueryRequest",
|
||||
# Responses
|
||||
"ModelConfig",
|
||||
"KnowledgeBaseResp",
|
||||
"RagFileResp",
|
||||
"RagChunkResp",
|
||||
"SearchResult",
|
||||
"PagedResponse",
|
||||
]
|
||||
@@ -0,0 +1,21 @@
|
||||
"""
|
||||
RAG 模块枚举定义
|
||||
|
||||
包含所有 RAG 相关的枚举类型，与 Java 枚举保持一致
|
||||
从 app.db.models.knowledge_gen 导入以避免循环依赖
|
||||
"""
|
||||
from enum import Enum
|
||||
|
||||
# 从模型导入以避免循环依赖
|
||||
|
||||
|
||||
class ProcessType(str, Enum):
|
||||
"""分块处理类型枚举
|
||||
|
||||
对应 Java: com.deepdata.rag.indexer.interfaces.dto.ProcessType
|
||||
"""
|
||||
PARAGRAPH_CHUNK = "PARAGRAPH_CHUNK" # 段落分块
|
||||
SENTENCE_CHUNK = "SENTENCE_CHUNK" # 按句子分块
|
||||
LENGTH_CHUNK = "LENGTH_CHUNK" # 按长度分块（字符）
|
||||
DEFAULT_CHUNK = "DEFAULT_CHUNK" # 默认分块（单词）
|
||||
CUSTOM_SEPARATOR_CHUNK = "CUSTOM_SEPARATOR_CHUNK" # 自定义分隔符分块
|
||||
@@ -0,0 +1,393 @@
|
||||
"""
|
||||
RAG 模块请求 DTO
|
||||
|
||||
使用 Pydantic 定义所有请求参数验证
|
||||
与 Java DTO 保持字段一致和验证规则一致
|
||||
"""
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
from typing import List, Optional
|
||||
from .enums import ProcessType
|
||||
from app.db.models.knowledge_gen import RagType, FileStatus
|
||||
|
||||
|
||||
class KnowledgeBaseCreateReq(BaseModel):
|
||||
"""知识库创建请求
|
||||
|
||||
对应 Java: com.deepdata.rag.indexer.interfaces.dto.KnowledgeBaseCreateReq
|
||||
"""
|
||||
name: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
max_length=255,
|
||||
pattern=r"^[a-zA-Z][a-zA-Z0-9_]*$",
|
||||
description="知识库名称（必须以字母开头，只能包含字母、数字和下划线）"
|
||||
)
|
||||
description: Optional[str] = Field(
|
||||
None,
|
||||
max_length=512,
|
||||
description="知识库描述"
|
||||
)
|
||||
type: RagType = Field(
|
||||
default=RagType.DOCUMENT,
|
||||
description="RAG 类型"
|
||||
)
|
||||
embedding_model: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
alias="embeddingModel",
|
||||
description="嵌入模型ID"
|
||||
)
|
||||
chat_model: Optional[str] = Field(
|
||||
None,
|
||||
alias="chatModel",
|
||||
description="聊天模型ID"
|
||||
)
|
||||
|
||||
class Config:
|
||||
populate_by_name = True
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"name": "my_knowledge_base",
|
||||
"description": "我的知识库",
|
||||
"type": "DOCUMENT",
|
||||
"embeddingModel": "text-embedding-ada-002",
|
||||
"chatModel": "gpt-4"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class KnowledgeBaseUpdateReq(BaseModel):
|
||||
"""知识库更新请求
|
||||
|
||||
对应 Java: com.deepdata.rag.indexer.interfaces.dto.KnowledgeBaseUpdateReq
|
||||
"""
|
||||
name: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
max_length=255,
|
||||
pattern=r"^[a-zA-Z][a-zA-Z0-9_]*$",
|
||||
description="知识库名称"
|
||||
)
|
||||
description: Optional[str] = Field(
|
||||
None,
|
||||
max_length=512,
|
||||
description="知识库描述"
|
||||
)
|
||||
|
||||
class Config:
|
||||
populate_by_name = True
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"name": "updated_knowledge_base",
|
||||
"description": "更新后的描述"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class KnowledgeBaseQueryReq(BaseModel):
|
||||
"""知识库查询请求
|
||||
|
||||
对应 Java: com.deepdata.rag.indexer.interfaces.dto.KnowledgeBaseQueryReq
|
||||
"""
|
||||
page: int = Field(
|
||||
default=1,
|
||||
ge=1,
|
||||
description="页码（从 1 开始）"
|
||||
)
|
||||
page_size: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
le=100,
|
||||
alias="size",
|
||||
description="每页数量"
|
||||
)
|
||||
keyword: Optional[str] = Field(
|
||||
None,
|
||||
max_length=255,
|
||||
alias="name",
|
||||
description="搜索关键词（模糊匹配知识库名称或描述）"
|
||||
)
|
||||
type: Optional[RagType] = Field(
|
||||
None,
|
||||
description="按 RAG 类型筛选"
|
||||
)
|
||||
|
||||
class Config:
|
||||
populate_by_name = True
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"page": 1,
|
||||
"size": 10,
|
||||
"name": "测试",
|
||||
"type": "DOCUMENT"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class FileInfo(BaseModel):
|
||||
"""文件信息
|
||||
|
||||
对应 Java: com.deepdata.rag.indexer.interfaces.dto.AddFilesReq.FileInfo
|
||||
"""
|
||||
id: str = Field(..., description="文件ID (对应 t_dm_dataset_files.id)")
|
||||
file_name: str = Field(alias="fileName", description="文件名")
|
||||
|
||||
class Config:
|
||||
populate_by_name = True
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"id": "file-uuid-123",
|
||||
"fileName": "document.pdf"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class AddFilesReq(BaseModel):
|
||||
"""添加文件请求
|
||||
|
||||
对应 Java: com.deepdata.rag.indexer.interfaces.dto.AddFilesReq
|
||||
"""
|
||||
knowledge_base_id: Optional[str] = Field(None, alias="knowledgeBaseId", description="知识库ID（从路径参数获取，这里保留用于兼容）")
|
||||
process_type: ProcessType = Field(
|
||||
default=ProcessType.DEFAULT_CHUNK,
|
||||
alias="processType",
|
||||
description="分块处理类型"
|
||||
)
|
||||
chunk_size: int = Field(
|
||||
default=500,
|
||||
ge=50,
|
||||
le=2000,
|
||||
alias="chunkSize",
|
||||
description="分块大小"
|
||||
)
|
||||
overlap_size: int = Field(
|
||||
default=50,
|
||||
ge=0,
|
||||
le=500,
|
||||
alias="overlapSize",
|
||||
description="重叠大小"
|
||||
)
|
||||
delimiter: Optional[str] = Field(
|
||||
None,
|
||||
description="自定义分隔符（仅用于 CUSTOM_SEPARATOR_CHUNK）"
|
||||
)
|
||||
files: List[FileInfo] = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="文件列表"
|
||||
)
|
||||
|
||||
@field_validator("delimiter")
|
||||
@classmethod
|
||||
def validate_delimiter(cls, v, info):
|
||||
"""验证自定义分隔符"""
|
||||
if info.data.get("process_type") == ProcessType.CUSTOM_SEPARATOR_CHUNK and not v:
|
||||
raise ValueError("使用自定义分隔符分块时，delimiter 不能为空")
|
||||
return v
|
||||
|
||||
class Config:
|
||||
populate_by_name = True
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"knowledgeBaseId": "kb-uuid-123",
|
||||
"processType": "DEFAULT_CHUNK",
|
||||
"chunkSize": 500,
|
||||
"overlapSize": 50,
|
||||
"files": [
|
||||
{"id": "file-1", "fileName": "doc1.pdf"},
|
||||
{"id": "file-2", "fileName": "doc2.pdf"}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class DeleteFilesReq(BaseModel):
|
||||
"""删除文件请求
|
||||
|
||||
对应 Java: com.deepdata.rag.indexer.interfaces.dto.DeleteFilesReq
|
||||
"""
|
||||
file_ids: List[str] = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
alias="ids",
|
||||
description="要删除的文件ID列表"
|
||||
)
|
||||
|
||||
class Config:
|
||||
populate_by_name = True
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"ids": ["file-1", "file-2", "file-3"]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class RagFileReq(BaseModel):
|
||||
"""RAG 文件查询请求
|
||||
|
||||
对应 Java: com.deepdata.rag.indexer.interfaces.dto.RagFileReq
|
||||
"""
|
||||
page: int = Field(
|
||||
default=1,
|
||||
ge=1,
|
||||
description="页码"
|
||||
)
|
||||
page_size: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
le=100,
|
||||
alias="size",
|
||||
description="每页数量"
|
||||
)
|
||||
keyword: Optional[str] = Field(
|
||||
None,
|
||||
max_length=255,
|
||||
alias="fileName",
|
||||
description="搜索关键词（模糊匹配文件名）"
|
||||
)
|
||||
status: Optional[FileStatus] = Field(
|
||||
None,
|
||||
description="按状态筛选"
|
||||
)
|
||||
|
||||
class Config:
|
||||
populate_by_name = True
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"page": 1,
|
||||
"size": 10,
|
||||
"fileName": "测试",
|
||||
"status": "PROCESSED"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class RetrieveReq(BaseModel):
|
||||
"""检索请求
|
||||
|
||||
对应 Java: com.deepdata.rag.indexer.interfaces.dto.RetrieveReq
|
||||
"""
|
||||
query: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
description="检索查询文本"
|
||||
)
|
||||
top_k: int = Field(
|
||||
default=5,
|
||||
ge=1,
|
||||
le=20,
|
||||
alias="topK",
|
||||
description="返回前 K 个结果"
|
||||
)
|
||||
threshold: Optional[float] = Field(
|
||||
None,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="相似度阈值（仅返回分数大于等于该值的结果）"
|
||||
)
|
||||
knowledge_base_ids: List[str] = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
alias="knowledgeBaseIds",
|
||||
description="要检索的知识库ID列表"
|
||||
)
|
||||
|
||||
class Config:
|
||||
populate_by_name = True
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"query": "什么是机器学习？",
|
||||
"topK": 5,
|
||||
"threshold": 0.7,
|
||||
"knowledgeBaseIds": ["kb-1", "kb-2"]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class PagingQuery(BaseModel):
|
||||
"""分页查询请求
|
||||
|
||||
对应 Java: com.deepdata.common.interfaces.PagingQuery
|
||||
"""
|
||||
page: int = Field(
|
||||
default=1,
|
||||
ge=1,
|
||||
description="页码（从 1 开始）"
|
||||
)
|
||||
size: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
le=100,
|
||||
description="每页数量"
|
||||
)
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"page": 1,
|
||||
"size": 10
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class ChunkFilterQuery(BaseModel):
|
||||
"""分块过滤查询请求
|
||||
|
||||
支持分页和 Milvus 表达式过滤
|
||||
"""
|
||||
page: int = Field(
|
||||
default=1,
|
||||
ge=1,
|
||||
description="页码（从 1 开始）"
|
||||
)
|
||||
size: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
le=100,
|
||||
description="每页数量"
|
||||
)
|
||||
expr: Optional[str] = Field(
|
||||
None,
|
||||
description="Milvus 过滤表达式（如 id > \"1\" && text like \"%keyword%\"）"
|
||||
)
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"page": 1,
|
||||
"size": 10,
|
||||
"expr": "id > \"1\""
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class QueryRequest(BaseModel):
|
||||
"""知识图谱查询请求"""
|
||||
knowledge_base_id: str = Field(..., description="知识库ID")
|
||||
query: str = Field(..., description="查询文本")
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"knowledge_base_id": "kb-uuid-123",
|
||||
"query": "什么是机器学习？"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class ChunkUpdateReq(BaseModel):
|
||||
"""Chunk更新请求"""
|
||||
text: str = Field(..., min_length=1, description="分块文本内容")
|
||||
metadata: Optional[dict] = Field(default=None, description="元数据")
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"text": "这是修改后的分块内容...",
|
||||
"metadata": {
|
||||
"fileName": "document.pdf",
|
||||
"chunkIndex": 0,
|
||||
"customField": "custom value"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
"""
|
||||
RAG 模块响应 DTO
|
||||
|
||||
定义所有 API 响应的数据结构
|
||||
与 Java 响应 DTO 保持字段一致
|
||||
"""
|
||||
from pydantic import BaseModel, Field, ConfigDict
|
||||
from typing import List, Optional, Any
|
||||
from datetime import datetime
|
||||
from app.db.models.knowledge_gen import RagType, FileStatus
|
||||
|
||||
|
||||
class ModelConfig(BaseModel):
|
||||
"""模型配置信息
|
||||
|
||||
对应 Java: com.deepdata.common.setting.domain.entity.ModelConfig
|
||||
"""
|
||||
model_config = ConfigDict(from_attributes=True, populate_by_name=True)
|
||||
|
||||
id: str = Field(..., description="模型ID")
|
||||
created_at: Optional[datetime] = Field(None, alias="createdAt", description="创建时间")
|
||||
updated_at: Optional[datetime] = Field(None, alias="updatedAt", description="更新时间")
|
||||
created_by: Optional[str] = Field(None, alias="createdBy", description="创建人")
|
||||
updated_by: Optional[str] = Field(None, alias="updatedBy", description="更新人")
|
||||
model_name: str = Field(..., alias="modelName", description="模型名称")
|
||||
provider: str = Field(..., description="模型提供商")
|
||||
base_url: str = Field(..., alias="baseUrl", description="API 基础地址")
|
||||
api_key: str = Field(default="", alias="apiKey", description="API 密钥")
|
||||
type: str = Field(..., description="模型类型")
|
||||
is_enabled: bool = Field(default=True, alias="isEnabled", description="是否启用")
|
||||
is_default: bool = Field(default=False, alias="isDefault", description="是否默认")
|
||||
is_deleted: bool = Field(default=False, alias="isDeleted", description="是否删除")
|
||||
|
||||
|
||||
class KnowledgeBaseResp(BaseModel):
|
||||
"""知识库响应
|
||||
|
||||
对应 Java: com.deepdata.rag.indexer.interfaces.dto.KnowledgeBaseResp
|
||||
"""
|
||||
model_config = ConfigDict(
|
||||
from_attributes=True,
|
||||
populate_by_name=True,
|
||||
json_schema_extra={
|
||||
"example": {
|
||||
"id": "kb-uuid-123",
|
||||
"name": "my_knowledge_base",
|
||||
"description": "我的知识库",
|
||||
"type": "DOCUMENT",
|
||||
"embeddingModel": "text-embedding-ada-002",
|
||||
"chatModel": "gpt-4",
|
||||
"fileCount": 10,
|
||||
"chunkCount": 150,
|
||||
"embedding": {
|
||||
"id": "model-1",
|
||||
"modelName": "text-embedding-ada-002",
|
||||
"provider": "openai"
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
id: str = Field(..., description="知识库ID")
|
||||
name: str = Field(..., description="知识库名称")
|
||||
description: Optional[str] = Field(None, description="知识库描述")
|
||||
type: RagType = Field(..., description="RAG类型")
|
||||
embedding_model: str = Field(alias="embeddingModel", description="嵌入模型ID")
|
||||
chat_model: Optional[str] = Field(None, alias="chatModel", description="聊天模型ID")
|
||||
file_count: Optional[int] = Field(None, alias="fileCount", description="文件数量")
|
||||
chunk_count: Optional[int] = Field(None, alias="chunkCount", description="分块数量")
|
||||
embedding: Optional[ModelConfig] = Field(None, description="嵌入模型配置")
|
||||
chat: Optional[ModelConfig] = Field(None, description="聊天模型配置")
|
||||
created_at: Optional[datetime] = Field(None, alias="createdAt", description="创建时间")
|
||||
updated_at: Optional[datetime] = Field(None, alias="updatedAt", description="更新时间")
|
||||
created_by: Optional[str] = Field(None, alias="createdBy", description="创建人")
|
||||
updated_by: Optional[str] = Field(None, alias="updatedBy", description="更新人")
|
||||
|
||||
|
||||
class RagFileResp(BaseModel):
|
||||
"""RAG 文件响应
|
||||
|
||||
对应 Java: com.deepdata.rag.indexer.domain.model.RagFile
|
||||
"""
|
||||
model_config = ConfigDict(
|
||||
from_attributes=True,
|
||||
populate_by_name=True,
|
||||
json_schema_extra={
|
||||
"example": {
|
||||
"id": "rag-file-uuid-123",
|
||||
"knowledgeBaseId": "kb-uuid-123",
|
||||
"fileName": "document.pdf",
|
||||
"fileId": "file-uuid-456",
|
||||
"chunkCount": 15,
|
||||
"metadata": {"size": 1024, "format": "pdf"},
|
||||
"status": "PROCESSED",
|
||||
"progress": 100,
|
||||
"createdAt": "2025-01-01T00:00:00"
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
id: str = Field(..., description="RAG文件ID")
|
||||
knowledge_base_id: str = Field(alias="knowledgeBaseId", description="知识库ID")
|
||||
file_name: str = Field(alias="fileName", description="文件名")
|
||||
file_id: str = Field(alias="fileId", description="原始文件ID")
|
||||
chunk_count: Optional[int] = Field(None, alias="chunkCount", description="分块数量")
|
||||
metadata: Optional[dict] = Field(None, validation_alias="file_metadata", description="元数据")
|
||||
status: FileStatus = Field(..., description="处理状态")
|
||||
err_msg: Optional[str] = Field(None, alias="errMsg", description="错误信息")
|
||||
progress: int = Field(default=0, ge=0, le=100, description="处理进度(0-100)")
|
||||
created_at: Optional[datetime] = Field(None, alias="createdAt", description="创建时间")
|
||||
updated_at: Optional[datetime] = Field(None, alias="updatedAt", description="更新时间")
|
||||
created_by: Optional[str] = Field(None, alias="createdBy", description="创建人")
|
||||
updated_by: Optional[str] = Field(None, alias="updatedBy", description="更新人")
|
||||
|
||||
|
||||
class RagChunkResp(BaseModel):
|
||||
"""RAG 分块响应
|
||||
|
||||
对应 Milvus 查询结果
|
||||
"""
|
||||
id: str = Field(..., description="分块ID")
|
||||
text: str = Field(..., description="分块文本")
|
||||
metadata: dict = Field(..., description="元数据")
|
||||
score: float = Field(..., description="相似度分数")
|
||||
distance: Optional[float] = Field(None, description="距离（可选）")
|
||||
|
||||
class Config:
|
||||
populate_by_name = True # 允许使用 snake_case 或 camelCase
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"id": "chunk-uuid-123",
|
||||
"text": "这是文档的一个分块内容...",
|
||||
"metadata": {
|
||||
"fileName": "document.pdf",
|
||||
"chunkIndex": 0
|
||||
},
|
||||
"score": 0.95
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class SearchResult(BaseModel):
|
||||
"""检索结果
|
||||
|
||||
对应 Java: io.milvus.v2.service.vector.response.SearchResp.SearchResult
|
||||
"""
|
||||
id: str = Field(..., description="结果ID")
|
||||
score: float = Field(..., description="相似度分数")
|
||||
text: str = Field(..., description="文本内容")
|
||||
metadata: dict = Field(default_factory=dict, description="元数据")
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"id": "chunk-uuid-123",
|
||||
"score": 0.95,
|
||||
"text": "相关文档内容...",
|
||||
"metadata": {"file_name": "doc.pdf"}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class UnifiedSearchResult(BaseModel):
|
||||
"""统一检索结果
|
||||
|
||||
所有知识库类型（向量、知识图谱）的检索接口必须返回此格式
|
||||
"""
|
||||
model_config = ConfigDict(
|
||||
populate_by_name=True,
|
||||
json_schema_extra={
|
||||
"example": {
|
||||
"id": "result-uuid-123",
|
||||
"text": "相关文档内容...",
|
||||
"score": 0.95,
|
||||
"metadata": {"fileName": "document.pdf", "chunkIndex": 0},
|
||||
"resultType": "vector",
|
||||
"knowledgeBaseId": "kb-uuid-123",
|
||||
"knowledgeBaseName": "my_knowledge_base"
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
id: str = Field(..., description="结果ID")
|
||||
text: str = Field(..., description="文本内容")
|
||||
score: float = Field(..., description="相似度分数")
|
||||
metadata: dict = Field(default_factory=dict, description="元数据")
|
||||
result_type: str = Field(alias="resultType", description="结果类型（vector/graph）")
|
||||
knowledge_base_id: str = Field(alias="knowledgeBaseId", description="知识库ID")
|
||||
knowledge_base_name: str = Field(alias="knowledgeBaseName", description="知识库名称")
|
||||
|
||||
|
||||
class PagedResponse(BaseModel):
|
||||
"""分页响应
|
||||
|
||||
对应 Java: com.deepdata.common.interfaces.PagedResponse
|
||||
"""
|
||||
content: List[Any] = Field(..., description="数据列表")
|
||||
total_elements: int = Field(alias="totalElements", description="总记录数")
|
||||
page: int = Field(..., description="当前页码")
|
||||
size: int = Field(..., description="每页数量")
|
||||
total_pages: int = Field(alias="totalPages", description="总页数")
|
||||
|
||||
@classmethod
|
||||
def create(cls, content: List[Any], total_elements: int, page: int, size: int):
|
||||
"""创建分页响应
|
||||
|
||||
Args:
|
||||
content: 数据列表
|
||||
total_elements: 总记录数
|
||||
page: 当前页码
|
||||
size: 每页数量
|
||||
|
||||
Returns:
|
||||
PagedResponse 实例
|
||||
"""
|
||||
total_pages = (total_elements + size - 1) // size if size > 0 else 0
|
||||
return cls(
|
||||
content=content,
|
||||
total_elements=total_elements,
|
||||
page=page,
|
||||
size=size,
|
||||
total_pages=total_pages
|
||||
)
|
||||
|
||||
class Config:
|
||||
populate_by_name = True # 允许使用 snake_case 或 camelCase
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"content": [],
|
||||
"totalElements": 100,
|
||||
"page": 1,
|
||||
"size": 10,
|
||||
"totalPages": 10
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
"""
|
||||
RAG 模块非 ORM 实体
|
||||
|
||||
RagChunk 为 Milvus 中存储的文档分块 DTO，非数据库表。
|
||||
ORM 实体 KnowledgeBase、RagFile 已移至 app.db.models.knowledge_gen。
|
||||
"""
|
||||
# noqa: D104 (RagChunk 是领域模型，不是 DB 表)
|
||||
|
||||
|
||||
class RagChunk:
|
||||
"""RAG 分块模型
|
||||
|
||||
对应 Java: com.deepdata.rag.indexer.domain.model.RagChunk
|
||||
注意：这不是数据库实体，而是 Milvus 中存储的文档分块
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
chunk_id: str,
|
||||
rag_file_id: str,
|
||||
text: str,
|
||||
metadata: dict,
|
||||
vector: list[float] = None,
|
||||
sparse_vector: dict[int, float] = None
|
||||
):
|
||||
"""初始化文档分块
|
||||
|
||||
Args:
|
||||
chunk_id: 分块ID
|
||||
rag_file_id: 关联的 RAG 文件 ID
|
||||
text: 分块文本内容
|
||||
metadata: 元数据（包含文件信息、分块索引等）
|
||||
vector: 密集向量（嵌入向量）
|
||||
sparse_vector: 稀疏向量（BM25 向量）
|
||||
"""
|
||||
self.chunk_id = chunk_id
|
||||
self.rag_file_id = rag_file_id
|
||||
self.text = text
|
||||
self.metadata = metadata
|
||||
self.vector = vector
|
||||
self.sparse_vector = sparse_vector
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""转换为字典格式（用于 Milvus 插入）
|
||||
|
||||
Returns:
|
||||
包含所有字段的字典
|
||||
"""
|
||||
return {
|
||||
"id": self.chunk_id,
|
||||
"text": self.text,
|
||||
"metadata": self.metadata,
|
||||
"vector": self.vector,
|
||||
"sparse": self.sparse_vector or {}
|
||||
}
|
||||
|
||||
def __repr__(self):
|
||||
return f"<RagChunk(id={self.chunk_id}, text={self.text[:50]}...)>"
|
||||
Reference in New Issue
Block a user