deepdata: core product code (backend, frontend, runtime, deepflow module)
This commit is contained in:
figmar
@@ -0,0 +1,244 @@
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Any, List, Optional, Dict
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator, ConfigDict
|
||||
|
||||
|
||||
class TextSplitConfig(BaseModel):
|
||||
"""文本切片配置"""
|
||||
chunk_size: int = Field(..., description="最大令牌数")
|
||||
chunk_overlap: int = Field(..., description="重叠令牌数")
|
||||
|
||||
|
||||
class SyntheConfig(BaseModel):
|
||||
"""合成配置"""
|
||||
model_id: str = Field(..., description="模型ID")
|
||||
prompt_template: str = Field(None, description="合成提示模板")
|
||||
number: Optional[int] = Field(None, description="单个chunk合成的数据数量")
|
||||
temperature: Optional[float] = Field(None, description="温度参数")
|
||||
|
||||
|
||||
class Config(BaseModel):
|
||||
"""配置"""
|
||||
text_split_config: TextSplitConfig = Field(None, description="文本切片配置")
|
||||
question_synth_config: SyntheConfig = Field(None, description="问题合成配置")
|
||||
answer_synth_config: SyntheConfig = Field(None, description="答案合成配置")
|
||||
# 新增：整个任务允许生成的 QA 总上限（问题/答案对数量）
|
||||
max_qa_pairs: Optional[int] = Field(
|
||||
default=None,
|
||||
description="整个任务允许生成的 QA 对总量上限；为 None 或 <=0 表示不限制",
|
||||
)
|
||||
|
||||
|
||||
class SynthesisType(Enum):
|
||||
"""合成类型"""
|
||||
QA = "QA"
|
||||
COT = "COT"
|
||||
QUESTION = "QUESTION"
|
||||
|
||||
|
||||
class CreateSynthesisTaskRequest(BaseModel):
|
||||
"""创建数据合成任务请求"""
|
||||
name: str = Field(..., description="合成任务名称")
|
||||
description: Optional[str] = Field(None, description="合成任务描述")
|
||||
synthesis_type: SynthesisType = Field(..., description="合成类型")
|
||||
source_file_id: list[str] = Field(..., description="原始文件ID列表")
|
||||
synth_config: Config = Field(..., description="合成配置")
|
||||
|
||||
@field_validator("description")
|
||||
@classmethod
|
||||
def empty_string_to_none(cls, v: Optional[str]) -> Optional[str]:
|
||||
"""前端如果传入空字符串，将其统一转化为 None，避免存库时看起来像有描述但实际上为空。"""
|
||||
if isinstance(v, str) and v.strip() == "":
|
||||
return None
|
||||
return v
|
||||
|
||||
|
||||
class DataSynthesisTaskItem(BaseModel):
|
||||
"""数据合成任务列表/详情项"""
|
||||
id: str
|
||||
name: str
|
||||
description: Optional[str] = None
|
||||
status: Optional[str] = None
|
||||
synthesis_type: str
|
||||
total_files: int
|
||||
created_at: Optional[datetime] = None
|
||||
updated_at: Optional[datetime] = None
|
||||
created_by: Optional[str] = None
|
||||
updated_by: Optional[str] = None
|
||||
|
||||
# 扩展字段（带默认值，用于列表查询等复杂场景）
|
||||
model_id: str = ""
|
||||
progress: int = 0
|
||||
result_data_location: Optional[str] = None
|
||||
text_split_config: Optional[Dict[str, Any]] = None
|
||||
synthesis_config: Optional[Dict[str, Any]] = None
|
||||
source_file_id: List[str] = []
|
||||
processed_files: int = 0
|
||||
total_chunks: int = 0
|
||||
processed_chunks: int = 0
|
||||
total_synthesis_data: int = 0
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
@classmethod
|
||||
def from_orm_with_config(cls, task: Any) -> "DataSynthesisTaskItem":
|
||||
"""从 ORM 对象转换，包含完整配置（用于列表查询）"""
|
||||
synth_cfg = getattr(task, "synth_config", {}) or {}
|
||||
text_split_cfg = synth_cfg.get("text_split_config") or {}
|
||||
synthesis_cfg = synth_cfg.get("synthesis_config") or {}
|
||||
source_file_ids = synth_cfg.get("source_file_id") or []
|
||||
model_id = synth_cfg.get("model_id", "")
|
||||
result_location = synth_cfg.get("result_data_location")
|
||||
|
||||
return cls(
|
||||
id=str(task.id),
|
||||
name=str(task.name),
|
||||
description=task.description,
|
||||
status=task.status,
|
||||
synthesis_type=str(task.synth_type),
|
||||
model_id=model_id or "",
|
||||
progress=int(task.progress or 0),
|
||||
result_data_location=result_location,
|
||||
text_split_config=text_split_cfg,
|
||||
synthesis_config=synthesis_cfg,
|
||||
source_file_id=list(source_file_ids),
|
||||
total_files=int(task.total_files or 0),
|
||||
processed_files=int(task.processed_files or 0),
|
||||
total_chunks=int(task.total_chunks or 0),
|
||||
processed_chunks=int(task.processed_chunks or 0),
|
||||
total_synthesis_data=int(task.total_synth_data or 0),
|
||||
created_at=task.created_at,
|
||||
updated_at=task.updated_at,
|
||||
created_by=task.created_by,
|
||||
updated_by=task.updated_by,
|
||||
)
|
||||
|
||||
|
||||
class PagedDataSynthesisTaskResponse(BaseModel):
|
||||
"""分页数据合成任务响应"""
|
||||
content: List[DataSynthesisTaskItem]
|
||||
totalElements: int
|
||||
totalPages: int
|
||||
page: int
|
||||
size: int
|
||||
|
||||
|
||||
class DataSynthesisFileTaskItem(BaseModel):
|
||||
"""数据合成任务下的文件任务项"""
|
||||
id: str
|
||||
synthesis_instance_id: str
|
||||
file_name: str
|
||||
source_file_id: str
|
||||
status: Optional[str] = None
|
||||
total_chunks: int
|
||||
processed_chunks: int
|
||||
created_at: Optional[datetime] = None
|
||||
updated_at: Optional[datetime] = None
|
||||
created_by: Optional[str] = None
|
||||
updated_by: Optional[str] = None
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class PagedDataSynthesisFileTaskResponse(BaseModel):
|
||||
"""分页数据合成任务文件任务响应"""
|
||||
content: List[DataSynthesisFileTaskItem]
|
||||
totalElements: int
|
||||
totalPages: int
|
||||
page: int
|
||||
size: int
|
||||
|
||||
|
||||
class DataSynthesisChunkItem(BaseModel):
|
||||
"""数据合成任务下的 chunk 记录"""
|
||||
id: str
|
||||
synthesis_file_instance_id: str
|
||||
chunk_index: Optional[int] = None
|
||||
chunk_content: Optional[str] = None
|
||||
chunk_metadata: Optional[Dict[str, Any]] = None
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class PagedDataSynthesisChunkResponse(BaseModel):
|
||||
"""分页 chunk 列表响应"""
|
||||
content: List[DataSynthesisChunkItem]
|
||||
totalElements: int
|
||||
totalPages: int
|
||||
page: int
|
||||
size: int
|
||||
|
||||
|
||||
class SynthesisDataItem(BaseModel):
|
||||
"""合成结果数据项"""
|
||||
id: str
|
||||
data: Optional[Dict[str, Any]] = None
|
||||
synthesis_file_instance_id: str
|
||||
chunk_instance_id: str
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class ChatRequest(BaseModel):
|
||||
"""聊天请求参数"""
|
||||
model_id: str
|
||||
prompt: str
|
||||
|
||||
|
||||
class SynthesisDataUpdateRequest(BaseModel):
|
||||
"""单条合成数据 data 字段整体更新请求（前端传入完整 JSON，后端直接覆盖）"""
|
||||
data: Dict[str, Any] = Field(..., description="新的完整 JSON 对象，将覆盖原有 data 字段")
|
||||
|
||||
|
||||
class BatchDeleteSynthesisDataRequest(BaseModel):
|
||||
"""批量删除合成数据请求"""
|
||||
ids: List[str] = Field(..., description="需要删除的合成数据 ID 列表")
|
||||
|
||||
|
||||
class BatchDeleteChunkInstancesRequest(BaseModel):
|
||||
"""批量删除分块及其关联合成数据请求"""
|
||||
chunk_ids: List[str] = Field(..., description="需要删除的 chunk 实例 ID 列表")
|
||||
|
||||
|
||||
class BatchDeleteChunkInstancesByFileRequest(BaseModel):
|
||||
"""按文件任务维度删除 chunk 及其合成数据的请求"""
|
||||
file_id: str = Field(..., description="数据合成文件任务 ID")
|
||||
|
||||
|
||||
class BatchDeleteChunkInstancesByTaskRequest(BaseModel):
|
||||
"""按任务维度删除 chunk 及其合成数据的请求"""
|
||||
task_id: str = Field(..., description="数据合成任务 ID")
|
||||
|
||||
|
||||
class SynthesisDataPatchItem(BaseModel):
|
||||
"""用于前端展示/编辑的合成数据项（包含 chunk 与文件信息，可按需扩展）"""
|
||||
id: str
|
||||
data: Optional[Dict[str, Any]] = None
|
||||
chunk_instance_id: str
|
||||
synthesis_file_instance_id: str
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class ExportFormat(str, Enum):
|
||||
"""导出格式枚举"""
|
||||
ALPACA = "alpaca"
|
||||
SHAREGPT = "sharegpt"
|
||||
RAW = "raw"
|
||||
|
||||
|
||||
class ExportSynthesisDataRequest(BaseModel):
|
||||
"""导出合成数据请求"""
|
||||
task_id: str = Field(..., description="合成任务ID")
|
||||
file_instance_ids: Optional[List[str]] = Field(None, description="文件实例ID列表，为空则导出全部")
|
||||
format: ExportFormat = Field(ExportFormat.ALPACA, description="导出格式")
|
||||
output_path: Optional[str] = Field(None, description="输出路径，为空则使用默认路径")
|
||||
|
||||
|
||||
class ExportSynthesisDataResponse(BaseModel):
|
||||
"""导出合成数据响应"""
|
||||
file_paths: List[str] = Field(..., description="导出文件路径列表")
|
||||
total_records: int = Field(..., description="总记录数")
|
||||
format: str = Field(..., description="导出格式")
|
||||
Reference in New Issue
Block a user