from datetime import datetime from enum import Enum from typing import Any, List, Optional, Dict from pydantic import BaseModel, Field, field_validator, ConfigDict class TextSplitConfig(BaseModel): """文本切片配置""" chunk_size: int = Field(..., description="最大令牌数") chunk_overlap: int = Field(..., description="重叠令牌数") class SyntheConfig(BaseModel): """合成配置""" model_id: str = Field(..., description="模型ID") prompt_template: str = Field(None, description="合成提示模板") number: Optional[int] = Field(None, description="单个chunk合成的数据数量") temperature: Optional[float] = Field(None, description="温度参数") class Config(BaseModel): """配置""" text_split_config: TextSplitConfig = Field(None, description="文本切片配置") question_synth_config: SyntheConfig = Field(None, description="问题合成配置") answer_synth_config: SyntheConfig = Field(None, description="答案合成配置") # 新增:整个任务允许生成的 QA 总上限(问题/答案对数量) max_qa_pairs: Optional[int] = Field( default=None, description="整个任务允许生成的 QA 对总量上限;为 None 或 <=0 表示不限制", ) class SynthesisType(Enum): """合成类型""" QA = "QA" COT = "COT" QUESTION = "QUESTION" class CreateSynthesisTaskRequest(BaseModel): """创建数据合成任务请求""" name: str = Field(..., description="合成任务名称") description: Optional[str] = Field(None, description="合成任务描述") synthesis_type: SynthesisType = Field(..., description="合成类型") source_file_id: list[str] = Field(..., description="原始文件ID列表") synth_config: Config = Field(..., description="合成配置") @field_validator("description") @classmethod def empty_string_to_none(cls, v: Optional[str]) -> Optional[str]: """前端如果传入空字符串,将其统一转化为 None,避免存库时看起来像有描述但实际上为空。""" if isinstance(v, str) and v.strip() == "": return None return v class DataSynthesisTaskItem(BaseModel): """数据合成任务列表/详情项""" id: str name: str description: Optional[str] = None status: Optional[str] = None synthesis_type: str total_files: int created_at: Optional[datetime] = None updated_at: Optional[datetime] = None created_by: Optional[str] = None updated_by: Optional[str] = None # 扩展字段(带默认值,用于列表查询等复杂场景) model_id: str = "" progress: int = 0 result_data_location: Optional[str] = None text_split_config: Optional[Dict[str, Any]] = None synthesis_config: Optional[Dict[str, Any]] = None source_file_id: List[str] = [] processed_files: int = 0 total_chunks: int = 0 processed_chunks: int = 0 total_synthesis_data: int = 0 model_config = ConfigDict(from_attributes=True) @classmethod def from_orm_with_config(cls, task: Any) -> "DataSynthesisTaskItem": """从 ORM 对象转换,包含完整配置(用于列表查询)""" synth_cfg = getattr(task, "synth_config", {}) or {} text_split_cfg = synth_cfg.get("text_split_config") or {} synthesis_cfg = synth_cfg.get("synthesis_config") or {} source_file_ids = synth_cfg.get("source_file_id") or [] model_id = synth_cfg.get("model_id", "") result_location = synth_cfg.get("result_data_location") return cls( id=str(task.id), name=str(task.name), description=task.description, status=task.status, synthesis_type=str(task.synth_type), model_id=model_id or "", progress=int(task.progress or 0), result_data_location=result_location, text_split_config=text_split_cfg, synthesis_config=synthesis_cfg, source_file_id=list(source_file_ids), total_files=int(task.total_files or 0), processed_files=int(task.processed_files or 0), total_chunks=int(task.total_chunks or 0), processed_chunks=int(task.processed_chunks or 0), total_synthesis_data=int(task.total_synth_data or 0), created_at=task.created_at, updated_at=task.updated_at, created_by=task.created_by, updated_by=task.updated_by, ) class PagedDataSynthesisTaskResponse(BaseModel): """分页数据合成任务响应""" content: List[DataSynthesisTaskItem] totalElements: int totalPages: int page: int size: int class DataSynthesisFileTaskItem(BaseModel): """数据合成任务下的文件任务项""" id: str synthesis_instance_id: str file_name: str source_file_id: str status: Optional[str] = None total_chunks: int processed_chunks: int created_at: Optional[datetime] = None updated_at: Optional[datetime] = None created_by: Optional[str] = None updated_by: Optional[str] = None model_config = ConfigDict(from_attributes=True) class PagedDataSynthesisFileTaskResponse(BaseModel): """分页数据合成任务文件任务响应""" content: List[DataSynthesisFileTaskItem] totalElements: int totalPages: int page: int size: int class DataSynthesisChunkItem(BaseModel): """数据合成任务下的 chunk 记录""" id: str synthesis_file_instance_id: str chunk_index: Optional[int] = None chunk_content: Optional[str] = None chunk_metadata: Optional[Dict[str, Any]] = None model_config = ConfigDict(from_attributes=True) class PagedDataSynthesisChunkResponse(BaseModel): """分页 chunk 列表响应""" content: List[DataSynthesisChunkItem] totalElements: int totalPages: int page: int size: int class SynthesisDataItem(BaseModel): """合成结果数据项""" id: str data: Optional[Dict[str, Any]] = None synthesis_file_instance_id: str chunk_instance_id: str model_config = ConfigDict(from_attributes=True) class ChatRequest(BaseModel): """聊天请求参数""" model_id: str prompt: str class SynthesisDataUpdateRequest(BaseModel): """单条合成数据 data 字段整体更新请求(前端传入完整 JSON,后端直接覆盖)""" data: Dict[str, Any] = Field(..., description="新的完整 JSON 对象,将覆盖原有 data 字段") class BatchDeleteSynthesisDataRequest(BaseModel): """批量删除合成数据请求""" ids: List[str] = Field(..., description="需要删除的合成数据 ID 列表") class BatchDeleteChunkInstancesRequest(BaseModel): """批量删除分块及其关联合成数据请求""" chunk_ids: List[str] = Field(..., description="需要删除的 chunk 实例 ID 列表") class BatchDeleteChunkInstancesByFileRequest(BaseModel): """按文件任务维度删除 chunk 及其合成数据的请求""" file_id: str = Field(..., description="数据合成文件任务 ID") class BatchDeleteChunkInstancesByTaskRequest(BaseModel): """按任务维度删除 chunk 及其合成数据的请求""" task_id: str = Field(..., description="数据合成任务 ID") class SynthesisDataPatchItem(BaseModel): """用于前端展示/编辑的合成数据项(包含 chunk 与文件信息,可按需扩展)""" id: str data: Optional[Dict[str, Any]] = None chunk_instance_id: str synthesis_file_instance_id: str model_config = ConfigDict(from_attributes=True) class ExportFormat(str, Enum): """导出格式枚举""" ALPACA = "alpaca" SHAREGPT = "sharegpt" RAW = "raw" class ExportSynthesisDataRequest(BaseModel): """导出合成数据请求""" task_id: str = Field(..., description="合成任务ID") file_instance_ids: Optional[List[str]] = Field(None, description="文件实例ID列表,为空则导出全部") format: ExportFormat = Field(ExportFormat.ALPACA, description="导出格式") output_path: Optional[str] = Field(None, description="输出路径,为空则使用默认路径") class ExportSynthesisDataResponse(BaseModel): """导出合成数据响应""" file_paths: List[str] = Field(..., description="导出文件路径列表") total_records: int = Field(..., description="总记录数") format: str = Field(..., description="导出格式")