deepdata: core product code (backend, frontend, runtime, deepflow module)
This commit is contained in:
figmar
@@ -0,0 +1,349 @@
|
||||
"""问答生成器模块 - 负责问题生成和答案生成"""
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.language_models import BaseChatModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models.data_synthesis import (
|
||||
DataSynthesisChunkInstance,
|
||||
DataSynthesisFileInstance,
|
||||
SynthesisData,
|
||||
)
|
||||
from app.db.session import logger
|
||||
from app.module.generation.schema.generation import SyntheConfig
|
||||
from app.module.generation.service.prompt import (
|
||||
ANSWER_GENERATOR_PROMPT,
|
||||
QUESTION_GENERATOR_PROMPT,
|
||||
)
|
||||
from app.module.shared.util.model_chat import extract_json_substring
|
||||
from app.module.shared.llm import LLMFactory
|
||||
|
||||
|
||||
# 图片URL提取正则
|
||||
IMG_URL_PATTERN = re.compile(r"!\[\]\((.*?)\)")
|
||||
|
||||
|
||||
def extract_img_urls(text: str) -> list[str]:
|
||||
"""提取文本中的图片URL"""
|
||||
return IMG_URL_PATTERN.findall(text)
|
||||
|
||||
|
||||
def _filter_docs_by_size(docs: list[Any], chunk_size: int, ratio: float = 0.7) -> list[Any]:
|
||||
"""过滤文档，保留长度大于 chunk_size * ratio 的文档
|
||||
|
||||
Args:
|
||||
docs: 文档列表
|
||||
chunk_size: 切片大小
|
||||
ratio: 长度比例阈值
|
||||
|
||||
Returns:
|
||||
过滤后的文档列表
|
||||
"""
|
||||
min_length = chunk_size * ratio
|
||||
return [doc for doc in docs if len(doc.page_content) >= min_length]
|
||||
|
||||
|
||||
class QuestionGenerator:
|
||||
"""问题生成器"""
|
||||
|
||||
def __init__(self, db: AsyncSession, semaphore: asyncio.Semaphore | None = None):
|
||||
self.db = db
|
||||
self.semaphore = semaphore or asyncio.Semaphore(10)
|
||||
|
||||
async def generate_questions(
|
||||
self,
|
||||
chunk_text: str,
|
||||
question_cfg: SyntheConfig,
|
||||
question_chat: BaseChatModel,
|
||||
) -> list[str]:
|
||||
"""针对单个切片文本，调用模型生成问题列表
|
||||
|
||||
Args:
|
||||
chunk_text: 切片文本内容
|
||||
question_cfg: 问题生成配置
|
||||
question_chat: 问题生成模型
|
||||
|
||||
Returns:
|
||||
问题列表
|
||||
"""
|
||||
# 计算问题数量：每千tokens生成指定数量的问题
|
||||
base_number = question_cfg.number or 5
|
||||
calculated_number = max(int(len(chunk_text) / 1000 * base_number), 1)
|
||||
|
||||
# 获取提示模板
|
||||
template = getattr(question_cfg, "prompt_template", None)
|
||||
template = template if template and template.strip() else QUESTION_GENERATOR_PROMPT
|
||||
|
||||
# 构建提示
|
||||
prompt = (
|
||||
template
|
||||
.replace("{text}", chunk_text)
|
||||
.replace("{number}", str(calculated_number))
|
||||
.replace("{textLength}", str(len(chunk_text)))
|
||||
)
|
||||
|
||||
# 调用模型
|
||||
async with self.semaphore:
|
||||
loop = asyncio.get_running_loop()
|
||||
raw_answer = await loop.run_in_executor(
|
||||
None,
|
||||
LLMFactory.invoke_sync,
|
||||
question_chat,
|
||||
prompt,
|
||||
)
|
||||
|
||||
# 解析问题列表
|
||||
return self._parse_questions(raw_answer)
|
||||
|
||||
def _parse_questions(self, raw_answer: str) -> list[str]:
|
||||
"""从模型返回中解析问题列表
|
||||
|
||||
Args:
|
||||
raw_answer: 模型原始返回
|
||||
|
||||
Returns:
|
||||
问题列表
|
||||
"""
|
||||
if not raw_answer:
|
||||
return []
|
||||
|
||||
cleaned = extract_json_substring(raw_answer)
|
||||
try:
|
||||
data = json.loads(cleaned)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse question list JSON: {e}")
|
||||
return []
|
||||
|
||||
if isinstance(data, list):
|
||||
return [str(q) for q in data if isinstance(q, str) and q.strip()]
|
||||
|
||||
# 容错：如果是单个字符串
|
||||
if isinstance(data, str) and data.strip():
|
||||
return [data.strip()]
|
||||
|
||||
return []
|
||||
|
||||
|
||||
class AnswerGenerator:
|
||||
"""答案生成器"""
|
||||
|
||||
def __init__(self, db: AsyncSession, semaphore: asyncio.Semaphore | None = None):
|
||||
self.db = db
|
||||
self.semaphore = semaphore or asyncio.Semaphore(10)
|
||||
|
||||
async def generate_answers_for_chunk(
|
||||
self,
|
||||
file_task: DataSynthesisFileInstance,
|
||||
chunk: DataSynthesisChunkInstance,
|
||||
questions: list[str],
|
||||
answer_cfg: SyntheConfig,
|
||||
answer_chat: BaseChatModel,
|
||||
) -> int:
|
||||
"""为一个切片的所有问题生成答案并写入数据库
|
||||
|
||||
Args:
|
||||
file_task: 文件任务实例
|
||||
chunk: 切片实例
|
||||
questions: 问题列表
|
||||
answer_cfg: 答案生成配置
|
||||
answer_chat: 答案生成模型
|
||||
|
||||
Returns:
|
||||
成功生成的QA对数量
|
||||
"""
|
||||
if not questions:
|
||||
return 0
|
||||
|
||||
chunk_text = chunk.chunk_content or ""
|
||||
template = getattr(answer_cfg, "prompt_template", None)
|
||||
template = template if template and template.strip() else ANSWER_GENERATOR_PROMPT
|
||||
|
||||
success_count = 0
|
||||
|
||||
async def process_question(question: str) -> bool:
|
||||
nonlocal success_count
|
||||
try:
|
||||
prompt = template.replace("{text}", chunk_text).replace("{question}", question)
|
||||
|
||||
async with self.semaphore:
|
||||
loop = asyncio.get_running_loop()
|
||||
answer = await loop.run_in_executor(
|
||||
None,
|
||||
LLMFactory.invoke_sync,
|
||||
answer_chat,
|
||||
prompt,
|
||||
)
|
||||
|
||||
# 构建数据对象
|
||||
data_obj = self._build_synthesis_data(chunk_text, question, answer)
|
||||
|
||||
# 保存记录
|
||||
record = SynthesisData(
|
||||
id=str(uuid.uuid4()),
|
||||
data=data_obj,
|
||||
synthesis_file_instance_id=file_task.id,
|
||||
chunk_instance_id=chunk.id,
|
||||
)
|
||||
self.db.add(record)
|
||||
success_count += 1
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to generate answer for question '{question[:50]}...': {e}")
|
||||
return False
|
||||
|
||||
# 并行处理所有问题
|
||||
tasks = [process_question(q) for q in questions]
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
if success_count > 0:
|
||||
await self.db.commit()
|
||||
|
||||
return success_count
|
||||
|
||||
def _build_synthesis_data(
|
||||
self,
|
||||
chunk_text: str,
|
||||
question: str,
|
||||
answer: str,
|
||||
) -> dict[str, Any]:
|
||||
"""构建合成数据结构
|
||||
|
||||
Args:
|
||||
chunk_text: 切片原文
|
||||
question: 问题
|
||||
answer: 答案
|
||||
|
||||
Returns:
|
||||
结构化数据字典
|
||||
"""
|
||||
# 基础结构
|
||||
base_obj: dict[str, Any] = {
|
||||
"input": chunk_text,
|
||||
"output": answer,
|
||||
}
|
||||
|
||||
# 尝试解析模型返回的JSON
|
||||
parsed_obj: dict[str, Any] | None = None
|
||||
if isinstance(answer, str):
|
||||
cleaned = extract_json_substring(answer)
|
||||
try:
|
||||
parsed = json.loads(cleaned)
|
||||
if isinstance(parsed, dict):
|
||||
parsed_obj = parsed
|
||||
except Exception:
|
||||
parsed_obj = None
|
||||
|
||||
# 合并解析结果
|
||||
if parsed_obj is not None:
|
||||
parsed_obj["instruction"] = question
|
||||
data_obj = parsed_obj
|
||||
else:
|
||||
base_obj["instruction"] = question
|
||||
data_obj = base_obj
|
||||
|
||||
# 提取图片URL
|
||||
img_urls = extract_img_urls(chunk_text)
|
||||
if img_urls:
|
||||
data_obj["img_urls"] = img_urls
|
||||
|
||||
return data_obj
|
||||
|
||||
|
||||
class QAGenerator:
|
||||
"""问答生成器 - 整合问题和答案生成"""
|
||||
|
||||
def __init__(self, db: AsyncSession):
|
||||
self.db = db
|
||||
self.question_semaphore = asyncio.Semaphore(10)
|
||||
self.answer_semaphore = asyncio.Semaphore(10)
|
||||
self.question_generator = QuestionGenerator(db, self.question_semaphore)
|
||||
self.answer_generator = AnswerGenerator(db, self.answer_semaphore)
|
||||
|
||||
async def process_chunk_qa(
|
||||
self,
|
||||
file_task: DataSynthesisFileInstance,
|
||||
chunk: DataSynthesisChunkInstance,
|
||||
question_cfg: SyntheConfig,
|
||||
answer_cfg: SyntheConfig,
|
||||
question_chat: Any,
|
||||
answer_chat: Any,
|
||||
) -> int:
|
||||
"""处理单个切片的QA生成
|
||||
|
||||
Args:
|
||||
file_task: 文件任务实例
|
||||
chunk: 切片实例
|
||||
question_cfg: 问题生成配置
|
||||
answer_cfg: 答案生成配置
|
||||
question_chat: 问题生成模型
|
||||
answer_chat: 答案生成模型
|
||||
|
||||
Returns:
|
||||
成功生成的QA对数量
|
||||
"""
|
||||
chunk_text = chunk.chunk_content or ""
|
||||
if not chunk_text.strip():
|
||||
logger.warning(f"Empty chunk text for chunk_index={chunk.chunk_index}")
|
||||
return 0
|
||||
|
||||
# 1. 生成问题
|
||||
try:
|
||||
questions = await self.question_generator.generate_questions(
|
||||
chunk_text=chunk_text,
|
||||
question_cfg=question_cfg,
|
||||
question_chat=question_chat,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Generate questions failed: {e}")
|
||||
questions = []
|
||||
|
||||
if not questions:
|
||||
logger.info(f"No questions generated for chunk_index={chunk.chunk_index}")
|
||||
return 0
|
||||
|
||||
# 2. 生成答案
|
||||
success_count = await self.answer_generator.generate_answers_for_chunk(
|
||||
file_task=file_task,
|
||||
chunk=chunk,
|
||||
questions=questions,
|
||||
answer_cfg=answer_cfg,
|
||||
answer_chat=answer_chat,
|
||||
)
|
||||
|
||||
return success_count
|
||||
|
||||
async def check_qa_limit(
|
||||
self,
|
||||
synth_task_id: str,
|
||||
max_qa_pairs: int | None,
|
||||
) -> bool:
|
||||
"""检查是否已达到QA对数量上限
|
||||
|
||||
Args:
|
||||
synth_task_id: 合成任务ID
|
||||
max_qa_pairs: 最大QA对数量限制
|
||||
|
||||
Returns:
|
||||
是否已达到上限
|
||||
"""
|
||||
if max_qa_pairs is None or max_qa_pairs <= 0:
|
||||
return False
|
||||
|
||||
result = await self.db.execute(
|
||||
select(func.count(SynthesisData.id)).where(
|
||||
SynthesisData.synthesis_file_instance_id.in_(
|
||||
select(DataSynthesisFileInstance.id).where(
|
||||
DataSynthesisFileInstance.synthesis_instance_id == synth_task_id
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
current_count = int(result.scalar() or 0)
|
||||
|
||||
return current_count >= max_qa_pairs
|
||||
Reference in New Issue
Block a user