沃巴蒂
ff22ed3a8b
本次同步 09-12 至 09-28 的代码变更，主要内容： 新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务 功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema） 结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本 未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
184 lines
8.6 KiB
Python
184 lines
8.6 KiB
Python
import asyncio
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from app.core.logging import get_logger
|
|
from app.module.cleaning.repository import CleaningTaskRepository
|
|
from app.module.cleaning.runtime_client import RuntimeClient
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
# 模块级单例
|
|
_scheduler: "CleaningTaskScheduler | None" = None
|
|
|
|
def get_scheduler() -> "CleaningTaskScheduler":
|
|
global _scheduler
|
|
if _scheduler is None:
|
|
_scheduler = CleaningTaskScheduler(
|
|
CleaningTaskRepository(None),
|
|
RuntimeClient()
|
|
)
|
|
return _scheduler
|
|
|
|
class CleaningTaskScheduler:
|
|
def __init__(self, task_repo: CleaningTaskRepository, runtime_client: RuntimeClient):
|
|
self.task_repo = task_repo
|
|
self.runtime_client = runtime_client
|
|
self._polling_task_ids: set[str] = set()
|
|
self._polling_started: bool = False
|
|
self._poll_failure_count: dict[str, int] = {}
|
|
self._not_found_counts: dict[str, int] = {} # 🔴 新增：记录 NOT_FOUND 次数
|
|
self._MAX_POLL_FAILURES = 1
|
|
|
|
async def execute_task(self, db: AsyncSession, task_id: str, retry_count: int) -> bool:
|
|
from app.module.cleaning.schema import CleaningTaskDto, CleaningTaskStatus
|
|
from datetime import datetime
|
|
|
|
logger.info(f"🚀 [Scheduler] 任务 {task_id} 开始执行...")
|
|
|
|
task = CleaningTaskDto()
|
|
task.id = task_id
|
|
task.status = CleaningTaskStatus.RUNNING
|
|
task.started_at = datetime.now()
|
|
task.retry_count = retry_count
|
|
|
|
submitted = await self.runtime_client.submit_task(task_id, retry_count)
|
|
|
|
if submitted:
|
|
await self.task_repo.update_task(db, task)
|
|
self._polling_task_ids.add(task_id)
|
|
return submitted
|
|
|
|
task.status = CleaningTaskStatus.FAILED
|
|
task.finished_at = datetime.now()
|
|
await self.task_repo.update_task(db, task)
|
|
return submitted
|
|
|
|
async def stop_task(self, db: AsyncSession, task_id: str) -> bool:
|
|
from app.module.cleaning.schema import CleaningTaskDto, CleaningTaskStatus
|
|
|
|
self._polling_task_ids.discard(task_id)
|
|
self._poll_failure_count.pop(task_id, None)
|
|
self._not_found_counts.pop(task_id, None) # 🔴 清理计数器
|
|
|
|
await self.runtime_client.stop_task(task_id)
|
|
|
|
task = CleaningTaskDto()
|
|
task.id = task_id
|
|
task.status = CleaningTaskStatus.STOPPED
|
|
await self.task_repo.update_task(db, task)
|
|
return True
|
|
|
|
async def startup(self):
|
|
from app.module.cleaning.schema import CleaningTaskStatus
|
|
from app.db.session import AsyncSessionLocal
|
|
try:
|
|
async with AsyncSessionLocal() as db:
|
|
tasks = await self.task_repo.find_tasks(db, status=CleaningTaskStatus.RUNNING)
|
|
for task in tasks:
|
|
logger.info(f"[Polling] Recovered RUNNING task: {task.id}")
|
|
self._polling_task_ids.add(task.id)
|
|
except Exception as e:
|
|
logger.error(f"[Polling] Failed to recover tasks: {e}")
|
|
|
|
if not self._polling_started:
|
|
self._polling_started = True
|
|
asyncio.create_task(self._poll_all_tasks())
|
|
|
|
async def _poll_all_tasks(self):
|
|
from app.module.cleaning.schema import CleaningTaskDto, CleaningTaskStatus
|
|
from app.db.session import AsyncSessionLocal
|
|
from datetime import datetime
|
|
|
|
logger.info("[Polling] Global polling loop started")
|
|
terminal_statuses = {"completed", "failed", "cancelled", "stopped"}
|
|
|
|
while True:
|
|
task_ids = list(self._polling_task_ids)
|
|
if not task_ids:
|
|
await asyncio.sleep(2)
|
|
continue
|
|
|
|
for task_id in task_ids:
|
|
try:
|
|
status_data = await self.runtime_client.get_task_status(task_id)
|
|
|
|
if status_data is None:
|
|
count = self._poll_failure_count.get(task_id, 0) + 1
|
|
self._poll_failure_count[task_id] = count
|
|
if count >= self._MAX_POLL_FAILURES:
|
|
logger.error(f"[Polling] Task {task_id} FAILED (runtime unreachable)")
|
|
async with AsyncSessionLocal() as db:
|
|
t = CleaningTaskDto()
|
|
t.id = task_id
|
|
t.status = CleaningTaskStatus.FAILED
|
|
t.finished_at = datetime.now()
|
|
await self.task_repo.update_task(db, t)
|
|
await db.commit()
|
|
self._polling_task_ids.discard(task_id)
|
|
self._poll_failure_count.pop(task_id, None)
|
|
continue
|
|
|
|
self._poll_failure_count.pop(task_id, None)
|
|
current_status = (status_data.get("status", "") or "").lower()
|
|
|
|
# 🔴 核心修复：处理 NOT_FOUND
|
|
if current_status == 'not_found':
|
|
cnt = self._not_found_counts.get(task_id, 0) + 1
|
|
self._not_found_counts[task_id] = cnt
|
|
|
|
# 阶段 1：启动容忍期 (10秒内认为是在初始化)
|
|
if cnt <= 5:
|
|
continue # 继续轮询，不报错
|
|
|
|
# 阶段 2：检查数据库状态 (判断是真丢失还是已完成)
|
|
try:
|
|
async with AsyncSessionLocal() as check_db:
|
|
db_task = await self.task_repo.find_task_by_id(check_db, task_id)
|
|
db_status = db_task.status if db_task else "UNKNOWN"
|
|
|
|
# ✅ 成功收尾：如果数据库已经是终态，说明任务其实跑完了
|
|
# 此时 Runtime 清理了内存状态返回 NOT_FOUND，我们应停止轮询
|
|
if db_status in ["COMPLETED", "FAILED", "STOPPED"]:
|
|
logger.info(f"[Scheduler] 任务 {task_id} 已结束 (DB: {db_status})，停止轮询。")
|
|
self._polling_task_ids.discard(task_id)
|
|
self._not_found_counts.pop(task_id, None)
|
|
continue
|
|
|
|
# ✅ 失败收尾：DB 还是 RUNNING，但 Runtime 彻底找不到它了 (>60秒)
|
|
if cnt > 30:
|
|
logger.error(f"[Scheduler] 任务 {task_id} 在 Runtime 丢失，标记 FAILED。")
|
|
t = CleaningTaskDto()
|
|
t.id = task_id
|
|
t.status = CleaningTaskStatus.FAILED
|
|
t.finished_at = datetime.now()
|
|
await self.task_repo.update_task(check_db, t)
|
|
await check_db.commit()
|
|
|
|
self._polling_task_ids.discard(task_id)
|
|
self._not_found_counts.pop(task_id, None)
|
|
except Exception:
|
|
pass
|
|
|
|
continue # 继续等待
|
|
|
|
# 如果找到了，重置计数器
|
|
self._not_found_counts.pop(task_id, None)
|
|
|
|
# ==========================================
|
|
|
|
if current_status in terminal_statuses:
|
|
logger.info(f"[Scheduler] 任务 {task_id} 结束: {current_status}")
|
|
async with AsyncSessionLocal() as db:
|
|
t = CleaningTaskDto()
|
|
t.id = task_id
|
|
t.status = CleaningTaskStatus.COMPLETED if current_status == "completed" else CleaningTaskStatus.FAILED
|
|
t.finished_at = datetime.now()
|
|
await self.task_repo.update_task(db, t)
|
|
await db.commit()
|
|
self._polling_task_ids.discard(task_id)
|
|
self._poll_failure_count.pop(task_id, None)
|
|
|
|
except Exception as e:
|
|
logger.error(f"[Polling] Error polling task {task_id}: {e}")
|
|
|
|
await asyncio.sleep(2)
|