feat(deepdata): 引入修改密码、文件预览组件与清洗算子增强，deepflow 改名为 deep-flow
本次同步 09-12 至 09-28 的代码变更，主要内容： 新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务 功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema） 结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本 未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
This commit is contained in:
沃巴蒂
@@ -6,12 +6,10 @@ from app.module.cleaning.runtime_client import RuntimeClient
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# 模块级单例：进程内所有请求共享同一个调度器
|
||||
# 模块级单例
|
||||
_scheduler: "CleaningTaskScheduler | None" = None
|
||||
|
||||
|
||||
def get_scheduler() -> "CleaningTaskScheduler":
|
||||
"""获取全局调度器单例，不存在则创建"""
|
||||
global _scheduler
|
||||
if _scheduler is None:
|
||||
_scheduler = CleaningTaskScheduler(
|
||||
@@ -20,23 +18,22 @@ def get_scheduler() -> "CleaningTaskScheduler":
|
||||
)
|
||||
return _scheduler
|
||||
|
||||
|
||||
class CleaningTaskScheduler:
|
||||
"""Scheduler for executing cleaning tasks"""
|
||||
|
||||
def __init__(self, task_repo: CleaningTaskRepository, runtime_client: RuntimeClient):
|
||||
self.task_repo = task_repo
|
||||
self.runtime_client = runtime_client
|
||||
self._polling_task_ids: set[str] = set() # 待轮询的任务 ID 集合
|
||||
self._polling_task_ids: set[str] = set()
|
||||
self._polling_started: bool = False
|
||||
self._poll_failure_count: dict[str, int] = {} # runtime 不可达连续失败计数
|
||||
self._MAX_POLL_FAILURES = 1 # 一次超时(60s) → 标记 FAILED
|
||||
self._poll_failure_count: dict[str, int] = {}
|
||||
self._not_found_counts: dict[str, int] = {} # 🔴 新增：记录 NOT_FOUND 次数
|
||||
self._MAX_POLL_FAILURES = 1
|
||||
|
||||
async def execute_task(self, db: AsyncSession, task_id: str, retry_count: int) -> bool:
|
||||
"""Execute cleaning task"""
|
||||
from app.module.cleaning.schema import CleaningTaskDto, CleaningTaskStatus
|
||||
from datetime import datetime
|
||||
|
||||
logger.info(f"🚀 [Scheduler] 任务 {task_id} 开始执行...")
|
||||
|
||||
task = CleaningTaskDto()
|
||||
task.id = task_id
|
||||
task.status = CleaningTaskStatus.RUNNING
|
||||
@@ -53,108 +50,130 @@ class CleaningTaskScheduler:
|
||||
task.status = CleaningTaskStatus.FAILED
|
||||
task.finished_at = datetime.now()
|
||||
await self.task_repo.update_task(db, task)
|
||||
|
||||
return submitted
|
||||
|
||||
async def stop_task(self, db: AsyncSession, task_id: str) -> bool:
|
||||
"""Stop cleaning task"""
|
||||
from app.module.cleaning.schema import CleaningTaskDto, CleaningTaskStatus
|
||||
|
||||
self._polling_task_ids.discard(task_id)
|
||||
self._poll_failure_count.pop(task_id, None)
|
||||
self._not_found_counts.pop(task_id, None) # 🔴 清理计数器
|
||||
|
||||
await self.runtime_client.stop_task(task_id)
|
||||
|
||||
task = CleaningTaskDto()
|
||||
task.id = task_id
|
||||
task.status = CleaningTaskStatus.STOPPED
|
||||
|
||||
await self.task_repo.update_task(db, task)
|
||||
return True
|
||||
|
||||
async def startup(self):
|
||||
"""进程启动时调用：恢复未完成任务的轮询 + 启动全局轮询协程"""
|
||||
from app.module.cleaning.schema import CleaningTaskStatus
|
||||
from app.db.session import AsyncSessionLocal
|
||||
|
||||
# 从数据库恢复所有 RUNNING 状态的任务
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
tasks = await self.task_repo.find_tasks(db, status=CleaningTaskStatus.RUNNING)
|
||||
for task in tasks:
|
||||
logger.info(f"[Polling] Recovered RUNNING task from DB: {task.id}")
|
||||
logger.info(f"[Polling] Recovered RUNNING task: {task.id}")
|
||||
self._polling_task_ids.add(task.id)
|
||||
except Exception as e:
|
||||
logger.error(f"[Polling] Failed to recover tasks from DB: {e}")
|
||||
logger.error(f"[Polling] Failed to recover tasks: {e}")
|
||||
|
||||
# 启动全局轮询协程
|
||||
if not self._polling_started:
|
||||
self._polling_started = True
|
||||
asyncio.create_task(self._poll_all_tasks())
|
||||
logger.info("[Polling] Global polling loop started")
|
||||
|
||||
async def _poll_all_tasks(self):
|
||||
"""全局轮询协程：每 2 秒轮询所有 RUNNING 任务的 runtime 状态"""
|
||||
from app.module.cleaning.schema import CleaningTaskDto, CleaningTaskStatus
|
||||
from app.db.session import AsyncSessionLocal
|
||||
from datetime import datetime
|
||||
|
||||
logger.info("[Polling] Global status polling loop started")
|
||||
terminal_statuses = {"completed", "failed", "cancelled", "stopped", "not_found"}
|
||||
logger.info("[Polling] Global polling loop started")
|
||||
terminal_statuses = {"completed", "failed", "cancelled", "stopped"}
|
||||
|
||||
while True:
|
||||
task_ids = list(self._polling_task_ids)
|
||||
if not task_ids:
|
||||
# 没有待轮询任务，休眠后继续等
|
||||
await asyncio.sleep(2)
|
||||
continue
|
||||
|
||||
for task_id in task_ids:
|
||||
try:
|
||||
status_data = await self.runtime_client.get_task_status(task_id)
|
||||
|
||||
if status_data is None:
|
||||
# runtime 不可达，累计失败计数
|
||||
count = self._poll_failure_count.get(task_id, 0) + 1
|
||||
self._poll_failure_count[task_id] = count
|
||||
logger.warning(
|
||||
f"[Polling] Task {task_id} unreachable ({count}/{self._MAX_POLL_FAILURES})"
|
||||
)
|
||||
if count >= self._MAX_POLL_FAILURES:
|
||||
logger.error(
|
||||
f"[Polling] Task {task_id} marked FAILED: "
|
||||
f"runtime unreachable after {count} attempts"
|
||||
)
|
||||
logger.error(f"[Polling] Task {task_id} FAILED (runtime unreachable)")
|
||||
async with AsyncSessionLocal() as db:
|
||||
task_dto = CleaningTaskDto()
|
||||
task_dto.id = task_id
|
||||
task_dto.status = CleaningTaskStatus.FAILED
|
||||
task_dto.finished_at = datetime.now()
|
||||
await self.task_repo.update_task(db, task_dto)
|
||||
t = CleaningTaskDto()
|
||||
t.id = task_id
|
||||
t.status = CleaningTaskStatus.FAILED
|
||||
t.finished_at = datetime.now()
|
||||
await self.task_repo.update_task(db, t)
|
||||
await db.commit()
|
||||
self._polling_task_ids.discard(task_id)
|
||||
self._poll_failure_count.pop(task_id, None)
|
||||
continue
|
||||
|
||||
# runtime 可达，重置失败计数
|
||||
self._poll_failure_count.pop(task_id, None)
|
||||
|
||||
current_status = (status_data.get("status", "") or "").lower()
|
||||
logger.debug(f"[Polling] Task {task_id} status: {current_status}")
|
||||
|
||||
# 🔴 核心修复：处理 NOT_FOUND
|
||||
if current_status == 'not_found':
|
||||
cnt = self._not_found_counts.get(task_id, 0) + 1
|
||||
self._not_found_counts[task_id] = cnt
|
||||
|
||||
# 阶段 1：启动容忍期 (10秒内认为是在初始化)
|
||||
if cnt <= 5:
|
||||
continue # 继续轮询，不报错
|
||||
|
||||
# 阶段 2：检查数据库状态 (判断是真丢失还是已完成)
|
||||
try:
|
||||
async with AsyncSessionLocal() as check_db:
|
||||
db_task = await self.task_repo.find_task_by_id(check_db, task_id)
|
||||
db_status = db_task.status if db_task else "UNKNOWN"
|
||||
|
||||
# ✅ 成功收尾：如果数据库已经是终态，说明任务其实跑完了
|
||||
# 此时 Runtime 清理了内存状态返回 NOT_FOUND，我们应停止轮询
|
||||
if db_status in ["COMPLETED", "FAILED", "STOPPED"]:
|
||||
logger.info(f"[Scheduler] 任务 {task_id} 已结束 (DB: {db_status})，停止轮询。")
|
||||
self._polling_task_ids.discard(task_id)
|
||||
self._not_found_counts.pop(task_id, None)
|
||||
continue
|
||||
|
||||
# ✅ 失败收尾：DB 还是 RUNNING，但 Runtime 彻底找不到它了 (>60秒)
|
||||
if cnt > 30:
|
||||
logger.error(f"[Scheduler] 任务 {task_id} 在 Runtime 丢失，标记 FAILED。")
|
||||
t = CleaningTaskDto()
|
||||
t.id = task_id
|
||||
t.status = CleaningTaskStatus.FAILED
|
||||
t.finished_at = datetime.now()
|
||||
await self.task_repo.update_task(check_db, t)
|
||||
await check_db.commit()
|
||||
|
||||
self._polling_task_ids.discard(task_id)
|
||||
self._not_found_counts.pop(task_id, None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
continue # 继续等待
|
||||
|
||||
# 如果找到了，重置计数器
|
||||
self._not_found_counts.pop(task_id, None)
|
||||
|
||||
# ==========================================
|
||||
|
||||
if current_status in terminal_statuses:
|
||||
logger.info(f"[Scheduler] 任务 {task_id} 结束: {current_status}")
|
||||
async with AsyncSessionLocal() as db:
|
||||
task = CleaningTaskDto()
|
||||
task.id = task_id
|
||||
if current_status == "completed":
|
||||
task.status = CleaningTaskStatus.COMPLETED
|
||||
else:
|
||||
task.status = CleaningTaskStatus.FAILED
|
||||
task.finished_at = datetime.now()
|
||||
await self.task_repo.update_task(db, task)
|
||||
t = CleaningTaskDto()
|
||||
t.id = task_id
|
||||
t.status = CleaningTaskStatus.COMPLETED if current_status == "completed" else CleaningTaskStatus.FAILED
|
||||
t.finished_at = datetime.now()
|
||||
await self.task_repo.update_task(db, t)
|
||||
await db.commit()
|
||||
logger.info(
|
||||
f"[Polling] Task {task_id} finished: {current_status}"
|
||||
)
|
||||
self._polling_task_ids.discard(task_id)
|
||||
self._poll_failure_count.pop(task_id, None)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user