deepdata: core product code (backend, frontend, runtime, deepflow module)
This commit is contained in:
figmar
@@ -0,0 +1,229 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.logging import get_logger
|
||||
from app.db.models import DatasetFiles
|
||||
|
||||
from ..client import LabelStudioClient
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class LSAnnotationSyncService:
|
||||
"""将 Label Studio 项目中的标注结果同步回 DM 数据库。
|
||||
|
||||
约定：
|
||||
- Label Studio task.data 中包含由 SyncService 写入的 DM 元信息：
|
||||
{"file_id": "...", "dataset_id": "...", "file_path": "...", "original_name": "..."}
|
||||
- 对于每个 task，我们会：
|
||||
* 读取该 task 的 annotations 列表；
|
||||
* 结合 task.predictions 一并写入 t_dm_dataset_files.annotation；
|
||||
* 从 annotations/predictions 的 result 字段中提取标签，写入 t_dm_dataset_files.tags；
|
||||
* 更新时间戳 tags_updated_at。
|
||||
"""
|
||||
|
||||
def __init__(self, db: AsyncSession, ls_client: LabelStudioClient) -> None:
|
||||
self.db = db
|
||||
self.ls_client = ls_client
|
||||
|
||||
async def sync_project_annotations_to_dm(self, project_id: str) -> int:
|
||||
"""从指定 Label Studio 项目中同步所有任务的标注到 DM 文件表。
|
||||
|
||||
Args:
|
||||
project_id: Label Studio 项目 ID（字符串形式）。
|
||||
|
||||
Returns:
|
||||
成功更新的文件数量。
|
||||
"""
|
||||
try:
|
||||
page_size = getattr(settings, "ls_task_page_size", 1000)
|
||||
result = await self.ls_client.get_project_tasks(
|
||||
project_id=project_id,
|
||||
page=None,
|
||||
page_size=page_size,
|
||||
)
|
||||
except Exception as e: # pragma: no cover - 防御性日志
|
||||
logger.error("Failed to fetch tasks from Label Studio project %s: %s", project_id, e)
|
||||
return 0
|
||||
|
||||
if not result:
|
||||
logger.warning("No tasks returned for project %s when syncing annotations", project_id)
|
||||
return 0
|
||||
|
||||
tasks: List[Dict[str, Any]] = result.get("tasks", []) # type: ignore[assignment]
|
||||
if not tasks:
|
||||
logger.info("Project %s has no tasks to sync", project_id)
|
||||
return 0
|
||||
|
||||
updated_files = 0
|
||||
|
||||
for task in tasks:
|
||||
data: Dict[str, Any] = task.get("data") or {}
|
||||
raw_file_id = data.get("file_id")
|
||||
if not raw_file_id:
|
||||
# 旧任务可能没有 DM file_id，跳过但记录日志
|
||||
logger.debug(
|
||||
"Skip LS task %s because data.file_id is missing",
|
||||
task.get("id"),
|
||||
)
|
||||
continue
|
||||
|
||||
file_id = str(raw_file_id)
|
||||
|
||||
# 获取该任务的 annotations 列表
|
||||
annotations: Optional[List[Dict[str, Any]]] = None
|
||||
try:
|
||||
annotations = await self.ls_client.get_task_annotations(int(task["id"])) # type: ignore[arg-type]
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(
|
||||
"Failed to fetch annotations for LS task %s (file_id=%s): %s",
|
||||
task.get("id"),
|
||||
file_id,
|
||||
e,
|
||||
)
|
||||
|
||||
predictions: List[Dict[str, Any]] = task.get("predictions") or []
|
||||
|
||||
annotation_payload: Dict[str, Any] = {
|
||||
"task": {
|
||||
"id": task.get("id"),
|
||||
"project": task.get("project"),
|
||||
"data": data,
|
||||
},
|
||||
"annotations": annotations or [],
|
||||
"predictions": predictions,
|
||||
}
|
||||
|
||||
tags = self._extract_tags_from_results(
|
||||
annotations or [],
|
||||
predictions,
|
||||
)
|
||||
|
||||
success = await self._update_dataset_file(file_id, tags, annotation_payload)
|
||||
if success:
|
||||
updated_files += 1
|
||||
|
||||
logger.info(
|
||||
"Synced annotations from LS project %s to DM, updated %d files",
|
||||
project_id,
|
||||
updated_files,
|
||||
)
|
||||
return updated_files
|
||||
|
||||
async def _update_dataset_file(
|
||||
self,
|
||||
file_id: str,
|
||||
tags: List[Dict[str, Any]],
|
||||
annotation: Dict[str, Any],
|
||||
) -> bool:
|
||||
"""将提取出的 tags 与 annotation 写回 t_dm_dataset_files。
|
||||
|
||||
直接更新 DatasetFiles.tags / tags_updated_at / annotation 字段，
|
||||
不走模板驱动的转换逻辑（模板转换在用户主动批量更新时使用）。
|
||||
"""
|
||||
try:
|
||||
result = await self.db.execute(
|
||||
select(DatasetFiles).where(DatasetFiles.id == file_id)
|
||||
)
|
||||
record = result.scalar_one_or_none()
|
||||
|
||||
if not record:
|
||||
logger.warning("Dataset file not found when syncing LS annotations: %s", file_id)
|
||||
return False
|
||||
|
||||
update_time = datetime.utcnow()
|
||||
record.tags = tags # type: ignore[assignment]
|
||||
record.tags_updated_at = update_time # type: ignore[assignment]
|
||||
# annotation 可能较大，但为 JSONB，直接整体覆盖
|
||||
setattr(record, "annotation", annotation)
|
||||
|
||||
await self.db.commit()
|
||||
await self.db.refresh(record)
|
||||
|
||||
logger.debug(
|
||||
"Updated DM file %s with %d tags from LS annotations",
|
||||
file_id,
|
||||
len(tags),
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error("Failed to update DM file %s from LS annotations: %s", file_id, e)
|
||||
try:
|
||||
await self.db.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _extract_tags_from_results(
|
||||
annotations: List[Dict[str, Any]],
|
||||
predictions: List[Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""从 annotations 和 predictions 的 result 字段中提取通用标签结构。
|
||||
|
||||
输出格式与 DM 前端当前解析逻辑兼容：
|
||||
- 每个元素包含: id/type/from_name/values；
|
||||
- 其中 values 是一个字典，键通常与 type 一致，例如 "rectanglelabels"；
|
||||
- 前端通过 tag.values[tag.type] 提取字符串标签集合。
|
||||
"""
|
||||
|
||||
def iter_results() -> List[Dict[str, Any]]:
|
||||
res: List[Dict[str, Any]] = []
|
||||
for ann in annotations or []:
|
||||
for item in ann.get("result", []) or []:
|
||||
if isinstance(item, dict):
|
||||
res.append(item)
|
||||
for pred in predictions or []:
|
||||
for item in pred.get("result", []) or []:
|
||||
if isinstance(item, dict):
|
||||
res.append(item)
|
||||
return res
|
||||
|
||||
results = iter_results()
|
||||
if not results:
|
||||
return []
|
||||
|
||||
normalized: List[Dict[str, Any]] = []
|
||||
|
||||
for r in results:
|
||||
r_type = r.get("type")
|
||||
if isinstance(r_type, str):
|
||||
r_type = r_type.strip().lower()
|
||||
|
||||
from_name = r.get("from_name") or r.get("fromName")
|
||||
to_name = r.get("to_name") or r.get("toName")
|
||||
value_obj = r.get("value") or {}
|
||||
if not isinstance(value_obj, dict):
|
||||
continue
|
||||
|
||||
# 将 Label Studio 的 value 映射为 values，方便前端统一解析
|
||||
values: Dict[str, Any] = {}
|
||||
for key, v in value_obj.items():
|
||||
normalized_key = str(key).strip().lower() if key is not None else ""
|
||||
if normalized_key:
|
||||
values[normalized_key] = v
|
||||
|
||||
if isinstance(r_type, str) and r_type:
|
||||
if r_type in values:
|
||||
values = {r_type: values[r_type]}
|
||||
elif len(values) == 1:
|
||||
only_value = next(iter(values.values()))
|
||||
values = {r_type: only_value}
|
||||
|
||||
tag = {
|
||||
"id": r.get("id"),
|
||||
"type": r_type,
|
||||
"from_name": from_name,
|
||||
"to_name": to_name,
|
||||
"values": values,
|
||||
}
|
||||
normalized.append(tag)
|
||||
|
||||
return normalized
|
||||
Reference in New Issue
Block a user