沃巴蒂
ff22ed3a8b
本次同步 09-12 至 09-28 的代码变更，主要内容： 新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务 功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema） 结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本 未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
1250 lines
53 KiB
Python
1250 lines
53 KiB
Python
from typing import Optional, List, Dict, Any, Tuple, Set
|
||
import json
|
||
import os
|
||
|
||
from app.module.dataset import DatasetManagementService
|
||
from sqlalchemy import update, select
|
||
from app.db.models import DatasetFiles
|
||
|
||
from app.core.logging import get_logger
|
||
from app.core.config import settings
|
||
from app.core.exception import ErrorCodes, BusinessError
|
||
|
||
from ..client import LabelStudioClient
|
||
from ..schema import (
|
||
SyncDatasetResponse,
|
||
DatasetMappingResponse,
|
||
SyncAnnotationsResponse
|
||
)
|
||
from ..service.mapping import DatasetMappingService
|
||
|
||
logger = get_logger(__name__)
|
||
|
||
class SyncService:
|
||
"""数据同步服务"""
|
||
|
||
def __init__(
|
||
self,
|
||
dm_client: DatasetManagementService,
|
||
ls_client: LabelStudioClient,
|
||
mapping_service: DatasetMappingService
|
||
):
|
||
self.dm_client = dm_client
|
||
self.ls_client = ls_client
|
||
self.mapping_service = mapping_service
|
||
|
||
def _determine_data_type(self, file_type: str) -> str:
|
||
"""根据文件类型确定数据类型"""
|
||
file_type_lower = file_type.lower()
|
||
|
||
type_mapping = {
|
||
'image': ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'svg', 'webp'],
|
||
'audio': ['mp3', 'wav', 'flac', 'aac', 'ogg'],
|
||
'video': ['mp4', 'avi', 'mov', 'wmv', 'flv', 'webm'],
|
||
'text': ['txt', 'doc', 'docx', 'pdf'],
|
||
'wsi': ['svs', 'tiff', 'ndpi', 'mrxs', 'sdpc'],
|
||
'ct': ['dcm', 'dicom', 'nii', 'nii.gz']
|
||
}
|
||
|
||
for data_type, extensions in type_mapping.items():
|
||
if any(ext in file_type_lower for ext in extensions):
|
||
return data_type
|
||
|
||
return 'image' # 默认为图像类型
|
||
|
||
@staticmethod
|
||
def _normalize_ls_result_for_compare(result_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||
"""Normalize LS result items for deterministic comparison."""
|
||
normalized: List[Dict[str, Any]] = []
|
||
|
||
for item in result_items or []:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
|
||
from_name = item.get("from_name") or item.get("fromName") or ""
|
||
to_name = item.get("to_name") or item.get("toName") or ""
|
||
type_name = item.get("type") or ""
|
||
if not isinstance(from_name, str):
|
||
from_name = str(from_name)
|
||
if not isinstance(to_name, str):
|
||
to_name = str(to_name)
|
||
if not isinstance(type_name, str):
|
||
type_name = str(type_name)
|
||
|
||
value_obj: Dict[str, Any] = {}
|
||
raw_value_obj = item.get("value")
|
||
if isinstance(raw_value_obj, dict):
|
||
value_obj = raw_value_obj
|
||
else:
|
||
raw_values_obj = item.get("values")
|
||
if isinstance(raw_values_obj, dict):
|
||
value_obj = raw_values_obj
|
||
|
||
normalized_value: Dict[str, Any] = {}
|
||
for value_key, value_content in value_obj.items():
|
||
key_str = str(value_key).strip().lower() if value_key is not None else ""
|
||
if key_str:
|
||
normalized_value[key_str] = value_content
|
||
|
||
normalized.append(
|
||
{
|
||
"from_name": from_name.strip(),
|
||
"to_name": to_name.strip(),
|
||
"type": type_name.strip().lower(),
|
||
"value": normalized_value,
|
||
}
|
||
)
|
||
|
||
normalized.sort(
|
||
key=lambda item: (
|
||
item.get("from_name", ""),
|
||
item.get("to_name", ""),
|
||
item.get("type", ""),
|
||
json.dumps(item.get("value", {}), sort_keys=True, ensure_ascii=False),
|
||
)
|
||
)
|
||
return normalized
|
||
|
||
def _ls_result_payload_equal(
|
||
self,
|
||
left: List[Dict[str, Any]],
|
||
right: List[Dict[str, Any]],
|
||
) -> bool:
|
||
return self._normalize_ls_result_for_compare(left) == self._normalize_ls_result_for_compare(right)
|
||
|
||
def _build_task_data(self, file_info: Any, dataset_id: str) -> dict:
|
||
"""构建Label Studio任务数据"""
|
||
data_type = self._determine_data_type(file_info.fileType)
|
||
|
||
# 默认仍然走 Label Studio 本地文件 URL
|
||
# 先替换文件路径前缀，构造 /data/local-files/?d=/... 形式
|
||
relative_path = file_info.filePath.removeprefix(settings.dm_file_path_prefix)
|
||
ls_file_url = settings.label_studio_file_path_prefix + relative_path
|
||
|
||
data_value: Any = ls_file_url
|
||
|
||
# 对于纯文本文件（例如 .txt），支持直接把文件内容写入到 data.text，
|
||
# 这样在 Label Studio 里会直接显示文本内容，而不是 URL。
|
||
if data_type == "text":
|
||
try:
|
||
_, ext = os.path.splitext(file_info.filePath)
|
||
ext = ext.lower()
|
||
|
||
# 目前只对 .txt 做内联，其他如 pdf/doc 仍然使用 URL
|
||
if ext == ".txt":
|
||
with open(file_info.filePath, "r", encoding="utf-8", errors="ignore") as f:
|
||
content = f.read()
|
||
if content:
|
||
data_value = content
|
||
except Exception as e:
|
||
# 读取失败时退回到原来的 URL 形式，避免中断同步流程
|
||
logger.warning(
|
||
"Failed to inline text content for file %s: %s",
|
||
getattr(file_info, "filePath", "<unknown>"),
|
||
str(e),
|
||
)
|
||
|
||
return {
|
||
"data": {
|
||
f"{data_type}": data_value,
|
||
"file_path": file_info.filePath,
|
||
"file_id": file_info.id,
|
||
"original_name": file_info.originalName,
|
||
"dataset_id": dataset_id,
|
||
}
|
||
}
|
||
|
||
async def _create_tasks_with_fallback(
|
||
self,
|
||
project_id: str,
|
||
tasks: List[dict]
|
||
) -> int:
|
||
"""批量创建任务，失败时回退到单个创建"""
|
||
if not tasks:
|
||
return 0
|
||
|
||
# 尝试批量创建
|
||
batch_result = await self.ls_client.create_tasks_batch(project_id, tasks)
|
||
|
||
if batch_result:
|
||
logger.debug(f"Successfully created {len(tasks)} tasks in batch")
|
||
return len(tasks)
|
||
|
||
# 批量失败，回退到单个创建
|
||
logger.warning(f"Batch creation failed, falling back to single creation")
|
||
created_count = 0
|
||
|
||
for task_data in tasks:
|
||
task_result = await self.ls_client.create_task(
|
||
project_id,
|
||
task_data["data"],
|
||
task_data.get("meta")
|
||
)
|
||
if task_result:
|
||
created_count += 1
|
||
|
||
logger.debug(f"Successfully created {created_count}/{len(tasks)} tasks individually")
|
||
return created_count
|
||
|
||
async def get_existing_dm_file_mapping(self, project_id: str) -> Dict[str, int]:
|
||
"""
|
||
获取Label Studio项目中已存在的DM文件ID到任务ID的映射
|
||
|
||
Args:
|
||
project_id: Label Studio项目ID
|
||
|
||
Returns:
|
||
file_id到task_id的映射字典
|
||
"""
|
||
try:
|
||
page_size = getattr(settings, 'ls_task_page_size', 1000)
|
||
result = await self.ls_client.get_project_tasks(
|
||
project_id=project_id,
|
||
page=None,
|
||
page_size=page_size
|
||
)
|
||
|
||
if not result:
|
||
logger.warning(f"Failed to fetch tasks for project {project_id}")
|
||
return {}
|
||
|
||
all_tasks = result.get("tasks", [])
|
||
logger.debug(f"Successfully fetched {len(all_tasks)} tasks")
|
||
|
||
# 使用字典推导式构建映射
|
||
dm_file_to_task_mapping = {
|
||
str(task.get('data', {}).get('file_id')): task.get('id')
|
||
for task in all_tasks
|
||
if task.get('data', {}).get('file_id') is not None
|
||
}
|
||
|
||
logger.debug(f"Found {len(dm_file_to_task_mapping)} existing task mappings")
|
||
return dm_file_to_task_mapping
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error while fetching existing tasks: {e}")
|
||
return {}
|
||
|
||
async def _fetch_dm_files_paginated(
|
||
self,
|
||
dataset_id: str,
|
||
batch_size: int,
|
||
existing_file_ids: Set[str],
|
||
project_id: str,
|
||
allowed_file_ids: Optional[Set[str]] = None,
|
||
) -> Tuple[Set[str], int]:
|
||
"""
|
||
分页获取DM文件并创建新任务
|
||
|
||
Returns:
|
||
(当前文件ID集合, 创建的任务数)
|
||
"""
|
||
current_file_ids = set()
|
||
total_created = 0
|
||
page = 0
|
||
|
||
while True:
|
||
files_response = await self.dm_client.get_dataset_files(
|
||
dataset_id,
|
||
page=page,
|
||
size=batch_size,
|
||
)
|
||
|
||
if not files_response or not files_response.content:
|
||
logger.debug(f"No more files on page {page + 1}")
|
||
break
|
||
|
||
logger.debug(f"Processing page {page + 1}, {len(files_response.content)} files")
|
||
|
||
# 筛选新文件并构建任务数据
|
||
new_tasks = []
|
||
for file_info in files_response.content:
|
||
file_id = str(file_info.id)
|
||
# 如果提供了允许的文件ID集合，则只同步这些文件
|
||
if allowed_file_ids is not None and file_id not in allowed_file_ids:
|
||
continue
|
||
|
||
current_file_ids.add(file_id)
|
||
|
||
if file_id not in existing_file_ids:
|
||
task_data = self._build_task_data(file_info, dataset_id)
|
||
new_tasks.append(task_data)
|
||
|
||
logger.debug(f"Page {page + 1}: {len(new_tasks)} new files, {len(files_response.content) - len(new_tasks)} existing")
|
||
|
||
# 批量创建任务
|
||
if new_tasks:
|
||
created = await self._create_tasks_with_fallback(project_id, new_tasks)
|
||
total_created += created
|
||
|
||
# 检查是否还有更多页面
|
||
if page >= files_response.totalPages - 1:
|
||
break
|
||
page += 1
|
||
|
||
return current_file_ids, total_created
|
||
|
||
async def _delete_orphaned_tasks(
|
||
self,
|
||
existing_dm_file_mapping: Dict[str, int],
|
||
current_file_ids: Set[str]
|
||
) -> int:
|
||
"""删除在DM中不存在的Label Studio任务"""
|
||
# 使用集合操作找出需要删除的文件ID
|
||
deleted_file_ids = set(existing_dm_file_mapping.keys()) - current_file_ids
|
||
|
||
if not deleted_file_ids:
|
||
logger.debug("No tasks to delete")
|
||
return 0
|
||
|
||
tasks_to_delete = [existing_dm_file_mapping[fid] for fid in deleted_file_ids]
|
||
logger.debug(f"Deleting {len(tasks_to_delete)} orphaned tasks")
|
||
|
||
delete_result = await self.ls_client.delete_tasks_batch(tasks_to_delete)
|
||
deleted_count = delete_result.get("successful", 0)
|
||
|
||
logger.debug(f"Successfully deleted {deleted_count} tasks")
|
||
return deleted_count
|
||
|
||
async def sync_dataset_files(
|
||
self,
|
||
mapping_id: str,
|
||
batch_size: int = 50
|
||
) -> SyncDatasetResponse:
|
||
"""
|
||
同步数据集文件到Label Studio (Legacy endpoint - 委托给sync_files)
|
||
|
||
Args:
|
||
mapping_id: 映射ID
|
||
batch_size: 批处理大小
|
||
|
||
Returns:
|
||
同步结果响应
|
||
"""
|
||
logger.debug(f"Start syncing dataset files by mapping: {mapping_id}")
|
||
|
||
# 获取映射关系
|
||
mapping = await self.mapping_service.get_mapping_by_uuid(mapping_id)
|
||
if not mapping:
|
||
logger.error(f"Dataset mapping not found: {mapping_id}")
|
||
return SyncDatasetResponse(
|
||
id="",
|
||
status="error",
|
||
synced_files=0,
|
||
total_files=0,
|
||
message=f"Dataset mapping not found: {mapping_id}"
|
||
)
|
||
|
||
try:
|
||
# 委托给sync_files执行实际同步
|
||
result = await self.sync_files(mapping, batch_size)
|
||
|
||
logger.info(f"Sync files completed: created={result['created']}, deleted={result['deleted']}, total={result['total']}")
|
||
|
||
return SyncDatasetResponse(
|
||
id=mapping.id,
|
||
status="success",
|
||
synced_files=result["created"],
|
||
total_files=result["total"],
|
||
message=f"Sync completed: created {result['created']} files, deleted {result['deleted']} tasks"
|
||
)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error while syncing dataset: {e}")
|
||
return SyncDatasetResponse(
|
||
id=mapping.id,
|
||
status="error",
|
||
synced_files=0,
|
||
total_files=0,
|
||
message=f"Sync failed: {str(e)}"
|
||
)
|
||
|
||
async def sync_dataset(
|
||
self,
|
||
mapping_id: str,
|
||
batch_size: int = 50,
|
||
file_priority: int = 0,
|
||
annotation_priority: int = 0
|
||
) -> SyncDatasetResponse:
|
||
"""
|
||
同步数据集文件和标注
|
||
|
||
Args:
|
||
mapping_id: 映射ID
|
||
batch_size: 批处理大小
|
||
file_priority: 文件同步优先级 (0: dataset优先, 1: annotation优先)
|
||
annotation_priority: 标注同步优先级 (0: dataset优先, 1: annotation优先)
|
||
|
||
Returns:
|
||
同步结果响应
|
||
"""
|
||
logger.info(f"Start syncing dataset by mapping: {mapping_id}")
|
||
|
||
# 检查映射是否存在
|
||
mapping = await self.mapping_service.get_mapping_by_uuid(mapping_id)
|
||
if not mapping:
|
||
logger.error(f"Dataset mapping not found: {mapping_id}")
|
||
return SyncDatasetResponse(
|
||
id="",
|
||
status="error",
|
||
synced_files=0,
|
||
total_files=0,
|
||
message=f"Dataset mapping not found: {mapping_id}"
|
||
)
|
||
|
||
try:
|
||
# 同步文件（不限制文件ID，完整同步映射对应数据集）
|
||
file_result = await self.sync_files(mapping, batch_size)
|
||
|
||
# TODO: 同步标注
|
||
# annotation_result = await self.sync_annotations(mapping, batch_size, annotation_priority)
|
||
|
||
logger.info(f"Sync completed: created={file_result['created']}, deleted={file_result['deleted']}, total={file_result['total']}")
|
||
|
||
return SyncDatasetResponse(
|
||
id=mapping.id,
|
||
status="success",
|
||
synced_files=file_result["created"],
|
||
total_files=file_result["total"],
|
||
message=f"Sync completed: created {file_result['created']} files, deleted {file_result['deleted']} tasks"
|
||
)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error while syncing dataset: {e}")
|
||
return SyncDatasetResponse(
|
||
id=mapping.id,
|
||
status="error",
|
||
synced_files=0,
|
||
total_files=0,
|
||
message=f"Sync failed: {str(e)}"
|
||
)
|
||
|
||
async def sync_files(
|
||
self,
|
||
mapping: DatasetMappingResponse,
|
||
batch_size: int,
|
||
allowed_file_ids: Optional[Set[str]] = None,
|
||
override_dataset_id: Optional[str] = None,
|
||
delete_orphans: bool = True,
|
||
) -> Dict[str, int]:
|
||
"""
|
||
同步DM和Label Studio之间的文件
|
||
|
||
Args:
|
||
mapping: 数据集映射信息
|
||
batch_size: 批处理大小
|
||
|
||
Returns:
|
||
同步统计信息: {"created": int, "deleted": int, "total": int}
|
||
"""
|
||
effective_dataset_id = override_dataset_id or mapping.dataset_id
|
||
|
||
logger.debug(
|
||
"Syncing files for dataset %s to project %s (mapping dataset_id=%s)",
|
||
effective_dataset_id,
|
||
mapping.labeling_project_id,
|
||
mapping.dataset_id,
|
||
)
|
||
|
||
# 获取DM数据集信息
|
||
dataset_info = await self.dm_client.get_dataset(effective_dataset_id)
|
||
if not dataset_info:
|
||
raise BusinessError(ErrorCodes.NOT_FOUND, data={"dataset_id": mapping.dataset_id})
|
||
|
||
total_files = dataset_info.fileCount
|
||
logger.debug(f"Total files in DM dataset: {total_files}")
|
||
|
||
# 获取Label Studio中已存在的文件映射
|
||
existing_dm_file_mapping = await self.get_existing_dm_file_mapping(mapping.labeling_project_id)
|
||
existing_file_ids = set(existing_dm_file_mapping.keys())
|
||
logger.debug(f"{len(existing_file_ids)} tasks already exist in Label Studio")
|
||
|
||
# 分页获取DM文件并创建新任务
|
||
current_file_ids, created_count = await self._fetch_dm_files_paginated(
|
||
effective_dataset_id,
|
||
batch_size,
|
||
existing_file_ids,
|
||
mapping.labeling_project_id,
|
||
allowed_file_ids=allowed_file_ids,
|
||
)
|
||
|
||
# 删除孤立任务：在多数据集、分批同步场景下可以选择关闭，
|
||
# 避免后一次同步把前一次其他数据集的任务当成“孤儿”删除。
|
||
if delete_orphans:
|
||
deleted_count = await self._delete_orphaned_tasks(
|
||
existing_dm_file_mapping,
|
||
current_file_ids,
|
||
)
|
||
else:
|
||
deleted_count = 0
|
||
|
||
logger.debug(f"File sync completed: total={total_files}, created={created_count}, deleted={deleted_count}")
|
||
|
||
return {
|
||
"created": created_count,
|
||
"deleted": deleted_count,
|
||
"total": total_files
|
||
}
|
||
|
||
async def sync_annotations(
|
||
self,
|
||
mapping: DatasetMappingResponse,
|
||
batch_size: int,
|
||
priority: int
|
||
) -> Dict[str, int]:
|
||
"""
|
||
同步DM和Label Studio之间的标注
|
||
|
||
Args:
|
||
mapping: 数据集映射信息
|
||
batch_size: 批处理大小
|
||
priority: 标注同步优先级 (0: dataset优先, 1: annotation优先)
|
||
|
||
Returns:
|
||
同步统计信息: {"synced_to_dm": int, "synced_to_ls": int}
|
||
"""
|
||
logger.info(f"Syncing annotations for dataset {mapping.dataset_id} (priority={priority})")
|
||
|
||
# TODO: 实现标注同步逻辑
|
||
# 1. 从DM获取标注结果
|
||
# 2. 从Label Studio获取标注结果
|
||
# 3. 根据优先级合并结果
|
||
# 4. 将差异写入DM和LS
|
||
|
||
logger.info("Annotation sync not yet implemented")
|
||
return {
|
||
"synced_to_dm": 0,
|
||
"synced_to_ls": 0
|
||
}
|
||
|
||
def _simplify_annotation_result(self, annotation: Dict[str, Any]) -> Tuple[List[Dict[str, Any]], str]:
|
||
"""
|
||
将Label Studio标注结果简化为指定格式
|
||
|
||
Args:
|
||
annotation: Label Studio原始标注数据
|
||
|
||
Returns:
|
||
Tuple of (简化后的标注结果列表, 标注更新时间ISO字符串)
|
||
"""
|
||
simplified = []
|
||
|
||
# 获取result字段（包含实际的标注数据）
|
||
results = annotation.get("result", [])
|
||
|
||
# 获取标注的更新时间，优先使用updated_at，否则使用created_at
|
||
updated_at = annotation.get("updated_at") or annotation.get("created_at", "")
|
||
|
||
for result_item in results:
|
||
result_type = result_item.get("type", "")
|
||
if isinstance(result_type, str):
|
||
result_type = result_type.strip().lower()
|
||
else:
|
||
result_type = str(result_type).strip().lower() if result_type is not None else ""
|
||
|
||
raw_value = result_item.get("value", {})
|
||
normalized_values: Dict[str, Any] = {}
|
||
if isinstance(raw_value, dict):
|
||
for value_key, value_content in raw_value.items():
|
||
key_str = str(value_key).strip().lower() if value_key is not None else ""
|
||
if key_str:
|
||
normalized_values[key_str] = value_content
|
||
|
||
if result_type:
|
||
if result_type in normalized_values:
|
||
normalized_values = {result_type: normalized_values[result_type]}
|
||
elif len(normalized_values) == 1:
|
||
only_value = next(iter(normalized_values.values()))
|
||
normalized_values = {result_type: only_value}
|
||
|
||
simplified_item = {
|
||
"from_name": result_item.get("from_name", ""),
|
||
"to_name": result_item.get("to_name", ""),
|
||
"type": result_type,
|
||
"values": normalized_values
|
||
}
|
||
simplified.append(simplified_item)
|
||
|
||
return simplified, updated_at
|
||
|
||
def _compare_timestamps(self, ts1: str, ts2: str) -> int:
|
||
"""
|
||
比较两个ISO格式时间戳
|
||
|
||
Args:
|
||
ts1: 第一个时间戳
|
||
ts2: 第二个时间戳
|
||
|
||
Returns:
|
||
1 如果 ts1 > ts2
|
||
-1 如果 ts1 < ts2
|
||
0 如果相等或无法比较
|
||
"""
|
||
try:
|
||
from dateutil import parser
|
||
from datetime import timezone
|
||
|
||
dt1 = parser.parse(ts1)
|
||
dt2 = parser.parse(ts2)
|
||
|
||
# Convert both to UTC timezone-aware if needed
|
||
if dt1.tzinfo is None:
|
||
dt1 = dt1.replace(tzinfo=timezone.utc)
|
||
if dt2.tzinfo is None:
|
||
dt2 = dt2.replace(tzinfo=timezone.utc)
|
||
|
||
if dt1 > dt2:
|
||
return 1
|
||
elif dt1 < dt2:
|
||
return -1
|
||
else:
|
||
return 0
|
||
except Exception as e:
|
||
logger.warning(f"Failed to compare timestamps {ts1} and {ts2}: {e}")
|
||
return 0
|
||
|
||
def _should_overwrite_dm(self, ls_updated_at: str, dm_tags_updated_at: Optional[str], overwrite: bool) -> bool:
|
||
"""
|
||
判断是否应该用Label Studio的标注覆盖DeepData的标注
|
||
|
||
Args:
|
||
ls_updated_at: Label Studio标注的更新时间
|
||
dm_tags_updated_at: DeepData中标注的更新时间（从tags_updated_at字段）
|
||
overwrite: 是否允许覆盖
|
||
|
||
Returns:
|
||
True 如果应该覆盖，False 如果不应该覆盖
|
||
"""
|
||
# 如果不允许覆盖，直接返回False
|
||
if not overwrite:
|
||
return False
|
||
|
||
# 如果DeepData没有标注时间戳，允许覆盖
|
||
if not dm_tags_updated_at:
|
||
return True
|
||
|
||
# 如果Label Studio的标注更新，允许覆盖
|
||
return self._compare_timestamps(ls_updated_at, dm_tags_updated_at) > 0
|
||
|
||
def _should_overwrite_ls(self, dm_tags_updated_at: Optional[str], ls_updated_at: str, overwrite_ls: bool) -> bool:
|
||
"""
|
||
判断是否应该用DeepData的标注覆盖Label Studio的标注
|
||
|
||
Args:
|
||
dm_tags_updated_at: DeepData中标注的更新时间（从tags_updated_at字段）
|
||
ls_updated_at: Label Studio标注的更新时间
|
||
overwrite_ls: 是否允许覆盖Label Studio
|
||
|
||
Returns:
|
||
True 如果应该覆盖，False 如果不应该覆盖
|
||
"""
|
||
# 如果不允许覆盖，直接返回False
|
||
if not overwrite_ls:
|
||
return False
|
||
|
||
# 如果DeepData没有标注时间戳，不应该覆盖Label Studio
|
||
if not dm_tags_updated_at:
|
||
return False
|
||
|
||
# 如果Label Studio没有标注，应该覆盖
|
||
if not ls_updated_at:
|
||
return True
|
||
|
||
# 如果DeepData的标注更新，允许覆盖
|
||
return self._compare_timestamps(dm_tags_updated_at, ls_updated_at) > 0
|
||
|
||
async def sync_annotations_from_ls_to_dm(
|
||
self,
|
||
mapping: DatasetMappingResponse,
|
||
batch_size: int = 50,
|
||
overwrite: bool = True,
|
||
sync_files_first: bool = True,
|
||
) -> SyncAnnotationsResponse:
|
||
"""
|
||
从Label Studio同步标注到数据集
|
||
|
||
Args:
|
||
mapping: 数据集映射信息
|
||
batch_size: 批处理大小
|
||
overwrite: 是否允许覆盖DeepData中的标注（基于时间戳比较）
|
||
|
||
Returns:
|
||
同步结果响应
|
||
"""
|
||
logger.info(f"Syncing annotations from LS to DM: dataset={mapping.dataset_id}, project={mapping.labeling_project_id}")
|
||
|
||
synced_count = 0
|
||
skipped_count = 0
|
||
failed_count = 0
|
||
conflicts_resolved = 0
|
||
|
||
try:
|
||
# 先做文件差异同步，保证 LS 任务集合与 DM 文件集合一致
|
||
if sync_files_first:
|
||
file_sync_result = await self.sync_files(mapping, batch_size)
|
||
logger.info(
|
||
"Pre-annotation file sync done (LS->DM): created=%s, deleted=%s, total=%s",
|
||
file_sync_result.get("created", 0),
|
||
file_sync_result.get("deleted", 0),
|
||
file_sync_result.get("total", 0),
|
||
)
|
||
|
||
# 获取Label Studio中的所有任务
|
||
ls_tasks_result = await self.ls_client.get_project_tasks(
|
||
mapping.labeling_project_id,
|
||
page=None
|
||
)
|
||
|
||
if not ls_tasks_result:
|
||
token_display = settings.label_studio_user_token[:10] + "..." if settings.label_studio_user_token else "None"
|
||
error_msg = f"Failed to fetch tasks from Label Studio project {mapping.labeling_project_id}. Please check:\n" \
|
||
f"1. Label Studio is running at {settings.label_studio_base_url}\n" \
|
||
f"2. Project ID {mapping.labeling_project_id} exists\n" \
|
||
f"3. API token is valid: {token_display}"
|
||
logger.error(error_msg)
|
||
return SyncAnnotationsResponse(
|
||
id=mapping.id,
|
||
status="error",
|
||
synced_to_dm=0,
|
||
synced_to_ls=0,
|
||
skipped=0,
|
||
failed=0,
|
||
conflicts_resolved=0,
|
||
message=f"Failed to connect to Label Studio at {settings.label_studio_base_url}"
|
||
)
|
||
|
||
all_tasks = ls_tasks_result.get("tasks", [])
|
||
logger.info(f"Found {len(all_tasks)} tasks in Label Studio project")
|
||
|
||
if len(all_tasks) == 0:
|
||
logger.warning(f"No tasks found in Label Studio project {mapping.labeling_project_id}")
|
||
return SyncAnnotationsResponse(
|
||
id=mapping.id,
|
||
status="success",
|
||
synced_to_dm=0,
|
||
synced_to_ls=0,
|
||
skipped=0,
|
||
failed=0,
|
||
conflicts_resolved=0,
|
||
message="No tasks found in Label Studio project"
|
||
)
|
||
|
||
# 批量处理任务
|
||
for i in range(0, len(all_tasks), batch_size):
|
||
batch_tasks = all_tasks[i:i + batch_size]
|
||
logger.info(f"Processing batch {i // batch_size + 1}, {len(batch_tasks)} tasks")
|
||
|
||
for task in batch_tasks:
|
||
task_id = task.get("id")
|
||
file_id = task.get("data", {}).get("file_id")
|
||
|
||
if not file_id:
|
||
logger.warning(f"Task {task_id} has no file_id, skipping")
|
||
skipped_count += 1
|
||
continue
|
||
|
||
# 获取任务的标注结果
|
||
annotations = await self.ls_client.get_task_annotations(task_id)
|
||
|
||
if not annotations:
|
||
logger.debug(f"No annotations for task {task_id}, skipping")
|
||
skipped_count += 1
|
||
continue
|
||
|
||
# 简化标注结果（取最新的标注）
|
||
latest_annotation = max(annotations, key=lambda a: a.get("updated_at") or a.get("created_at", ""))
|
||
simplified_annotations, ls_updated_at = self._simplify_annotation_result(latest_annotation)
|
||
|
||
if not simplified_annotations:
|
||
logger.debug(f"Task {task_id} has no valid annotation results")
|
||
skipped_count += 1
|
||
continue
|
||
|
||
# 更新数据库中的tags字段
|
||
try:
|
||
# 检查文件是否存在以及是否已有标注
|
||
result = await self.dm_client.db.execute(
|
||
select(DatasetFiles).where(
|
||
DatasetFiles.id == file_id,
|
||
DatasetFiles.dataset_id == mapping.dataset_id
|
||
)
|
||
)
|
||
file_record = result.scalar_one_or_none()
|
||
|
||
if not file_record:
|
||
logger.warning(f"File {file_id} not found in dataset {mapping.dataset_id}")
|
||
failed_count += 1
|
||
continue
|
||
|
||
# 检查是否应该覆盖DeepData的标注（使用文件级别的tags_updated_at）
|
||
dm_tags_updated_at: Optional[str] = None
|
||
if file_record.tags_updated_at: # type: ignore
|
||
dm_tags_updated_at = file_record.tags_updated_at.isoformat() # type: ignore
|
||
|
||
if not self._should_overwrite_dm(ls_updated_at, dm_tags_updated_at, overwrite):
|
||
logger.debug(
|
||
"File %s: DeepData has newer or equal annotations, skipping (overwrite=%s, ls_updated_at=%s, dm_tags_updated_at=%s)",
|
||
file_id,
|
||
overwrite,
|
||
ls_updated_at,
|
||
dm_tags_updated_at,
|
||
)
|
||
skipped_count += 1
|
||
continue
|
||
|
||
# 如果存在冲突（两边都有标注且时间戳不同），记录为冲突解决
|
||
if file_record.tags and ls_updated_at: # type: ignore
|
||
conflicts_resolved += 1
|
||
logger.debug(f"File {file_id}: Resolved conflict, Label Studio annotation is newer")
|
||
|
||
# 更新tags字段和tags_updated_at
|
||
from datetime import datetime
|
||
tags_updated_datetime = datetime.fromisoformat(ls_updated_at.replace('Z', '+00:00'))
|
||
|
||
await self.dm_client.db.execute(
|
||
update(DatasetFiles)
|
||
.where(DatasetFiles.id == file_id)
|
||
.values(
|
||
tags=simplified_annotations,
|
||
tags_updated_at=tags_updated_datetime.replace(tzinfo=None)
|
||
)
|
||
)
|
||
await self.dm_client.db.commit()
|
||
|
||
synced_count += 1
|
||
logger.debug(f"Synced annotations for file {file_id}: {len(simplified_annotations)} results")
|
||
|
||
except Exception as e:
|
||
logger.error(f"Failed to update annotations for file {file_id}: {e}")
|
||
failed_count += 1
|
||
await self.dm_client.db.rollback()
|
||
|
||
logger.info(f"Annotation sync completed: synced={synced_count}, skipped={skipped_count}, failed={failed_count}, conflicts_resolved={conflicts_resolved}")
|
||
|
||
status = "success" if failed_count == 0 else ("partial" if synced_count > 0 else "error")
|
||
|
||
return SyncAnnotationsResponse(
|
||
id=mapping.id,
|
||
status=status,
|
||
synced_to_dm=synced_count,
|
||
synced_to_ls=0,
|
||
skipped=skipped_count,
|
||
failed=failed_count,
|
||
conflicts_resolved=conflicts_resolved,
|
||
message=f"Synced {synced_count} annotations from Label Studio to dataset. Skipped: {skipped_count}, Failed: {failed_count}, Conflicts resolved: {conflicts_resolved}"
|
||
)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error while syncing annotations from LS to DM: {e}")
|
||
return SyncAnnotationsResponse(
|
||
id=mapping.id,
|
||
status="error",
|
||
synced_to_dm=synced_count,
|
||
synced_to_ls=0,
|
||
skipped=skipped_count,
|
||
failed=failed_count,
|
||
conflicts_resolved=conflicts_resolved,
|
||
message=f"Sync failed: {str(e)}"
|
||
)
|
||
|
||
async def sync_annotations_from_dm_to_ls(
|
||
self,
|
||
mapping: DatasetMappingResponse,
|
||
batch_size: int = 50,
|
||
overwrite_ls: bool = True,
|
||
sync_files_first: bool = True,
|
||
) -> SyncAnnotationsResponse:
|
||
"""
|
||
从DeepData数据集同步标注到Label Studio
|
||
|
||
Args:
|
||
mapping: 数据集映射信息
|
||
batch_size: 批处理大小
|
||
overwrite_ls: 是否允许覆盖Label Studio中的标注（基于时间戳比较）
|
||
|
||
Returns:
|
||
同步结果响应
|
||
"""
|
||
logger.info(f"Syncing annotations from DM to LS: dataset={mapping.dataset_id}, project={mapping.labeling_project_id}")
|
||
|
||
synced_count = 0
|
||
skipped_count = 0
|
||
failed_count = 0
|
||
conflicts_resolved = 0
|
||
|
||
try:
|
||
# 先做文件差异同步，保证 LS 任务集合与 DM 文件集合一致
|
||
if sync_files_first:
|
||
file_sync_result = await self.sync_files(mapping, batch_size)
|
||
logger.info(
|
||
"Pre-annotation file sync done (DM->LS): created=%s, deleted=%s, total=%s",
|
||
file_sync_result.get("created", 0),
|
||
file_sync_result.get("deleted", 0),
|
||
file_sync_result.get("total", 0),
|
||
)
|
||
|
||
# 获取Label Studio中的文件ID到任务ID的映射
|
||
dm_file_to_task_mapping = await self.get_existing_dm_file_mapping(mapping.labeling_project_id)
|
||
|
||
if not dm_file_to_task_mapping:
|
||
logger.warning(f"No task mapping found for project {mapping.labeling_project_id}")
|
||
return SyncAnnotationsResponse(
|
||
id=mapping.id,
|
||
status="error",
|
||
synced_to_dm=0,
|
||
synced_to_ls=0,
|
||
skipped=0,
|
||
failed=0,
|
||
conflicts_resolved=0,
|
||
message="No tasks found in Label Studio project"
|
||
)
|
||
|
||
logger.info(f"Found {len(dm_file_to_task_mapping)} task mappings")
|
||
|
||
# 分页获取DeepData中的文件
|
||
page = 0
|
||
processed_count = 0
|
||
|
||
while True:
|
||
files_response = await self.dm_client.get_dataset_files(
|
||
mapping.dataset_id,
|
||
page=page,
|
||
size=batch_size,
|
||
)
|
||
|
||
if not files_response or not files_response.content:
|
||
logger.info(f"No more files on page {page + 1}")
|
||
break
|
||
|
||
logger.info(f"Processing page {page + 1}, {len(files_response.content)} files")
|
||
|
||
for file_info in files_response.content:
|
||
file_id = str(file_info.id)
|
||
processed_count += 1
|
||
|
||
# 检查该文件是否在Label Studio中有对应的任务
|
||
task_id = dm_file_to_task_mapping.get(file_id)
|
||
if not task_id:
|
||
logger.debug(f"File {file_id} has no corresponding task in Label Studio, skipping")
|
||
skipped_count += 1
|
||
continue
|
||
|
||
# 获取DeepData中的标注
|
||
dm_tags: List[Dict[str, Any]] = file_info.tags if file_info.tags else [] # type: ignore
|
||
|
||
if not dm_tags:
|
||
logger.debug(f"File {file_id} has no annotations in DeepData, skipping")
|
||
skipped_count += 1
|
||
continue
|
||
|
||
# 获取DeepData中标注的更新时间
|
||
dm_tags_updated_at: Optional[str] = None
|
||
if file_info.tags_updated_at: # type: ignore
|
||
dm_tags_updated_at = file_info.tags_updated_at.isoformat() # type: ignore
|
||
|
||
try:
|
||
# 获取Label Studio中该任务的现有标注
|
||
ls_annotations = await self.ls_client.get_task_annotations(task_id)
|
||
|
||
# 获取Label Studio标注的更新时间
|
||
ls_updated_at = ""
|
||
if ls_annotations:
|
||
latest_ls_annotation = max(
|
||
ls_annotations,
|
||
key=lambda a: a.get("updated_at") or a.get("created_at", "")
|
||
)
|
||
ls_updated_at = latest_ls_annotation.get("updated_at") or latest_ls_annotation.get("created_at", "")
|
||
|
||
# 检查是否应该覆盖Label Studio的标注
|
||
should_overwrite_ls = self._should_overwrite_ls(dm_tags_updated_at, ls_updated_at, overwrite_ls)
|
||
|
||
if not should_overwrite_ls:
|
||
forced_on_equal_ts = False
|
||
if (
|
||
overwrite_ls
|
||
and dm_tags_updated_at
|
||
and ls_updated_at
|
||
and self._compare_timestamps(dm_tags_updated_at, ls_updated_at) == 0
|
||
and ls_annotations
|
||
):
|
||
latest_ls_result_payload = latest_ls_annotation.get("result", []) if latest_ls_annotation else []
|
||
if not self._ls_result_payload_equal(ls_result, latest_ls_result_payload):
|
||
forced_on_equal_ts = True
|
||
logger.info(
|
||
"Task %s: timestamps are equal but payload differs, force syncing DM -> LS",
|
||
task_id,
|
||
)
|
||
|
||
if not forced_on_equal_ts:
|
||
logger.debug(
|
||
"Task %s: Label Studio has newer or equal annotations, skipping (overwrite_ls=%s, dm_tags_updated_at=%s, ls_updated_at=%s)",
|
||
task_id,
|
||
overwrite_ls,
|
||
dm_tags_updated_at,
|
||
ls_updated_at,
|
||
)
|
||
skipped_count += 1
|
||
continue
|
||
|
||
# 如果存在冲突，记录为冲突解决
|
||
if ls_annotations and dm_tags:
|
||
conflicts_resolved += 1
|
||
logger.debug(f"Task {task_id}: Resolved conflict, DeepData annotation is newer")
|
||
|
||
# 将DeepData的标注转换为Label Studio格式（兼容 value/values 与大小写差异）
|
||
ls_result = []
|
||
for tag in dm_tags:
|
||
if not isinstance(tag, dict):
|
||
continue
|
||
|
||
from_name = tag.get("from_name") or tag.get("fromName") or ""
|
||
to_name = tag.get("to_name") or tag.get("toName") or ""
|
||
if not isinstance(from_name, str):
|
||
from_name = str(from_name)
|
||
if not isinstance(to_name, str):
|
||
to_name = str(to_name)
|
||
from_name = from_name.strip()
|
||
to_name = to_name.strip()
|
||
|
||
if not from_name or not to_name:
|
||
logger.warning(
|
||
"Skip invalid DM tag for LS sync due to missing from_name/to_name: %s",
|
||
tag,
|
||
)
|
||
continue
|
||
|
||
raw_type = tag.get("type", "")
|
||
if not isinstance(raw_type, str):
|
||
raw_type = str(raw_type) if raw_type is not None else ""
|
||
normalized_type = raw_type.strip().lower()
|
||
|
||
raw_values = tag.get("values")
|
||
if not isinstance(raw_values, dict):
|
||
raw_values = tag.get("value")
|
||
|
||
normalized_value: Dict[str, Any] = {}
|
||
|
||
if isinstance(raw_values, dict):
|
||
for value_key, value_content in raw_values.items():
|
||
key_str = str(value_key).strip().lower() if value_key is not None else ""
|
||
if key_str:
|
||
normalized_value[key_str] = value_content
|
||
|
||
if not normalized_type and len(normalized_value) == 1:
|
||
normalized_type = next(iter(normalized_value.keys()))
|
||
|
||
if normalized_type:
|
||
if normalized_type in normalized_value:
|
||
normalized_value = {normalized_type: normalized_value[normalized_type]}
|
||
elif len(normalized_value) == 1:
|
||
only_value = next(iter(normalized_value.values()))
|
||
normalized_value = {normalized_type: only_value}
|
||
|
||
if not normalized_type or not normalized_value:
|
||
logger.warning(
|
||
"Skip invalid DM tag for LS sync: from_name=%s, to_name=%s, type=%s, values=%s",
|
||
from_name,
|
||
to_name,
|
||
raw_type,
|
||
raw_values,
|
||
)
|
||
continue
|
||
|
||
ls_result_item = {
|
||
"from_name": from_name,
|
||
"to_name": to_name,
|
||
"type": normalized_type,
|
||
"value": normalized_value,
|
||
}
|
||
ls_result.append(ls_result_item)
|
||
|
||
if not ls_result:
|
||
logger.warning(f"Task {task_id}: no valid DM annotations after normalization, skipping")
|
||
skipped_count += 1
|
||
continue
|
||
|
||
# 如果Label Studio已有标注，更新它；否则创建新标注
|
||
if ls_annotations:
|
||
# 更新最新的标注
|
||
latest_annotation_id = latest_ls_annotation.get("id")
|
||
if not latest_annotation_id:
|
||
logger.error(f"Task {task_id} has no annotation ID")
|
||
failed_count += 1
|
||
continue
|
||
|
||
update_result = await self.ls_client.update_annotation(
|
||
int(latest_annotation_id),
|
||
ls_result
|
||
)
|
||
if update_result:
|
||
synced_count += 1
|
||
logger.debug(f"Updated annotation for task {task_id}")
|
||
else:
|
||
failed_count += 1
|
||
logger.error(f"Failed to update annotation for task {task_id}")
|
||
else:
|
||
# 创建新标注
|
||
create_result = await self.ls_client.create_annotation(
|
||
task_id,
|
||
ls_result
|
||
)
|
||
if create_result:
|
||
synced_count += 1
|
||
logger.debug(f"Created annotation for task {task_id}")
|
||
else:
|
||
failed_count += 1
|
||
logger.error(f"Failed to create annotation for task {task_id}")
|
||
|
||
except Exception as e:
|
||
logger.error(f"Failed to sync annotations for file {file_id} (task {task_id}): {e}")
|
||
failed_count += 1
|
||
|
||
# 检查是否还有更多页面
|
||
if page >= files_response.totalPages - 1:
|
||
break
|
||
page += 1
|
||
|
||
logger.info(f"Annotation sync completed: synced={synced_count}, skipped={skipped_count}, failed={failed_count}, conflicts_resolved={conflicts_resolved}")
|
||
|
||
status = "success" if failed_count == 0 else ("partial" if synced_count > 0 else "error")
|
||
|
||
return SyncAnnotationsResponse(
|
||
id=mapping.id,
|
||
status=status,
|
||
synced_to_dm=0,
|
||
synced_to_ls=synced_count,
|
||
skipped=skipped_count,
|
||
failed=failed_count,
|
||
conflicts_resolved=conflicts_resolved,
|
||
message=f"Synced {synced_count} annotations from DeepData to Label Studio. Skipped: {skipped_count}, Failed: {failed_count}, Conflicts resolved: {conflicts_resolved}"
|
||
)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error while syncing annotations from DM to LS: {e}")
|
||
return SyncAnnotationsResponse(
|
||
id=mapping.id,
|
||
status="error",
|
||
synced_to_dm=0,
|
||
synced_to_ls=synced_count,
|
||
skipped=skipped_count,
|
||
failed=failed_count,
|
||
conflicts_resolved=conflicts_resolved,
|
||
message=f"Sync failed: {str(e)}"
|
||
)
|
||
|
||
async def sync_annotations_bidirectional(
|
||
self,
|
||
mapping: DatasetMappingResponse,
|
||
batch_size: int = 50,
|
||
overwrite: bool = True,
|
||
overwrite_ls: bool = True,
|
||
sync_files_first: bool = True,
|
||
) -> SyncAnnotationsResponse:
|
||
"""
|
||
双向同步标注结果
|
||
|
||
Args:
|
||
mapping: 数据集映射信息
|
||
batch_size: 批处理大小
|
||
overwrite: 是否允许覆盖DeepData中的标注
|
||
overwrite_ls: 是否允许覆盖Label Studio中的标注
|
||
|
||
Returns:
|
||
同步结果响应
|
||
"""
|
||
logger.info(f"Bidirectional annotation sync: dataset={mapping.dataset_id}, project={mapping.labeling_project_id}")
|
||
|
||
try:
|
||
# 先做文件差异同步，保证双向标注同步处理的是同一批文件
|
||
if sync_files_first:
|
||
file_sync_result = await self.sync_files(mapping, batch_size)
|
||
logger.info(
|
||
"Pre-annotation file sync done (bidirectional): created=%s, deleted=%s, total=%s",
|
||
file_sync_result.get("created", 0),
|
||
file_sync_result.get("deleted", 0),
|
||
file_sync_result.get("total", 0),
|
||
)
|
||
|
||
# 先从Label Studio同步到DeepData
|
||
ls_to_dm_result = await self.sync_annotations_from_ls_to_dm(
|
||
mapping,
|
||
batch_size,
|
||
overwrite,
|
||
sync_files_first=False,
|
||
)
|
||
|
||
# 再从DeepData同步到Label Studio
|
||
dm_to_ls_result = await self.sync_annotations_from_dm_to_ls(
|
||
mapping,
|
||
batch_size,
|
||
overwrite_ls,
|
||
sync_files_first=False,
|
||
)
|
||
|
||
# 合并结果
|
||
total_synced_to_dm = ls_to_dm_result.synced_to_dm
|
||
total_synced_to_ls = dm_to_ls_result.synced_to_ls
|
||
total_skipped = ls_to_dm_result.skipped + dm_to_ls_result.skipped
|
||
total_failed = ls_to_dm_result.failed + dm_to_ls_result.failed
|
||
total_conflicts = ls_to_dm_result.conflicts_resolved + dm_to_ls_result.conflicts_resolved
|
||
|
||
# 判断状态
|
||
if ls_to_dm_result.status == "error" and dm_to_ls_result.status == "error":
|
||
status = "error"
|
||
elif total_failed > 0:
|
||
status = "partial"
|
||
else:
|
||
status = "success"
|
||
|
||
logger.info(f"Bidirectional sync completed: to_dm={total_synced_to_dm}, to_ls={total_synced_to_ls}, skipped={total_skipped}, failed={total_failed}, conflicts={total_conflicts}")
|
||
|
||
return SyncAnnotationsResponse(
|
||
id=mapping.id,
|
||
status=status,
|
||
synced_to_dm=total_synced_to_dm,
|
||
synced_to_ls=total_synced_to_ls,
|
||
skipped=total_skipped,
|
||
failed=total_failed,
|
||
conflicts_resolved=total_conflicts,
|
||
message=f"Bidirectional sync completed: {total_synced_to_dm} to DeepData, {total_synced_to_ls} to Label Studio. Skipped: {total_skipped}, Failed: {total_failed}, Conflicts resolved: {total_conflicts}"
|
||
)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error during bidirectional sync: {e}")
|
||
return SyncAnnotationsResponse(
|
||
id=mapping.id,
|
||
status="error",
|
||
synced_to_dm=0,
|
||
synced_to_ls=0,
|
||
skipped=0,
|
||
failed=0,
|
||
conflicts_resolved=0,
|
||
message=f"Bidirectional sync failed: {str(e)}"
|
||
)
|
||
|
||
async def get_sync_status(
|
||
self,
|
||
dataset_id: str
|
||
) -> Optional[Dict[str, Any]]:
|
||
"""获取同步状态"""
|
||
mapping = await self.mapping_service.get_mapping_by_source_uuid(dataset_id)
|
||
if not mapping:
|
||
return None
|
||
|
||
# 获取DM数据集信息
|
||
dataset_info = await self.dm_client.get_dataset(dataset_id)
|
||
|
||
# 获取Label Studio项目任务数量
|
||
tasks_info = await self.ls_client.get_project_tasks(mapping.labeling_project_id)
|
||
|
||
return {
|
||
"id": mapping.id,
|
||
"dataset_id": dataset_id,
|
||
"labeling_project_id": mapping.labeling_project_id,
|
||
"dm_total_files": dataset_info.fileCount if dataset_info else 0,
|
||
"ls_total_tasks": tasks_info.get("count", 0) if tasks_info else 0,
|
||
"sync_ratio": (
|
||
tasks_info.get("count", 0) / dataset_info.fileCount
|
||
if dataset_info and dataset_info.fileCount > 0 and tasks_info else 0
|
||
)
|
||
} |