deepdata: core product code (backend, frontend, runtime, deepflow module)
This commit is contained in:
figmar
@@ -0,0 +1,203 @@
|
||||
"""Service layer for Auto Annotation tasks"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Optional
|
||||
from datetime import datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models.annotation_management import AutoAnnotationTask
|
||||
from app.db.models.dataset_management import Dataset, DatasetFiles
|
||||
|
||||
from ..schema.auto import (
|
||||
CreateAutoAnnotationTaskRequest,
|
||||
AutoAnnotationTaskResponse,
|
||||
)
|
||||
|
||||
|
||||
class AutoAnnotationTaskService:
|
||||
"""自动标注任务服务（仅管理任务元数据，真正执行由 runtime 负责）"""
|
||||
|
||||
async def create_task(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
request: CreateAutoAnnotationTaskRequest,
|
||||
dataset_name: Optional[str] = None,
|
||||
total_images: int = 0,
|
||||
) -> AutoAnnotationTaskResponse:
|
||||
"""创建自动标注任务，初始状态为 pending。
|
||||
|
||||
这里仅插入任务记录，不负责真正执行 YOLO 推理，
|
||||
后续可以由调度器/worker 读取该表并更新进度。
|
||||
"""
|
||||
|
||||
now = datetime.now()
|
||||
|
||||
task = AutoAnnotationTask(
|
||||
id=str(uuid4()),
|
||||
name=request.name,
|
||||
dataset_id=request.dataset_id,
|
||||
dataset_name=dataset_name,
|
||||
config=request.config.model_dump(by_alias=True),
|
||||
file_ids=request.file_ids, # 存储用户选择的文件ID列表
|
||||
status="pending",
|
||||
progress=0,
|
||||
total_images=total_images,
|
||||
processed_images=0,
|
||||
detected_objects=0,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
db.add(task)
|
||||
await db.commit()
|
||||
await db.refresh(task)
|
||||
|
||||
# 创建后附带 sourceDatasets 信息（通常只有一个原始数据集）
|
||||
resp = AutoAnnotationTaskResponse.model_validate(task)
|
||||
try:
|
||||
resp.source_datasets = await self._compute_source_datasets(db, task)
|
||||
except Exception:
|
||||
resp.source_datasets = [dataset_name] if dataset_name else [request.dataset_id]
|
||||
return resp
|
||||
|
||||
async def list_tasks(self, db: AsyncSession) -> List[AutoAnnotationTaskResponse]:
|
||||
"""获取未软删除的自动标注任务列表，按创建时间倒序。"""
|
||||
|
||||
result = await db.execute(
|
||||
select(AutoAnnotationTask)
|
||||
.where(AutoAnnotationTask.deleted_at.is_(None))
|
||||
.order_by(AutoAnnotationTask.created_at.desc())
|
||||
)
|
||||
tasks: List[AutoAnnotationTask] = list(result.scalars().all())
|
||||
|
||||
responses: List[AutoAnnotationTaskResponse] = []
|
||||
for task in tasks:
|
||||
resp = AutoAnnotationTaskResponse.model_validate(task)
|
||||
try:
|
||||
resp.source_datasets = await self._compute_source_datasets(db, task)
|
||||
except Exception:
|
||||
# 出错时降级为单个 datasetName/datasetId
|
||||
fallback_name = getattr(task, "dataset_name", None)
|
||||
fallback_id = getattr(task, "dataset_id", "")
|
||||
resp.source_datasets = [fallback_name] if fallback_name else [fallback_id]
|
||||
responses.append(resp)
|
||||
|
||||
return responses
|
||||
|
||||
async def get_task(self, db: AsyncSession, task_id: str) -> Optional[AutoAnnotationTaskResponse]:
|
||||
result = await db.execute(
|
||||
select(AutoAnnotationTask).where(
|
||||
AutoAnnotationTask.id == task_id,
|
||||
AutoAnnotationTask.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
task = result.scalar_one_or_none()
|
||||
if not task:
|
||||
return None
|
||||
|
||||
resp = AutoAnnotationTaskResponse.model_validate(task)
|
||||
try:
|
||||
resp.source_datasets = await self._compute_source_datasets(db, task)
|
||||
except Exception:
|
||||
fallback_name = getattr(task, "dataset_name", None)
|
||||
fallback_id = getattr(task, "dataset_id", "")
|
||||
resp.source_datasets = [fallback_name] if fallback_name else [fallback_id]
|
||||
return resp
|
||||
|
||||
async def _compute_source_datasets(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
task: AutoAnnotationTask,
|
||||
) -> List[str]:
|
||||
"""根据任务的 file_ids 推断实际涉及到的所有数据集名称。
|
||||
|
||||
- 如果存在 file_ids，则通过 t_dm_dataset_files 反查 dataset_id，再关联 t_dm_datasets 获取名称；
|
||||
- 如果没有 file_ids，则退回到任务上冗余的 dataset_name/dataset_id。
|
||||
"""
|
||||
|
||||
file_ids = task.file_ids or []
|
||||
if file_ids:
|
||||
stmt = (
|
||||
select(Dataset.name)
|
||||
.join(DatasetFiles, Dataset.id == DatasetFiles.dataset_id)
|
||||
.where(DatasetFiles.id.in_(file_ids))
|
||||
.distinct()
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
names = [row[0] for row in result.fetchall() if row[0]]
|
||||
if names:
|
||||
return names
|
||||
|
||||
# 回退：只显示一个数据集
|
||||
if task.dataset_name:
|
||||
return [task.dataset_name]
|
||||
if task.dataset_id:
|
||||
return [task.dataset_id]
|
||||
return []
|
||||
|
||||
async def soft_delete_task(self, db: AsyncSession, task_id: str) -> bool:
|
||||
result = await db.execute(
|
||||
select(AutoAnnotationTask).where(
|
||||
AutoAnnotationTask.id == task_id,
|
||||
AutoAnnotationTask.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
task = result.scalar_one_or_none()
|
||||
if not task:
|
||||
return False
|
||||
|
||||
task.deleted_at = datetime.now()
|
||||
await db.commit()
|
||||
return True
|
||||
|
||||
async def update_task_files(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
task_id: str,
|
||||
*,
|
||||
dataset_id: str,
|
||||
file_ids: List[str],
|
||||
) -> Optional[AutoAnnotationTaskResponse]:
|
||||
"""更新自动标注任务关联的数据集与文件列表。
|
||||
|
||||
- 覆盖任务的 dataset_id 与 file_ids；
|
||||
- 将任务重置为 pending，供 worker 重新调度；
|
||||
- 保留已有的 output_path 与统计信息，确保同一任务始终复用同一个输出数据集，
|
||||
worker 内部会根据 output_path 中已有的 images 仅对新增文件执行自动标注。
|
||||
"""
|
||||
|
||||
result = await db.execute(
|
||||
select(AutoAnnotationTask).where(
|
||||
AutoAnnotationTask.id == task_id,
|
||||
AutoAnnotationTask.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
task = result.scalar_one_or_none()
|
||||
if not task:
|
||||
return None
|
||||
|
||||
now = datetime.now()
|
||||
|
||||
task.dataset_id = dataset_id
|
||||
task.file_ids = file_ids or []
|
||||
|
||||
# 将任务标记为待处理，让 worker 在同一个输出目录下仅对新增文件执行自动标注
|
||||
task.status = "pending"
|
||||
task.progress = 0
|
||||
task.updated_at = now
|
||||
|
||||
db.add(task)
|
||||
await db.commit()
|
||||
await db.refresh(task)
|
||||
|
||||
resp = AutoAnnotationTaskResponse.model_validate(task)
|
||||
try:
|
||||
resp.source_datasets = await self._compute_source_datasets(db, task)
|
||||
except Exception:
|
||||
fallback_name = getattr(task, "dataset_name", None)
|
||||
fallback_id = getattr(task, "dataset_id", "")
|
||||
resp.source_datasets = [fallback_name] if fallback_name else [fallback_id]
|
||||
return resp
|
||||
@@ -0,0 +1,229 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.logging import get_logger
|
||||
from app.db.models import DatasetFiles
|
||||
|
||||
from ..client import LabelStudioClient
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class LSAnnotationSyncService:
|
||||
"""将 Label Studio 项目中的标注结果同步回 DM 数据库。
|
||||
|
||||
约定：
|
||||
- Label Studio task.data 中包含由 SyncService 写入的 DM 元信息：
|
||||
{"file_id": "...", "dataset_id": "...", "file_path": "...", "original_name": "..."}
|
||||
- 对于每个 task，我们会：
|
||||
* 读取该 task 的 annotations 列表；
|
||||
* 结合 task.predictions 一并写入 t_dm_dataset_files.annotation；
|
||||
* 从 annotations/predictions 的 result 字段中提取标签，写入 t_dm_dataset_files.tags；
|
||||
* 更新时间戳 tags_updated_at。
|
||||
"""
|
||||
|
||||
def __init__(self, db: AsyncSession, ls_client: LabelStudioClient) -> None:
|
||||
self.db = db
|
||||
self.ls_client = ls_client
|
||||
|
||||
async def sync_project_annotations_to_dm(self, project_id: str) -> int:
|
||||
"""从指定 Label Studio 项目中同步所有任务的标注到 DM 文件表。
|
||||
|
||||
Args:
|
||||
project_id: Label Studio 项目 ID（字符串形式）。
|
||||
|
||||
Returns:
|
||||
成功更新的文件数量。
|
||||
"""
|
||||
try:
|
||||
page_size = getattr(settings, "ls_task_page_size", 1000)
|
||||
result = await self.ls_client.get_project_tasks(
|
||||
project_id=project_id,
|
||||
page=None,
|
||||
page_size=page_size,
|
||||
)
|
||||
except Exception as e: # pragma: no cover - 防御性日志
|
||||
logger.error("Failed to fetch tasks from Label Studio project %s: %s", project_id, e)
|
||||
return 0
|
||||
|
||||
if not result:
|
||||
logger.warning("No tasks returned for project %s when syncing annotations", project_id)
|
||||
return 0
|
||||
|
||||
tasks: List[Dict[str, Any]] = result.get("tasks", []) # type: ignore[assignment]
|
||||
if not tasks:
|
||||
logger.info("Project %s has no tasks to sync", project_id)
|
||||
return 0
|
||||
|
||||
updated_files = 0
|
||||
|
||||
for task in tasks:
|
||||
data: Dict[str, Any] = task.get("data") or {}
|
||||
raw_file_id = data.get("file_id")
|
||||
if not raw_file_id:
|
||||
# 旧任务可能没有 DM file_id，跳过但记录日志
|
||||
logger.debug(
|
||||
"Skip LS task %s because data.file_id is missing",
|
||||
task.get("id"),
|
||||
)
|
||||
continue
|
||||
|
||||
file_id = str(raw_file_id)
|
||||
|
||||
# 获取该任务的 annotations 列表
|
||||
annotations: Optional[List[Dict[str, Any]]] = None
|
||||
try:
|
||||
annotations = await self.ls_client.get_task_annotations(int(task["id"])) # type: ignore[arg-type]
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(
|
||||
"Failed to fetch annotations for LS task %s (file_id=%s): %s",
|
||||
task.get("id"),
|
||||
file_id,
|
||||
e,
|
||||
)
|
||||
|
||||
predictions: List[Dict[str, Any]] = task.get("predictions") or []
|
||||
|
||||
annotation_payload: Dict[str, Any] = {
|
||||
"task": {
|
||||
"id": task.get("id"),
|
||||
"project": task.get("project"),
|
||||
"data": data,
|
||||
},
|
||||
"annotations": annotations or [],
|
||||
"predictions": predictions,
|
||||
}
|
||||
|
||||
tags = self._extract_tags_from_results(
|
||||
annotations or [],
|
||||
predictions,
|
||||
)
|
||||
|
||||
success = await self._update_dataset_file(file_id, tags, annotation_payload)
|
||||
if success:
|
||||
updated_files += 1
|
||||
|
||||
logger.info(
|
||||
"Synced annotations from LS project %s to DM, updated %d files",
|
||||
project_id,
|
||||
updated_files,
|
||||
)
|
||||
return updated_files
|
||||
|
||||
async def _update_dataset_file(
|
||||
self,
|
||||
file_id: str,
|
||||
tags: List[Dict[str, Any]],
|
||||
annotation: Dict[str, Any],
|
||||
) -> bool:
|
||||
"""将提取出的 tags 与 annotation 写回 t_dm_dataset_files。
|
||||
|
||||
直接更新 DatasetFiles.tags / tags_updated_at / annotation 字段，
|
||||
不走模板驱动的转换逻辑（模板转换在用户主动批量更新时使用）。
|
||||
"""
|
||||
try:
|
||||
result = await self.db.execute(
|
||||
select(DatasetFiles).where(DatasetFiles.id == file_id)
|
||||
)
|
||||
record = result.scalar_one_or_none()
|
||||
|
||||
if not record:
|
||||
logger.warning("Dataset file not found when syncing LS annotations: %s", file_id)
|
||||
return False
|
||||
|
||||
update_time = datetime.utcnow()
|
||||
record.tags = tags # type: ignore[assignment]
|
||||
record.tags_updated_at = update_time # type: ignore[assignment]
|
||||
# annotation 可能较大，但为 JSONB，直接整体覆盖
|
||||
setattr(record, "annotation", annotation)
|
||||
|
||||
await self.db.commit()
|
||||
await self.db.refresh(record)
|
||||
|
||||
logger.debug(
|
||||
"Updated DM file %s with %d tags from LS annotations",
|
||||
file_id,
|
||||
len(tags),
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error("Failed to update DM file %s from LS annotations: %s", file_id, e)
|
||||
try:
|
||||
await self.db.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _extract_tags_from_results(
|
||||
annotations: List[Dict[str, Any]],
|
||||
predictions: List[Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""从 annotations 和 predictions 的 result 字段中提取通用标签结构。
|
||||
|
||||
输出格式与 DM 前端当前解析逻辑兼容：
|
||||
- 每个元素包含: id/type/from_name/values；
|
||||
- 其中 values 是一个字典，键通常与 type 一致，例如 "rectanglelabels"；
|
||||
- 前端通过 tag.values[tag.type] 提取字符串标签集合。
|
||||
"""
|
||||
|
||||
def iter_results() -> List[Dict[str, Any]]:
|
||||
res: List[Dict[str, Any]] = []
|
||||
for ann in annotations or []:
|
||||
for item in ann.get("result", []) or []:
|
||||
if isinstance(item, dict):
|
||||
res.append(item)
|
||||
for pred in predictions or []:
|
||||
for item in pred.get("result", []) or []:
|
||||
if isinstance(item, dict):
|
||||
res.append(item)
|
||||
return res
|
||||
|
||||
results = iter_results()
|
||||
if not results:
|
||||
return []
|
||||
|
||||
normalized: List[Dict[str, Any]] = []
|
||||
|
||||
for r in results:
|
||||
r_type = r.get("type")
|
||||
if isinstance(r_type, str):
|
||||
r_type = r_type.strip().lower()
|
||||
|
||||
from_name = r.get("from_name") or r.get("fromName")
|
||||
to_name = r.get("to_name") or r.get("toName")
|
||||
value_obj = r.get("value") or {}
|
||||
if not isinstance(value_obj, dict):
|
||||
continue
|
||||
|
||||
# 将 Label Studio 的 value 映射为 values，方便前端统一解析
|
||||
values: Dict[str, Any] = {}
|
||||
for key, v in value_obj.items():
|
||||
normalized_key = str(key).strip().lower() if key is not None else ""
|
||||
if normalized_key:
|
||||
values[normalized_key] = v
|
||||
|
||||
if isinstance(r_type, str) and r_type:
|
||||
if r_type in values:
|
||||
values = {r_type: values[r_type]}
|
||||
elif len(values) == 1:
|
||||
only_value = next(iter(values.values()))
|
||||
values = {r_type: only_value}
|
||||
|
||||
tag = {
|
||||
"id": r.get("id"),
|
||||
"type": r_type,
|
||||
"from_name": from_name,
|
||||
"to_name": to_name,
|
||||
"values": values,
|
||||
}
|
||||
normalized.append(tag)
|
||||
|
||||
return normalized
|
||||
@@ -0,0 +1,484 @@
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.future import select
|
||||
from sqlalchemy import update, func
|
||||
from sqlalchemy.orm import aliased
|
||||
from typing import Optional, List, Tuple
|
||||
from datetime import datetime
|
||||
import uuid
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.db.models import LabelingProject, AnnotationTemplate
|
||||
from app.db.models.dataset_management import Dataset
|
||||
from app.module.annotation.schema import (
|
||||
DatasetMappingCreateRequest,
|
||||
DatasetMappingUpdateRequest,
|
||||
DatasetMappingResponse,
|
||||
AnnotationTemplateResponse
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
class DatasetMappingService:
|
||||
"""数据集映射服务"""
|
||||
|
||||
def __init__(self, db: AsyncSession):
|
||||
self.db = db
|
||||
|
||||
def _build_query_with_dataset_name(self):
|
||||
"""Build base query with dataset name joined"""
|
||||
return select(
|
||||
LabelingProject,
|
||||
Dataset.name.label('dataset_name')
|
||||
).outerjoin(
|
||||
Dataset,
|
||||
LabelingProject.dataset_id == Dataset.id
|
||||
)
|
||||
|
||||
async def _to_response_from_row(
|
||||
self,
|
||||
row,
|
||||
include_template: bool = False
|
||||
) -> DatasetMappingResponse:
|
||||
"""
|
||||
Convert query row (mapping + dataset_name) to response
|
||||
|
||||
Args:
|
||||
row: Query result row containing (LabelingProject, dataset_name)
|
||||
include_template: If True, fetch and include full template details
|
||||
"""
|
||||
mapping = row[0] # LabelingProject object
|
||||
dataset_name = row[1] # dataset_name from join
|
||||
|
||||
# Get template_id from mapping
|
||||
template_id = getattr(mapping, 'template_id', None)
|
||||
|
||||
# Optionally fetch full template details
|
||||
template_response = None
|
||||
if include_template and template_id:
|
||||
from ..service.template import AnnotationTemplateService
|
||||
template_service = AnnotationTemplateService()
|
||||
template_response = await template_service.get_template(self.db, template_id)
|
||||
logger.debug(f"Included template details for template_id: {template_id}")
|
||||
|
||||
response_data = {
|
||||
"id": mapping.id,
|
||||
"dataset_id": mapping.dataset_id,
|
||||
"dataset_name": dataset_name,
|
||||
"labeling_project_id": mapping.labeling_project_id,
|
||||
"name": mapping.name,
|
||||
"description": getattr(mapping, 'description', None),
|
||||
"template_id": template_id,
|
||||
"template": template_response,
|
||||
"created_at": mapping.created_at,
|
||||
"updated_at": mapping.updated_at,
|
||||
"deleted_at": mapping.deleted_at,
|
||||
}
|
||||
|
||||
return DatasetMappingResponse(**response_data)
|
||||
|
||||
async def _to_response(
|
||||
self,
|
||||
mapping: LabelingProject,
|
||||
include_template: bool = False
|
||||
) -> DatasetMappingResponse:
|
||||
"""
|
||||
Convert ORM model to response with dataset name (for single entity operations)
|
||||
|
||||
Args:
|
||||
mapping: LabelingProject ORM object
|
||||
include_template: If True, fetch and include full template details
|
||||
"""
|
||||
# Fetch dataset name
|
||||
dataset_name = None
|
||||
dataset_id = getattr(mapping, 'dataset_id', None)
|
||||
if dataset_id:
|
||||
dataset_result = await self.db.execute(
|
||||
select(Dataset.name).where(Dataset.id == dataset_id)
|
||||
)
|
||||
dataset_name = dataset_result.scalar_one_or_none()
|
||||
|
||||
# Get template_id from mapping
|
||||
template_id = getattr(mapping, 'template_id', None)
|
||||
|
||||
# Optionally fetch full template details
|
||||
template_response = None
|
||||
if include_template and template_id:
|
||||
from ..service.template import AnnotationTemplateService
|
||||
template_service = AnnotationTemplateService()
|
||||
template_response = await template_service.get_template(self.db, template_id)
|
||||
logger.debug(f"Included template details for template_id: {template_id}")
|
||||
|
||||
# Create response dict with all fields
|
||||
response_data = {
|
||||
"id": mapping.id,
|
||||
"dataset_id": dataset_id,
|
||||
"dataset_name": dataset_name,
|
||||
"labeling_project_id": mapping.labeling_project_id,
|
||||
"name": mapping.name,
|
||||
"description": getattr(mapping, 'description', None),
|
||||
"template_id": template_id,
|
||||
"template": template_response,
|
||||
"created_at": mapping.created_at,
|
||||
"updated_at": mapping.updated_at,
|
||||
"deleted_at": mapping.deleted_at,
|
||||
}
|
||||
|
||||
return DatasetMappingResponse(**response_data)
|
||||
|
||||
async def create_mapping(
|
||||
self,
|
||||
labeling_project: LabelingProject
|
||||
) -> DatasetMappingResponse:
|
||||
"""创建数据集映射"""
|
||||
logger.debug(f"Create dataset mapping: {labeling_project.dataset_id} -> {labeling_project.labeling_project_id}")
|
||||
|
||||
# Use the passed object directly
|
||||
self.db.add(labeling_project)
|
||||
await self.db.commit()
|
||||
await self.db.refresh(labeling_project)
|
||||
|
||||
logger.debug(f"Mapping created: {labeling_project.id}")
|
||||
return await self._to_response(labeling_project)
|
||||
|
||||
async def get_mapping_by_source_uuid(
|
||||
self,
|
||||
dataset_id: str
|
||||
) -> Optional[DatasetMappingResponse]:
|
||||
"""根据源数据集ID获取映射（返回第一个未删除的）"""
|
||||
logger.debug(f"Get mapping by source dataset id: {dataset_id}")
|
||||
|
||||
result = await self.db.execute(
|
||||
select(LabelingProject).where(
|
||||
LabelingProject.dataset_id == dataset_id,
|
||||
LabelingProject.deleted_at.is_(None)
|
||||
)
|
||||
)
|
||||
mapping = result.scalar_one_or_none()
|
||||
|
||||
if mapping:
|
||||
logger.debug(f"Found mapping: {mapping.id}")
|
||||
return await self._to_response(mapping)
|
||||
|
||||
logger.debug(f"No mapping found for source dataset id: {dataset_id}")
|
||||
return None
|
||||
|
||||
async def get_mappings_by_dataset_id(
|
||||
self,
|
||||
dataset_id: str,
|
||||
include_deleted: bool = False
|
||||
) -> List[DatasetMappingResponse]:
|
||||
"""根据源数据集ID获取所有映射关系"""
|
||||
logger.debug(f"Get all mappings by source dataset id: {dataset_id}")
|
||||
|
||||
query = self._build_query_with_dataset_name().where(
|
||||
LabelingProject.dataset_id == dataset_id
|
||||
)
|
||||
|
||||
if not include_deleted:
|
||||
query = query.where(LabelingProject.deleted_at.is_(None))
|
||||
|
||||
result = await self.db.execute(
|
||||
query.order_by(LabelingProject.created_at.desc())
|
||||
)
|
||||
rows = result.all()
|
||||
|
||||
logger.debug(f"Found {len(rows)} mappings")
|
||||
# Convert rows to responses (async comprehension)
|
||||
responses = []
|
||||
for row in rows:
|
||||
response = await self._to_response_from_row(row, include_template=False)
|
||||
responses.append(response)
|
||||
return responses
|
||||
|
||||
async def get_mapping_by_labeling_project_id(
|
||||
self,
|
||||
labeling_project_id: str
|
||||
) -> Optional[DatasetMappingResponse]:
|
||||
"""根据Label Studio项目ID获取映射"""
|
||||
logger.debug(f"Get mapping by Label Studio project id: {labeling_project_id}")
|
||||
|
||||
result = await self.db.execute(
|
||||
select(LabelingProject).where(
|
||||
LabelingProject.labeling_project_id == labeling_project_id,
|
||||
LabelingProject.deleted_at.is_(None)
|
||||
)
|
||||
)
|
||||
mapping = result.scalar_one_or_none()
|
||||
|
||||
if mapping:
|
||||
logger.debug(f"Found mapping: {mapping.id}")
|
||||
return await self._to_response(mapping)
|
||||
|
||||
logger.debug(f"No mapping found for Label Studio project id: {labeling_project_id}")
|
||||
return None
|
||||
|
||||
async def get_mapping_by_uuid(
|
||||
self,
|
||||
mapping_id: str,
|
||||
include_template: bool = False
|
||||
) -> Optional[DatasetMappingResponse]:
|
||||
"""
|
||||
根据映射UUID获取映射
|
||||
|
||||
Args:
|
||||
mapping_id: 映射UUID
|
||||
include_template: 是否包含完整的模板信息
|
||||
"""
|
||||
logger.debug(f"Get mapping: {mapping_id}, include_template={include_template}")
|
||||
|
||||
result = await self.db.execute(
|
||||
select(LabelingProject).where(
|
||||
LabelingProject.id == mapping_id,
|
||||
LabelingProject.deleted_at.is_(None)
|
||||
)
|
||||
)
|
||||
mapping = result.scalar_one_or_none()
|
||||
|
||||
if mapping:
|
||||
logger.debug(f"Found mapping: {mapping.id}")
|
||||
return await self._to_response(mapping, include_template=include_template)
|
||||
|
||||
logger.debug(f"No mapping found for mapping id: {mapping_id}")
|
||||
return None
|
||||
|
||||
async def update_mapping(
|
||||
self,
|
||||
mapping_id: str,
|
||||
update_data: DatasetMappingUpdateRequest
|
||||
) -> Optional[DatasetMappingResponse]:
|
||||
"""更新映射信息"""
|
||||
logger.info(f"Update mapping: {mapping_id}")
|
||||
|
||||
mapping = await self.get_mapping_by_uuid(mapping_id)
|
||||
if not mapping:
|
||||
return None
|
||||
|
||||
update_values = update_data.model_dump(exclude_unset=True)
|
||||
update_values["last_updated_at"] = datetime.now()
|
||||
|
||||
result = await self.db.execute(
|
||||
update(LabelingProject)
|
||||
.where(LabelingProject.id == mapping_id)
|
||||
.values(**update_values)
|
||||
)
|
||||
await self.db.commit()
|
||||
|
||||
if result.rowcount and result.rowcount > 0: # type: ignore
|
||||
return await self.get_mapping_by_uuid(mapping_id)
|
||||
return None
|
||||
|
||||
async def soft_delete_mapping(self, mapping_id: str) -> bool:
|
||||
"""软删除映射"""
|
||||
logger.info(f"Soft delete mapping: {mapping_id}")
|
||||
|
||||
result = await self.db.execute(
|
||||
update(LabelingProject)
|
||||
.where(
|
||||
LabelingProject.id == mapping_id,
|
||||
LabelingProject.deleted_at.is_(None)
|
||||
)
|
||||
.values(deleted_at=datetime.now())
|
||||
)
|
||||
await self.db.commit()
|
||||
|
||||
success = result.rowcount and result.rowcount > 0 # type: ignore
|
||||
if success:
|
||||
logger.info(f"Mapping soft-deleted: {mapping_id}")
|
||||
else:
|
||||
logger.warning(f"Mapping not exists or already deleted: {mapping_id}")
|
||||
|
||||
return success
|
||||
|
||||
async def get_all_mappings(
|
||||
self,
|
||||
skip: int = 0,
|
||||
limit: int = 100
|
||||
) -> List[DatasetMappingResponse]:
|
||||
"""获取所有有效映射"""
|
||||
logger.debug(f"List all mappings, skip: {skip}, limit: {limit}")
|
||||
|
||||
query = self._build_query_with_dataset_name().where(
|
||||
LabelingProject.deleted_at.is_(None)
|
||||
)
|
||||
|
||||
result = await self.db.execute(
|
||||
query
|
||||
.offset(skip)
|
||||
.limit(limit)
|
||||
.order_by(LabelingProject.created_at.desc())
|
||||
)
|
||||
rows = result.all()
|
||||
|
||||
logger.debug(f"Found {len(rows)} mappings")
|
||||
# Convert rows to responses (async comprehension)
|
||||
responses = []
|
||||
for row in rows:
|
||||
response = await self._to_response_from_row(row, include_template=False)
|
||||
responses.append(response)
|
||||
return responses
|
||||
|
||||
async def count_mappings(self, include_deleted: bool = False) -> int:
|
||||
"""统计映射总数"""
|
||||
query = select(func.count()).select_from(LabelingProject)
|
||||
|
||||
if not include_deleted:
|
||||
query = query.where(LabelingProject.deleted_at.is_(None))
|
||||
|
||||
result = await self.db.execute(query)
|
||||
return result.scalar_one()
|
||||
|
||||
async def get_all_mappings_with_count(
|
||||
self,
|
||||
skip: int = 0,
|
||||
limit: int = 100,
|
||||
include_deleted: bool = False,
|
||||
include_template: bool = False
|
||||
) -> Tuple[List[DatasetMappingResponse], int]:
|
||||
"""
|
||||
获取所有映射及总数（用于分页）
|
||||
|
||||
Args:
|
||||
skip: 跳过记录数
|
||||
limit: 返回记录数
|
||||
include_deleted: 是否包含已删除的记录
|
||||
include_template: 是否包含完整的模板信息
|
||||
"""
|
||||
logger.debug(f"List all mappings with count, skip: {skip}, limit: {limit}, include_template={include_template}")
|
||||
|
||||
# 构建查询
|
||||
query = self._build_query_with_dataset_name()
|
||||
if not include_deleted:
|
||||
query = query.where(LabelingProject.deleted_at.is_(None))
|
||||
|
||||
# 获取总数
|
||||
count_query = select(func.count()).select_from(LabelingProject)
|
||||
if not include_deleted:
|
||||
count_query = count_query.where(LabelingProject.deleted_at.is_(None))
|
||||
|
||||
count_result = await self.db.execute(count_query)
|
||||
total = count_result.scalar_one()
|
||||
|
||||
# 获取数据
|
||||
result = await self.db.execute(
|
||||
query
|
||||
.offset(skip)
|
||||
.limit(limit)
|
||||
.order_by(LabelingProject.created_at.desc())
|
||||
)
|
||||
rows = result.all()
|
||||
|
||||
# 过滤掉由自动标注任务自动创建的隐藏工程（configuration.autoTaskId 存在）
|
||||
filtered_rows = []
|
||||
for row in rows:
|
||||
mapping: LabelingProject = row[0]
|
||||
cfg = getattr(mapping, "configuration", None) or {}
|
||||
if isinstance(cfg, dict) and cfg.get("autoTaskId"):
|
||||
continue
|
||||
filtered_rows.append(row)
|
||||
|
||||
logger.debug(
|
||||
f"Found {len(filtered_rows)} visible mappings (filtered from {len(rows)}), total: {total}"
|
||||
)
|
||||
# Convert rows to responses (async comprehension)
|
||||
responses = []
|
||||
for row in filtered_rows:
|
||||
response = await self._to_response_from_row(row, include_template=include_template)
|
||||
responses.append(response)
|
||||
# total 仍然返回数据库总数，目前前端只关注当前页内容数量，故不调整 total
|
||||
return responses, total
|
||||
|
||||
async def get_template_id_by_dataset_id(self, dataset_id: str) -> Optional[str]:
|
||||
"""
|
||||
Get template ID for a dataset by finding its labeling project
|
||||
|
||||
Args:
|
||||
dataset_id: Dataset UUID
|
||||
|
||||
Returns:
|
||||
Template ID or None if no labeling project found or no template associated
|
||||
"""
|
||||
logger.debug(f"Looking up template for dataset: {dataset_id}")
|
||||
|
||||
result = await self.db.execute(
|
||||
select(LabelingProject.template_id)
|
||||
.where(
|
||||
LabelingProject.dataset_id == dataset_id,
|
||||
LabelingProject.deleted_at.is_(None)
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
template_id = result.scalar_one_or_none()
|
||||
|
||||
if template_id:
|
||||
logger.debug(f"Found template {template_id} for dataset {dataset_id}")
|
||||
else:
|
||||
logger.warning(f"No template found for dataset {dataset_id}")
|
||||
|
||||
return template_id
|
||||
|
||||
async def get_mappings_by_source_with_count(
|
||||
self,
|
||||
dataset_id: str,
|
||||
skip: int = 0,
|
||||
limit: int = 100,
|
||||
include_deleted: bool = False,
|
||||
include_template: bool = False
|
||||
) -> Tuple[List[DatasetMappingResponse], int]:
|
||||
"""
|
||||
根据源数据集ID获取映射关系及总数（用于分页）
|
||||
|
||||
Args:
|
||||
dataset_id: 数据集ID
|
||||
skip: 跳过记录数
|
||||
limit: 返回记录数
|
||||
include_deleted: 是否包含已删除的记录
|
||||
include_template: 是否包含完整的模板信息
|
||||
"""
|
||||
logger.debug(f"Get mappings by source dataset id with count: {dataset_id}, include_template={include_template}")
|
||||
|
||||
# 构建查询
|
||||
query = self._build_query_with_dataset_name().where(
|
||||
LabelingProject.dataset_id == dataset_id
|
||||
)
|
||||
|
||||
if not include_deleted:
|
||||
query = query.where(LabelingProject.deleted_at.is_(None))
|
||||
|
||||
# 获取总数
|
||||
count_query = select(func.count()).select_from(LabelingProject).where(
|
||||
LabelingProject.dataset_id == dataset_id
|
||||
)
|
||||
if not include_deleted:
|
||||
count_query = count_query.where(LabelingProject.deleted_at.is_(None))
|
||||
|
||||
count_result = await self.db.execute(count_query)
|
||||
total = count_result.scalar_one()
|
||||
|
||||
# 获取数据
|
||||
result = await self.db.execute(
|
||||
query
|
||||
.offset(skip)
|
||||
.limit(limit)
|
||||
.order_by(LabelingProject.created_at.desc())
|
||||
)
|
||||
rows = result.all()
|
||||
|
||||
# 过滤掉由自动标注任务自动创建的隐藏工程（configuration.autoTaskId 存在）
|
||||
filtered_rows = []
|
||||
for row in rows:
|
||||
mapping: LabelingProject = row[0]
|
||||
cfg = getattr(mapping, "configuration", None) or {}
|
||||
if isinstance(cfg, dict) and cfg.get("autoTaskId"):
|
||||
continue
|
||||
filtered_rows.append(row)
|
||||
|
||||
logger.debug(
|
||||
f"Found {len(filtered_rows)} mappings for dataset {dataset_id} (filtered from {len(rows)}), total: {total}"
|
||||
)
|
||||
# Convert rows to responses (async comprehension)
|
||||
responses = []
|
||||
for row in filtered_rows:
|
||||
response = await self._to_response_from_row(row, include_template=include_template)
|
||||
responses.append(response)
|
||||
return responses, total
|
||||
@@ -0,0 +1,208 @@
|
||||
"""服务：将自动标注（YOLO）结果同步到 Label Studio 作为 predictions。
|
||||
|
||||
该服务不负责执行模型推理，只负责：
|
||||
- 读取本地 annotations JSON（由 runtime/ops/annotation/image_object_detection_bounding_box 产出）；
|
||||
- 根据 Label Studio 项目的任务列表，将每个文件对应到具体 task；
|
||||
- 将检测结果转换为 Label Studio 的 prediction 结构并写入。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
from app.core.logging import get_logger
|
||||
|
||||
from ..client import LabelStudioClient
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class PredictionSyncService:
|
||||
"""将 YOLO 检测结果推送到 Label Studio 的预测服务。"""
|
||||
|
||||
def __init__(self, ls_client: LabelStudioClient) -> None:
|
||||
self.ls_client = ls_client
|
||||
|
||||
async def _build_task_index(self, project_id: str) -> Dict[str, int]:
|
||||
"""为指定项目构建 "文件名 -> task_id" 索引。
|
||||
|
||||
优先使用 data.original_name，其次退回到从 data.image / data.file_path
|
||||
中解析出的文件名，保证与 YOLO JSON 中的 annotations["image"] 对得上。
|
||||
"""
|
||||
|
||||
result = await self.ls_client.get_project_tasks(project_id=project_id, page=None)
|
||||
if not result:
|
||||
logger.warning("No tasks found for project %s", project_id)
|
||||
return {}
|
||||
|
||||
tasks = result.get("tasks", []) or []
|
||||
index: Dict[str, int] = {}
|
||||
|
||||
for task in tasks:
|
||||
task_id = task.get("id")
|
||||
data = task.get("data", {}) or {}
|
||||
if task_id is None:
|
||||
continue
|
||||
|
||||
# candidate names that may appear in YOLO annotations["image"]
|
||||
candidates: List[str] = []
|
||||
|
||||
original_name = data.get("original_name")
|
||||
if isinstance(original_name, str) and original_name:
|
||||
candidates.append(original_name)
|
||||
|
||||
image_val = data.get("image") or data.get("file_path")
|
||||
if isinstance(image_val, str) and image_val:
|
||||
basename = os.path.basename(image_val)
|
||||
if basename:
|
||||
candidates.append(basename)
|
||||
|
||||
for name in candidates:
|
||||
# 后写覆盖前写没关系，同一文件名对应同一个 task 的预期
|
||||
index[name] = task_id
|
||||
|
||||
logger.debug(
|
||||
"Built task index for project %s with %d entries", project_id, len(index)
|
||||
)
|
||||
return index
|
||||
|
||||
@staticmethod
|
||||
def _build_ls_result_from_detections(
|
||||
annotations: Dict[str, Any],
|
||||
from_name: str = "label",
|
||||
to_name: str = "image",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""将 YOLO annotations["detections"] 转为 Label Studio result 列表。"""
|
||||
|
||||
width = float(annotations.get("width") or 0) or 1.0
|
||||
height = float(annotations.get("height") or 0) or 1.0
|
||||
detections = annotations.get("detections") or []
|
||||
|
||||
results: List[Dict[str, Any]] = []
|
||||
|
||||
for det in detections:
|
||||
try:
|
||||
label = det.get("label")
|
||||
bbox = det.get("bbox_xywh") or det.get("bbox_xyxy")
|
||||
if not label or not bbox or len(bbox) < 4:
|
||||
continue
|
||||
|
||||
# 优先使用 xywh，单位像素
|
||||
if len(bbox) == 4:
|
||||
x, y, w, h = map(float, bbox)
|
||||
else:
|
||||
# 回退：如果给的是 xyxy
|
||||
x1, y1, x2, y2 = map(float, bbox[:4])
|
||||
x, y, w, h = x1, y1, x2 - x1, y2 - y1
|
||||
|
||||
x_pct = max(0.0, min(100.0, x / width * 100.0))
|
||||
y_pct = max(0.0, min(100.0, y / height * 100.0))
|
||||
w_pct = max(0.0, min(100.0, w / width * 100.0))
|
||||
h_pct = max(0.0, min(100.0, h / height * 100.0))
|
||||
|
||||
region_id = uuid.uuid4().hex[:10]
|
||||
|
||||
results.append(
|
||||
{
|
||||
"id": region_id,
|
||||
"type": "rectanglelabels",
|
||||
"value": {
|
||||
"x": x_pct,
|
||||
"y": y_pct,
|
||||
"width": w_pct,
|
||||
"height": h_pct,
|
||||
"rotation": 0,
|
||||
"rectanglelabels": [label],
|
||||
},
|
||||
"origin": "prediction",
|
||||
"to_name": to_name,
|
||||
"from_name": from_name,
|
||||
"image_rotation": 0,
|
||||
"original_width": width,
|
||||
"original_height": height,
|
||||
}
|
||||
)
|
||||
except Exception as exc: # 防御性：单条失败不影响其他
|
||||
logger.warning("Failed to convert detection to LS result: %s", exc)
|
||||
continue
|
||||
|
||||
return results
|
||||
|
||||
async def sync_predictions_from_dir(
|
||||
self,
|
||||
project_id: str,
|
||||
annotations_dir: str,
|
||||
*,
|
||||
model_version: Optional[str] = None,
|
||||
from_name: str = "label",
|
||||
to_name: str = "image",
|
||||
) -> int:
|
||||
"""将某个目录下的 YOLO JSON 同步为给定项目的 predictions。
|
||||
|
||||
返回成功写入 prediction 的任务数。
|
||||
"""
|
||||
|
||||
if not os.path.isdir(annotations_dir):
|
||||
logger.warning("annotations_dir '%s' is not a directory", annotations_dir)
|
||||
return 0
|
||||
|
||||
task_index = await self._build_task_index(project_id)
|
||||
if not task_index:
|
||||
logger.warning(
|
||||
"No tasks found when syncing predictions for project %s", project_id
|
||||
)
|
||||
return 0
|
||||
|
||||
created_count = 0
|
||||
|
||||
for filename in os.listdir(annotations_dir):
|
||||
if not filename.lower().endswith(".json"):
|
||||
continue
|
||||
|
||||
json_path = os.path.join(annotations_dir, filename)
|
||||
|
||||
try:
|
||||
with open(json_path, "r", encoding="utf-8") as f:
|
||||
annotations = json.load(f)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to load annotations json %s: %s", json_path, exc)
|
||||
continue
|
||||
|
||||
image_name = annotations.get("image") or filename.rsplit(".", 1)[0]
|
||||
task_id = task_index.get(image_name)
|
||||
if task_id is None:
|
||||
logger.info(
|
||||
"No matching task for image '%s' in project %s, skip", image_name, project_id
|
||||
)
|
||||
continue
|
||||
|
||||
result = self._build_ls_result_from_detections(
|
||||
annotations, from_name=from_name, to_name=to_name
|
||||
)
|
||||
if not result:
|
||||
logger.info("No detections for %s, skip creating prediction", image_name)
|
||||
continue
|
||||
|
||||
mv = model_version
|
||||
if mv is None:
|
||||
model_size = annotations.get("model_size") or ""
|
||||
mv = f"yolov8{model_size}" if model_size else "yolov8"
|
||||
|
||||
prediction = await self.ls_client.create_prediction(
|
||||
task_id=task_id,
|
||||
result=result,
|
||||
model_version=mv,
|
||||
)
|
||||
if prediction:
|
||||
created_count += 1
|
||||
|
||||
logger.info(
|
||||
"Synced YOLO predictions from '%s' to project %s, created %d predictions",
|
||||
annotations_dir,
|
||||
project_id,
|
||||
created_count,
|
||||
)
|
||||
return created_count
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,341 @@
|
||||
"""
|
||||
Annotation Template Service
|
||||
"""
|
||||
from typing import Optional, List
|
||||
from datetime import datetime
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from uuid import uuid4
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.db.models.annotation_management import AnnotationTemplate
|
||||
from app.module.annotation.schema.template import (
|
||||
CreateAnnotationTemplateRequest,
|
||||
UpdateAnnotationTemplateRequest,
|
||||
AnnotationTemplateResponse,
|
||||
AnnotationTemplateListResponse,
|
||||
TemplateConfiguration
|
||||
)
|
||||
from app.module.annotation.utils.config_validator import LabelStudioConfigValidator
|
||||
from app.module.annotation.config import LabelStudioTagConfig
|
||||
|
||||
|
||||
class AnnotationTemplateService:
|
||||
"""标注模板服务"""
|
||||
|
||||
@staticmethod
|
||||
def generate_label_studio_config(config: TemplateConfiguration) -> str:
|
||||
"""
|
||||
从配置JSON生成Label Studio XML配置
|
||||
|
||||
Args:
|
||||
config: 模板配置对象
|
||||
|
||||
Returns:
|
||||
Label Studio XML字符串
|
||||
"""
|
||||
tag_config = LabelStudioTagConfig()
|
||||
xml_parts = ['<View>']
|
||||
|
||||
# 生成对象定义
|
||||
for obj in config.objects:
|
||||
obj_attrs = [
|
||||
f'name="{obj.name}"',
|
||||
f'value="{obj.value}"'
|
||||
]
|
||||
xml_parts.append(f' <{obj.type} {" ".join(obj_attrs)}/>')
|
||||
|
||||
# 生成标签定义
|
||||
for label in config.labels:
|
||||
label_attrs = [
|
||||
f'name="{label.from_name}"',
|
||||
f'toName="{label.to_name}"'
|
||||
]
|
||||
|
||||
# 添加可选属性
|
||||
if label.required:
|
||||
label_attrs.append('required="true"')
|
||||
|
||||
if label.choice:
|
||||
label_attrs.append(f'choice="{label.choice}"')
|
||||
|
||||
# xuhuanqing
|
||||
#tag_type = label.type.capitalize() if label.type else "Choices"
|
||||
tag_type = label.type if label.type else "Choices"
|
||||
|
||||
# 检查是否需要子元素
|
||||
if label.options or label.labels:
|
||||
choices = label.options or label.labels or []
|
||||
xml_parts.append(f' <{tag_type} {" ".join(label_attrs)}>')
|
||||
|
||||
# 从配置获取子元素标签名
|
||||
child_tag = tag_config.get_child_tag(tag_type)
|
||||
if not child_tag:
|
||||
# 默认使用 Label
|
||||
child_tag = "Label"
|
||||
|
||||
for choice in choices:
|
||||
xml_parts.append(f' <{child_tag} value="{choice}"/>')
|
||||
xml_parts.append(f' </{tag_type}>')
|
||||
else:
|
||||
# 处理简单标签类型（不需要子元素）
|
||||
xml_parts.append(f' <{tag_type} {" ".join(label_attrs)}/>')
|
||||
|
||||
xml_parts.append('</View>')
|
||||
return '\n'.join(xml_parts)
|
||||
|
||||
async def create_template(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
request: CreateAnnotationTemplateRequest
|
||||
) -> AnnotationTemplateResponse:
|
||||
"""
|
||||
创建标注模板
|
||||
|
||||
Args:
|
||||
db: 数据库会话
|
||||
request: 创建请求
|
||||
|
||||
Returns:
|
||||
创建的模板响应
|
||||
"""
|
||||
# 验证配置JSON
|
||||
config_dict = request.configuration.model_dump(mode='json', by_alias=False)
|
||||
valid, error = LabelStudioConfigValidator.validate_configuration_json(config_dict)
|
||||
if not valid:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid configuration: {error}")
|
||||
|
||||
# 生成Label Studio XML配置（用于验证，但不存储）
|
||||
label_config = self.generate_label_studio_config(request.configuration)
|
||||
|
||||
# 验证生成的XML
|
||||
valid, error = LabelStudioConfigValidator.validate_xml(label_config)
|
||||
if not valid:
|
||||
raise HTTPException(status_code=400, detail=f"Generated XML is invalid: {error}")
|
||||
|
||||
# 创建模板对象（不包含label_config字段）
|
||||
template = AnnotationTemplate(
|
||||
id=str(uuid4()),
|
||||
name=request.name,
|
||||
description=request.description,
|
||||
data_type=request.data_type,
|
||||
labeling_type=request.labeling_type,
|
||||
configuration=config_dict,
|
||||
style=request.style,
|
||||
category=request.category,
|
||||
built_in=False,
|
||||
version="1.0.0",
|
||||
created_at=datetime.now()
|
||||
)
|
||||
|
||||
db.add(template)
|
||||
await db.commit()
|
||||
await db.refresh(template)
|
||||
|
||||
return self._to_response(template)
|
||||
|
||||
async def get_template(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
template_id: str
|
||||
) -> Optional[AnnotationTemplateResponse]:
|
||||
"""
|
||||
获取单个模板
|
||||
|
||||
Args:
|
||||
db: 数据库会话
|
||||
template_id: 模板ID
|
||||
|
||||
Returns:
|
||||
模板响应或None
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(AnnotationTemplate)
|
||||
.where(
|
||||
AnnotationTemplate.id == template_id,
|
||||
AnnotationTemplate.deleted_at.is_(None)
|
||||
)
|
||||
)
|
||||
template = result.scalar_one_or_none()
|
||||
|
||||
if template:
|
||||
return self._to_response(template)
|
||||
return None
|
||||
|
||||
async def list_templates(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
page: int = 1,
|
||||
size: int = 10,
|
||||
category: Optional[str] = None,
|
||||
data_type: Optional[str] = None,
|
||||
labeling_type: Optional[str] = None,
|
||||
built_in: Optional[bool] = None
|
||||
) -> AnnotationTemplateListResponse:
|
||||
"""
|
||||
获取模板列表
|
||||
|
||||
Args:
|
||||
db: 数据库会话
|
||||
page: 页码
|
||||
size: 每页大小
|
||||
category: 分类筛选
|
||||
data_type: 数据类型筛选
|
||||
labeling_type: 标注类型筛选
|
||||
built_in: 是否内置模板筛选
|
||||
|
||||
Returns:
|
||||
模板列表响应
|
||||
"""
|
||||
# 构建查询条件
|
||||
conditions: List = [AnnotationTemplate.deleted_at.is_(None)]
|
||||
|
||||
if category:
|
||||
conditions.append(AnnotationTemplate.category == category) # type: ignore
|
||||
if data_type:
|
||||
conditions.append(AnnotationTemplate.data_type == data_type) # type: ignore
|
||||
if labeling_type:
|
||||
conditions.append(AnnotationTemplate.labeling_type == labeling_type) # type: ignore
|
||||
if built_in is not None:
|
||||
conditions.append(AnnotationTemplate.built_in == built_in) # type: ignore
|
||||
|
||||
# 查询总数
|
||||
count_result = await db.execute(
|
||||
select(func.count()).select_from(AnnotationTemplate).where(*conditions)
|
||||
)
|
||||
total = count_result.scalar() or 0
|
||||
|
||||
# 分页查询
|
||||
result = await db.execute(
|
||||
select(AnnotationTemplate)
|
||||
.where(*conditions)
|
||||
.order_by(AnnotationTemplate.created_at.desc())
|
||||
.limit(size)
|
||||
.offset((page - 1) * size)
|
||||
)
|
||||
templates = result.scalars().all()
|
||||
|
||||
return AnnotationTemplateListResponse(
|
||||
content=[self._to_response(t) for t in templates],
|
||||
totalElements=total,
|
||||
page=page,
|
||||
size=size,
|
||||
totalPages=(total + size - 1) // size
|
||||
)
|
||||
|
||||
async def update_template(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
template_id: str,
|
||||
request: UpdateAnnotationTemplateRequest
|
||||
) -> Optional[AnnotationTemplateResponse]:
|
||||
"""
|
||||
更新模板
|
||||
|
||||
Args:
|
||||
db: 数据库会话
|
||||
template_id: 模板ID
|
||||
request: 更新请求
|
||||
|
||||
Returns:
|
||||
更新后的模板响应或None
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(AnnotationTemplate)
|
||||
.where(
|
||||
AnnotationTemplate.id == template_id,
|
||||
AnnotationTemplate.deleted_at.is_(None)
|
||||
)
|
||||
)
|
||||
template = result.scalar_one_or_none()
|
||||
|
||||
if not template:
|
||||
return None
|
||||
|
||||
# 更新字段
|
||||
update_data = request.model_dump(exclude_unset=True, by_alias=False)
|
||||
|
||||
for field, value in update_data.items():
|
||||
if field == 'configuration' and value is not None:
|
||||
# 验证配置JSON
|
||||
config_dict = value.model_dump(mode='json', by_alias=False)
|
||||
valid, error = LabelStudioConfigValidator.validate_configuration_json(config_dict)
|
||||
if not valid:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid configuration: {error}")
|
||||
|
||||
# 重新生成Label Studio XML配置（用于验证）
|
||||
label_config = self.generate_label_studio_config(value)
|
||||
|
||||
# 验证生成的XML
|
||||
valid, error = LabelStudioConfigValidator.validate_xml(label_config)
|
||||
if not valid:
|
||||
raise HTTPException(status_code=400, detail=f"Generated XML is invalid: {error}")
|
||||
|
||||
# 只更新configuration字段，不存储label_config
|
||||
setattr(template, field, config_dict)
|
||||
else:
|
||||
setattr(template, field, value)
|
||||
|
||||
template.updated_at = datetime.now() # type: ignore
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(template)
|
||||
|
||||
return self._to_response(template)
|
||||
|
||||
async def delete_template(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
template_id: str
|
||||
) -> bool:
|
||||
"""
|
||||
删除模板（软删除）
|
||||
|
||||
Args:
|
||||
db: 数据库会话
|
||||
template_id: 模板ID
|
||||
|
||||
Returns:
|
||||
是否删除成功
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(AnnotationTemplate)
|
||||
.where(
|
||||
AnnotationTemplate.id == template_id,
|
||||
AnnotationTemplate.deleted_at.is_(None)
|
||||
)
|
||||
)
|
||||
template = result.scalar_one_or_none()
|
||||
|
||||
if not template:
|
||||
return False
|
||||
|
||||
template.deleted_at = datetime.now() # type: ignore
|
||||
await db.commit()
|
||||
|
||||
return True
|
||||
|
||||
def _to_response(self, template: AnnotationTemplate) -> AnnotationTemplateResponse:
|
||||
"""
|
||||
转换为响应对象
|
||||
|
||||
Args:
|
||||
template: 数据库模型对象
|
||||
|
||||
Returns:
|
||||
模板响应对象
|
||||
"""
|
||||
# 将配置JSON转换为TemplateConfiguration对象
|
||||
from typing import cast, Dict, Any
|
||||
config_dict = cast(Dict[str, Any], template.configuration)
|
||||
config = TemplateConfiguration(**config_dict)
|
||||
|
||||
# 动态生成Label Studio XML配置
|
||||
label_config = self.generate_label_studio_config(config)
|
||||
|
||||
# 使用model_validate从ORM对象创建响应对象
|
||||
response = AnnotationTemplateResponse.model_validate(template)
|
||||
response.configuration = config
|
||||
response.label_config = label_config # type: ignore
|
||||
|
||||
return response
|
||||
Reference in New Issue
Block a user