deepdata: core product code (backend, frontend, runtime, deepflow module)

2026-08-30 19:23:38 +08:00
commit bd3db30be9
1405 changed files with 271727 additions and 0 deletions
@@ -0,0 +1,11 @@
+from .cleaning_task_repository import CleaningTaskRepository
+from .cleaning_template_repository import CleaningTemplateRepository
+from .cleaning_result_repository import CleaningResultRepository
+from .operator_instance_repository import OperatorInstanceRepository
+
+__all__ = [
+    "CleaningTaskRepository",
+    "CleaningTemplateRepository",
+    "CleaningResultRepository",
+    "OperatorInstanceRepository",
+]
@@ -0,0 +1,110 @@
+from typing import List, Optional
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import select, delete, func, Integer
+from app.db.models.cleaning import CleaningResult
+from app.module.cleaning.schema import CleaningResultDto
+
+
+class CleaningResultRepository:
+    """Repository for cleaning result operations"""
+
+    def __init__(self, model=None):
+        self.model = model if model else CleaningResult
+
+    async def find_by_instance_id(
+        self,
+        db: AsyncSession,
+        instance_id: str,
+        status: Optional[str] = None
+    ) -> List[CleaningResultDto]:
+        """Query results by instance ID"""
+        query = select(self.model).where(self.model.instance_id == instance_id)
+
+        if status:
+            query = query.where(self.model.status == status)
+
+        result = await db.execute(query)
+        results = result.scalars().all()
+
+        return [
+            CleaningResultDto(
+                instance_id=res.instance_id,
+                src_file_id=res.src_file_id,
+                dest_file_id=res.dest_file_id,
+                src_name=res.src_name,
+                dest_name=res.dest_name,
+                src_type=res.src_type,
+                dest_type=res.dest_type,
+                src_size=res.src_size,
+                dest_size=res.dest_size,
+                status=res.status,
+                result=res.result
+            )
+            for res in results
+        ]
+
+    async def count_by_instance_id(
+        self,
+        db: AsyncSession,
+        instance_id: str
+    ) -> tuple[int, int]:
+        """Count results by instance ID (completed, failed) using single SQL"""
+        query = select(
+            func.sum(func.cast(self.model.status == "COMPLETED", Integer)).label("completed"),
+            func.sum(func.cast(self.model.status == "FAILED", Integer)).label("failed"),
+        ).where(self.model.instance_id == instance_id)
+        result = await db.execute(query)
+        row = result.one()
+        return (row.completed or 0, row.failed or 0)
+
+    async def count_total_by_instance_id(
+        self,
+        db: AsyncSession,
+        instance_id: str
+    ) -> int:
+        """Count total results by instance ID using efficient SQL COUNT"""
+        query = select(func.count()).select_from(self.model).where(
+            self.model.instance_id == instance_id
+        )
+        result = await db.scalar(query)
+        return result or 0
+
+    async def batch_count_by_instance_ids(
+        self,
+        db: AsyncSession,
+        instance_ids: list[str]
+    ) -> dict[str, tuple[int, int, int]]:
+        """Batch count completed/failed/total for multiple instance IDs using single SQL.
+
+        Returns dict: {instance_id: (completed, failed, total)}
+        """
+        if not instance_ids:
+            return {}
+
+        query = select(
+            self.model.instance_id,
+            func.sum(func.cast(self.model.status == "COMPLETED", Integer)).label("completed"),
+            func.sum(func.cast(self.model.status == "FAILED", Integer)).label("failed"),
+            func.count().label("total"),
+        ).where(self.model.instance_id.in_(instance_ids)).group_by(self.model.instance_id)
+
+        result = await db.execute(query)
+        return {
+            row.instance_id: (row.completed or 0, row.failed or 0, row.total or 0)
+            for row in result.all()
+        }
+
+    async def delete_by_instance_id(
+        self,
+        db: AsyncSession,
+        instance_id: str,
+        status: Optional[str] = None
+    ) -> None:
+        """Delete results by instance ID"""
+        query = delete(self.model).where(self.model.instance_id == instance_id)
+
+        if status:
+            query = query.where(self.model.status == status)
+
+        await db.execute(query)
+        await db.flush()
@@ -0,0 +1,161 @@
+from typing import List, Optional
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import select, delete, func
+from app.db.models.cleaning import CleaningTask
+from app.module.cleaning.schema import CleaningTaskDto
+
+
+class CleaningTaskRepository:
+    """Repository for cleaning task operations"""
+
+    def __init__(self, model=None):
+        self.model = model if model else CleaningTask
+
+    async def find_tasks(
+        self,
+        db: AsyncSession,
+        status: Optional[str] = None,
+        keyword: Optional[str] = None,
+        page: Optional[int] = None,
+        size: Optional[int] = None
+    ) -> List[CleaningTaskDto]:
+        """Query cleaning tasks"""
+        query = select(self.model)
+
+        if status:
+            query = query.where(self.model.status == status)
+
+        if keyword:
+            keyword_pattern = f"%{keyword}%"
+            query = query.where(
+                self.model.name.ilike(keyword_pattern) | self.model.description.ilike(keyword_pattern)
+            )
+
+        query = query.order_by(self.model.created_at.desc())
+
+        if page is not None and size is not None:
+            offset = max(page * size, 0)
+            query = query.offset(offset).limit(size)
+
+        result = await db.execute(query)
+        tasks = result.scalars().all()
+
+        return [
+            CleaningTaskDto(
+                id=task.id,
+                name=task.name,
+                description=task.description,
+                status=task.status,
+                src_dataset_id=task.src_dataset_id,
+                src_dataset_name=task.src_dataset_name,
+                dest_dataset_id=task.dest_dataset_id,
+                dest_dataset_name=task.dest_dataset_name,
+                before_size=task.before_size,
+                after_size=task.after_size,
+                file_count=task.file_count,
+                retry_count=task.retry_count,
+                started_at=task.started_at,
+                finished_at=task.finished_at,
+                created_at=task.created_at
+            )
+            for task in tasks
+        ]
+
+    async def find_task_by_id(self, db: AsyncSession, task_id: str) -> Optional[CleaningTaskDto]:
+        """Query task by ID"""
+        query = select(self.model).where(self.model.id == task_id)
+        result = await db.execute(query)
+        task = result.scalar_one_or_none()
+
+        if not task:
+            return None
+
+        return CleaningTaskDto(
+            id=task.id,
+            name=task.name,
+            description=task.description,
+            status=task.status,
+            src_dataset_id=task.src_dataset_id,
+            src_dataset_name=task.src_dataset_name,
+            dest_dataset_id=task.dest_dataset_id,
+            dest_dataset_name=task.dest_dataset_name,
+            before_size=task.before_size,
+            after_size=task.after_size,
+            file_count=task.file_count,
+            retry_count=task.retry_count,
+            started_at=task.started_at,
+            finished_at=task.finished_at,
+            created_at=task.created_at
+        )
+
+    async def insert_task(self, db: AsyncSession, task: CleaningTaskDto) -> None:
+        """Insert new task"""
+        from app.db.models.cleaning import CleaningTask as CleaningTaskModel
+        
+        db_task = CleaningTaskModel(
+            id=task.id,
+            name=task.name,
+            description=task.description,
+            status=task.status,
+            src_dataset_id=task.src_dataset_id,
+            src_dataset_name=task.src_dataset_name,
+            dest_dataset_id=task.dest_dataset_id,
+            dest_dataset_name=task.dest_dataset_name,
+            before_size=task.before_size,
+            after_size=task.after_size,
+            file_count=task.file_count,
+            retry_count=task.retry_count
+        )
+        db.add(db_task)
+        await db.flush()
+
+    async def update_task(self, db: AsyncSession, task: CleaningTaskDto) -> None:
+        """Update task"""
+        query = select(CleaningTask).where(CleaningTask.id == task.id)
+        result = await db.execute(query)
+        db_task = result.scalar_one_or_none()
+
+        if db_task:
+            if task.status:
+                db_task.status = task.status
+            if task.started_at:
+                db_task.started_at = task.started_at
+            if task.finished_at:
+                db_task.finished_at = task.finished_at
+            if task.retry_count is not None:
+                db_task.retry_count = task.retry_count
+
+            await db.flush()
+
+    async def delete_task_by_id(self, db: AsyncSession, task_id: str) -> None:
+        """Delete task by ID"""
+        query = delete(self.model).where(self.model.id == task_id)
+        await db.execute(query)
+        await db.flush()
+
+    async def count_tasks(
+        self,
+        db: AsyncSession,
+        status: Optional[str] = None,
+        keyword: Optional[str] = None
+    ) -> int:
+        """Count cleaning tasks using SQL COUNT"""
+        query = select(func.count()).select_from(self.model)
+
+        if status:
+            query = query.where(self.model.status == status)
+
+        if keyword:
+            keyword_pattern = f"%{keyword}%"
+            query = query.where(
+                self.model.name.ilike(keyword_pattern) | self.model.description.ilike(keyword_pattern)
+            )
+
+        result = await db.scalar(query)
+        return result or 0
+
+    async def is_name_exist(self, db: AsyncSession, name: str) -> bool:
+        """Check if task name exists"""
+        query = select(func.count()).select_from(self.model).where(self.model.name == name)
+        result = await db.execute(query)
+        return result.scalar_one() > 0 if result else False
@@ -0,0 +1,95 @@
+from typing import List, Optional
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import select, delete, func
+from app.db.models.cleaning import CleaningTemplate
+
+
+class CleaningTemplateRepository:
+    """Repository for cleaning template operations"""
+
+    def __init__(self, model=None):
+        self.model = model if model else CleaningTemplate
+
+    async def find_all_templates(
+        self,
+        db: AsyncSession,
+        keyword: Optional[str] = None,
+        page: Optional[int] = None,
+        size: Optional[int] = None,
+    ) -> List[CleaningTemplate]:
+        """Query all templates with optional pagination"""
+        query = select(self.model)
+
+        if keyword:
+            keyword_pattern = f"%{keyword}%"
+            query = query.where(
+                self.model.name.ilike(keyword_pattern)
+                | self.model.description.ilike(keyword_pattern)
+            )
+
+        query = query.order_by(self.model.created_at.desc())
+
+        if page is not None and size is not None:
+            offset = page * size
+            query = query.offset(offset).limit(size)
+
+        result = await db.execute(query)
+        return result.scalars().all()
+
+    async def count_templates(
+        self, db: AsyncSession, keyword: Optional[str] = None
+    ) -> int:
+        """Count templates"""
+        query = select(func.count()).select_from(self.model)
+
+        if keyword:
+            keyword_pattern = f"%{keyword}%"
+            query = query.where(
+                self.model.name.ilike(keyword_pattern)
+                | self.model.description.ilike(keyword_pattern)
+            )
+
+        result = await db.execute(query)
+        return result.scalar_one()
+
+    async def find_template_by_id(
+        self, db: AsyncSession, template_id: str
+    ) -> Optional[CleaningTemplate]:
+        """Query template by ID"""
+        query = select(self.model).where(self.model.id == template_id)
+        result = await db.execute(query)
+        return result.scalar_one_or_none()
+
+    async def insert_template(
+        self, db: AsyncSession, template: CleaningTemplate
+    ) -> None:
+        """Insert new template"""
+        db.add(template)
+        await db.flush()
+
+    async def update_template(
+        self, db: AsyncSession, template: CleaningTemplate
+    ) -> None:
+        """Update template"""
+        query = select(self.model).where(self.model.id == template.id)
+        result = await db.execute(query)
+        db_template = result.scalar_one_or_none()
+
+        if db_template:
+            db_template.name = template.name
+            db_template.description = template.description
+            await db.flush()
+
+    async def delete_template(self, db: AsyncSession, template_id: str) -> None:
+        """Delete template"""
+        query = delete(self.model).where(self.model.id == template_id)
+        await db.execute(query)
+        await db.flush()
+
+    async def is_name_exist(self, db: AsyncSession, name: str) -> bool:
+        """Check if template name exists"""
+        query = (
+            select(func.count()).select_from(self.model).where(self.model.name == name)
+        )
+        result = await db.execute(query)
+        return result.scalar_one() > 0 if result else False
@@ -0,0 +1,56 @@
+import json
+from typing import List
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import select, delete
+from app.db.models.cleaning import OperatorInstance
+
+
+class OperatorInstanceRepository:
+    """Repository for operator instance operations"""
+
+    def __init__(self, model=None):
+        self.model = model if model else OperatorInstance
+
+    async def find_operator_by_instance_id(
+        self,
+        db: AsyncSession,
+        instance_id: str
+    ) -> List[OperatorInstance]:
+        """Query operator instances by instance ID"""
+        query = select(self.model).where(self.model.instance_id == instance_id)
+        query = query.order_by(self.model.op_index.asc())
+        result = await db.execute(query)
+        return result.scalars().all()
+
+    async def find_instance_by_instance_id(
+        self,
+        db: AsyncSession,
+        instance_id: str
+    ) -> List[OperatorInstance]:
+        """Query instances for template (same as find_operator_by_instance_id)"""
+        return await self.find_operator_by_instance_id(db, instance_id)
+
+    async def insert_instance(
+        self,
+        db: AsyncSession,
+        instance_id: str,
+        instances: List
+    ) -> None:
+        """Insert operator instances"""
+        from app.db.models.cleaning import OperatorInstance as OperatorInstanceModel
+        
+        for idx, instance in enumerate(instances):
+            db_instance = OperatorInstanceModel(
+                instance_id=instance_id,
+                operator_id=instance.id,
+                op_index=idx,
+                settings_override=json.dumps(instance.overrides),
+            )
+            db.add(db_instance)
+        await db.flush()
+
+    async def delete_by_instance_id(self, db: AsyncSession, instance_id: str) -> None:
+        """Delete instances by instance ID"""
+        query = delete(self.model).where(self.model.instance_id == instance_id)
+        await db.execute(query)
+        await db.flush()