figmar/deepdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

deepdata: core product code (backend, frontend, runtime, deepflow module)

Browse Source
This commit is contained in:
figmar
2026-08-30 19:23:38 +08:00
commit bd3db30be9
1405 changed files with 271727 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/deepdata-python/app/module/operator/README.md
+138
View File
@@ -0,0 +1,138 @@
# Operator Market Service - Python Implementation
## 概述
这是 `operator-market-service` 的 Python 实现，已集成到 `runtime/deepdata-python` 项目中。
## 功能
- **算子管理**：创建、查询、更新、删除算子
- **分类管理**：树状分类结构查询
- **文件上传**：支持算子文件上传和解析（支持 tar/zip 格式）
- **MCP 工具集成**：通过 fastapi-mcp 提供 MCP 工具接口
## 目录结构
```
app/module/operator_market/
├── __init__.py # 模块入口
├── constants.py # 常量定义
├── exceptions.py # 异常定义
├── schema/ # Pydantic Schema 定义
│ ├── __init__.py
│ ├── operator.py # 算子相关 Schema
│ ├── category.py # 分类相关 Schema
│ └── release.py # 发布版本 Schema
├── parsers/ # 文件解析器
│ ├── __init__.py
│ ├── abstract_parser.py # 抽象解析器基类
│ ├── tar_parser.py # TAR 文件解析器
│ ├── zip_parser.py # ZIP 文件解析器
│ └── parser_holder.py # 解析器持有者
├── repository/ # 数据访问层
│ ├── __init__.py
│ ├── operator_repository.py
│ ├── category_repository.py
│ ├── category_relation_repository.py
│ └── operator_release_repository.py
├── service/ # 服务层
│ ├── __init__.py
│ ├── operator_service.py
│ └── category_service.py
└── interface/ # API 接口层
├── __init__.py
├── operator_routes.py
└── category_routes.py
```
## API 端点
### 算子相关 (`/api/operator-market/operators`)
| 方法 | 路径 | 描述 |
|------|--------|------|
| POST | `/list` | 查询算子列表（支持分页、分类过滤、关键词搜索） |
| GET | `/{operator_id}` | 获取算子详情 |
| PUT | `/{operator_id}` | 更新算子信息 |
| POST | `/create` | 创建新算子 |
| POST | `/upload` | 上传算子文件 |
| POST | `/upload/pre-upload` | 预上传（获取请求 ID） |
| POST | `/upload/chunk` | 分块上传 |
| DELETE | `/{operator_id}` | 删除算子 |
| GET | `/examples/download` | 下载示例算子 |
### 分类相关 (`/api/operator-market/categories`)
| 方法 | 路径 | 描述 |
|------|--------|------|
| GET | `/tree` | 获取分类树状结构 |
## 数据库表
- `t_operator` - 算子表
- `t_operator_category` - 分类表
- `t_operator_category_relation` - 分类关系表
- `t_operator_release` - 算子发布版本表
- `v_operator` - 算子视图（包含分类信息）
## 文件格式支持
算子文件需包含 `metadata.yml` 文件，格式如下：
```yaml
raw_id: "operator-id"
name: "算子名称"
description: "算子描述"
version: "1.0.0"
language: "python" # python, java
modal: "text" # text, image, audio, video
vendor: "deepdata" # deepdata, data-juicer, or other
inputs: {...}
outputs: {...}
runtime: {...}
settings: {...}
metrics: {...}
release:
- "更新日志1"
- "更新日志2"
```
## 待实现功能
- [ ] 算子收藏功能完善
- [ ] 标签过滤功能
## 使用示例
### 查询算子列表
```bash
curl -X POST "http://localhost:18000/api/operator-market/operators/list" \
-H "Content-Type: application/json" \
-d '{
"page": 1,
"size": 10,
"keyword": "test",
"isStar": false
}'
```
### 获取分类树
```bash
curl -X GET "http://localhost:18000/api/operator-market/categories/tree"
```
### 创建算子
```bash
curl -X POST "http://localhost:18000/api/operator-market/operators/create" \
-H "Content-Type: application/json" \
-d '{
"id": "new-operator-id",
"name": "新算子",
"description": "这是一个新算子",
"version": "1.0.0",
"fileName": "operator.tar"
}'
```
runtime/deepdata-python/app/module/operator/__init__.py
+4
View File
@@ -0,0 +1,4 @@
"""
Operator Market Service Module
算子市场服务模块
"""
runtime/deepdata-python/app/module/operator/constants.py
+54
View File
@@ -0,0 +1,54 @@
"""
Operator Market Constants
算子市场常量定义
"""
# Service ID
SERVICE_ID = "operator"
# YAML metadata path
YAML_PATH = "metadata.yml"
# Example operator file path
EXAMPLE_OPERATOR_PATH = "/app/test_operator.tar"
# Category IDs
CATEGORY_PYTHON = "python"
CATEGORY_PYTHON_ID = "9eda9d5d-072b-499b-916c-797a0a8750e1"
CATEGORY_JAVA = "java"
CATEGORY_JAVA_ID = "b5bfc548-8ef6-417c-b8a6-a4197c078249"
CATEGORY_CUSTOMIZED_ID = "ec2cdd17-8b93-4a81-88c4-ac9e98d10757"
CATEGORY_TEXT_ID = "d8a5df7a-52a9-42c2-83c4-01062e60f597"
CATEGORY_IMAGE_ID = "de36b61c-9e8a-4422-8c31-d30585c7100f"
CATEGORY_AUDIO_ID = "42dd9392-73e4-458c-81ff-41751ada47b5"
CATEGORY_VIDEO_ID = "a233d584-73c8-4188-ad5d-8f7c8dda9c27"
CATEGORY_ALL_ID = "4d7dbd77-0a92-44f3-9056-2cd62d4a71e4"
CATEGORY_STAR_ID = "51847c24-bba9-11f0-888b-5b143cb738aa"
CATEGORY_PREDEFINED_ID = "96a3b07a-3439-4557-a835-525faad60ca3"
CATEGORY_DEEPDATA_ID = "431e7798-5426-4e1a-aae6-b9905a836b34"
CATEGORY_DATA_JUICER_ID = "79b385b4-fde8-4617-bcba-02a176938996"
CATEGORY_OTHER_VENDOR_ID = "f00eaa3e-96c1-4de4-96cd-9848ef5429ec"
CATEGORY_CLEANING_ID = "8c09476a-a922-418f-a908-733f8a0de521"
CATEGORY_ANNOTATION_ID = "cfa9d8e2-5b5f-4f1e-9f12-1234567890ab"
# Category mapping
CATEGORY_MAP = {
CATEGORY_PYTHON: CATEGORY_PYTHON_ID,
CATEGORY_JAVA: CATEGORY_JAVA_ID,
"text": CATEGORY_TEXT_ID,
"image": CATEGORY_IMAGE_ID,
"audio": CATEGORY_AUDIO_ID,
"video": CATEGORY_VIDEO_ID,
"all": CATEGORY_ALL_ID,
"deepdata": CATEGORY_DEEPDATA_ID,
"data-juicer": CATEGORY_DATA_JUICER_ID,
"cleaning": CATEGORY_CLEANING_ID,
"annotation": CATEGORY_ANNOTATION_ID,
}
# File paths
OPERATOR_BASE_PATH = "/operators"
UPLOAD_DIR = "upload"
EXTRACT_DIR = "extract"
runtime/deepdata-python/app/module/operator/exceptions.py
+72
View File
@@ -0,0 +1,72 @@
"""
Operator Market Exceptions
算子市场异常定义
"""
from enum import Enum
from typing import Optional
class OperatorErrorCode:
"""算子错误码"""
def __init__(self, message: str, error_code: str):
self.message = message
self.error_code = error_code
class OperatorException(RuntimeError):
"""算子异常基类"""
def __init__(self, operator_error_code: OperatorErrorCode):
self.message = operator_error_code.message
self.error_code = operator_error_code.error_code
super().__init__(self.message)
class OperatorErrorCodeEnum(Enum):
"""算子错误码枚举"""
FIELD_NOT_FOUND = OperatorErrorCode(
"必填字段缺失", "OPERATOR_FIELD_NOT_FOUND"
)
SETTINGS_PARSE_FAILED = OperatorErrorCode(
"设置解析失败", "OPERATOR_SETTINGS_PARSE_FAILED"
)
OPERATOR_IN_INSTANCE = OperatorErrorCode(
"算子正在使用中", "OPERATOR_IN_INSTANCE"
)
CANT_DELETE_PREDEFINED_OPERATOR = OperatorErrorCode(
"无法删除预定义算子", "CANT_DELETE_PREDEFINED_OPERATOR"
)
class FieldNotFoundError(OperatorException):
"""必填字段缺失"""
def __init__(self, field_name: str):
super().__init__(
OperatorErrorCodeEnum.FIELD_NOT_FOUND.value
)
self.message = f"Required field '{field_name}' is missing"
self.field_name = field_name
class SettingsParseError(OperatorException):
"""设置解析失败"""
def __init__(self, detail: Optional[str] = None):
super().__init__(
OperatorErrorCodeEnum.SETTINGS_PARSE_FAILED.value
)
self.detail = detail
class OperatorInInstanceError(OperatorException):
"""算子正在使用中"""
def __init__(self):
super().__init__(
OperatorErrorCodeEnum.OPERATOR_IN_INSTANCE.value
)
class CannotDeletePredefinedOperatorError(OperatorException):
"""无法删除预定义算子"""
def __init__(self):
super().__init__(
OperatorErrorCodeEnum.CANT_DELETE_PREDEFINED_OPERATOR.value
)
runtime/deepdata-python/app/module/operator/interface/__init__.py
+9
View File
@@ -0,0 +1,9 @@
"""
Operator Market API Interfaces
算子市场 API 接口层
"""
from .operator_routes import router as operator_router
from .category_routes import router as category_router
__all__ = ["operator_router", "category_router"]
runtime/deepdata-python/app/module/operator/interface/category_routes.py
+65
View File
@@ -0,0 +1,65 @@
"""
Category API Routes
分类 API 路由
"""
from typing import Optional
from fastapi import APIRouter, Depends, Header
from app.db.models.operator import Category, CategoryRelation, Operator
from app.db.session import get_db
from app.module.operator.repository import (
CategoryRepository,
CategoryRelationRepository,
)
from app.module.operator.repository.operator_repository import OperatorRepository
from app.module.operator.schema import CategoryTreePagedResponse
from app.module.operator.schema.category import PaginatedCategoryTree
from app.module.operator.service import CategoryService
from app.module.shared.schema import StandardResponse
router = APIRouter(prefix="/categories", tags=["Category"])
def get_category_service() -> CategoryService:
"""获取分类服务实例"""
return CategoryService(
category_repo=CategoryRepository(Category()),
category_relation_repo=CategoryRelationRepository(CategoryRelation()),
operator_repo=OperatorRepository(Operator()),
)
@router.get(
"/tree",
response_model=StandardResponse[PaginatedCategoryTree],
summary="获取分类树",
description="获取算子树状分类结构，包含分组维度（如语言、模态）及资源统计数量。支持locale参数返回对应语言的分类名称.",
tags=["mcp"],
)
async def get_category_tree(
service: CategoryService = Depends(get_category_service),
db=Depends(get_db),
accept_language: Optional[str] = Header(None, alias="Accept-Language"),
):
"""获取分类树"""
# 解析语言参数
lang = None
if accept_language:
# Accept-Language: zh-CN,zh;q=0.0 -> 取 zh
lang = accept_language.split(",")[0].split("-")[0].strip()
result = await service.get_all_categories(db, locale=lang)
return StandardResponse(
code="0",
message="success",
data=PaginatedCategoryTree(
page=0,
size=len(result.categories),
total_elements=len(result.categories),
total_pages=1,
star_count=result.star_count,
content=result.categories,
),
)
runtime/deepdata-python/app/module/operator/interface/operator_routes.py
+256
View File
@@ -0,0 +1,256 @@
"""
Operator API Routes
算子 API 路由
"""
from typing import Optional
from fastapi import APIRouter, Depends, UploadFile, Form, File, Body, Header
from fastapi.responses import FileResponse
from app.core.logging import get_logger
from app.db.models.operator import Operator, CategoryRelation, OperatorRelease
from app.db.session import get_db
from app.module.operator.parsers import ParserHolder
from app.module.operator.repository import (
OperatorRepository,
CategoryRelationRepository,
OperatorReleaseRepository,
)
from app.module.operator.schema import (
OperatorDto,
OperatorUpdateDto,
OperatorListRequest,
)
from app.module.operator.service import OperatorService
from app.module.shared.chunk_upload_repository import ChunkUploadRepository
from app.module.shared.file_service import FileService
from app.module.shared.schema import StandardResponse, PaginatedData
logger = get_logger(__name__)
router = APIRouter(prefix="/operators", tags=["Operator"])
def get_operator_service() -> OperatorService:
"""获取算子服务实例"""
return OperatorService(
operator_repo=OperatorRepository(Operator()),
category_relation_repo=CategoryRelationRepository(CategoryRelation()),
operator_release_repo=OperatorReleaseRepository(OperatorRelease()),
parser_holder=ParserHolder(),
file_service=FileService(ChunkUploadRepository()),
)
@router.post(
"/list",
response_model=StandardResponse[PaginatedData[OperatorDto]],
summary="查询算子列表",
description="根据参数查询算子列表（支持分页、分类过滤、关键词搜索）",
tags=['mcp']
)
async def list_operators(
request: OperatorListRequest,
service: OperatorService = Depends(get_operator_service),
db = Depends(get_db),
):
"""查询算子列表"""
operators = await service.get_operators(
page=request.page,
size=request.size,
categories=request.categories,
keyword=request.keyword,
is_star=request.is_star,
db=db
)
count = await service.count_operators(
categories=request.categories,
keyword=request.keyword,
is_star=request.is_star,
db=db
)
total_pages = (count + request.size - 1) // request.size
return StandardResponse(
code="0",
message="success",
data=PaginatedData(
page=request.page,
size=request.size,
total_elements=count,
total_pages=total_pages,
content=operators,
)
)
@router.get(
"/{operator_id}",
response_model=StandardResponse[OperatorDto],
summary="获取算子详情",
description="根据 ID 获取算子详细信息"
)
async def get_operator(
operator_id: str,
service: OperatorService = Depends(get_operator_service),
db = Depends(get_db),
accept_language: Optional[str] = Header(None, alias="Accept-Language"),
):
"""获取算子详情"""
# 解析语言参数
lang = None
if accept_language:
# Accept-Language: zh-CN,zh;q=0.0 -> 取 zh
lang = accept_language.split(",")[0].split("-")[0].strip()
operator = await service.get_operator_by_id(operator_id, db, locale=lang)
operator.file_name = None
return StandardResponse(code="0", message="success", data=operator)
@router.put(
"/{operator_id}",
response_model=StandardResponse[OperatorDto],
summary="更新算子",
description="更新算子信息"
)
async def update_operator(
operator_id: str,
request: OperatorUpdateDto,
service: OperatorService = Depends(get_operator_service),
db = Depends(get_db)
):
"""更新算子"""
operator = await service.update_operator(operator_id, request, db)
await db.commit()
return StandardResponse(code="0", message="success", data=operator)
@router.post(
"/create",
response_model=StandardResponse[OperatorDto],
summary="创建算子",
description="创建新算子"
)
async def create_operator(
request: OperatorDto,
service: OperatorService = Depends(get_operator_service),
db = Depends(get_db)
):
"""创建算子"""
operator = await service.create_operator(request, db)
await db.commit()
return StandardResponse(code="0", message="success", data=operator)
@router.post(
"/upload",
response_model=StandardResponse[OperatorDto],
summary="上传算子",
description="上传算子文件并解析元数据"
)
async def upload_operator(
request: dict = Body(...),
service: OperatorService = Depends(get_operator_service),
db = Depends(get_db),
):
"""上传算子"""
file_name = request.get("fileName")
if not file_name:
from fastapi import HTTPException
raise HTTPException(status_code=422, detail="fileName is required")
operator = await service.upload_operator(file_name, db)
return StandardResponse(code="0", message="success", data=operator)
@router.post(
"/upload/pre-upload",
response_model=StandardResponse[str],
summary="预上传",
description="获取预上传 ID，用于分块上传"
)
async def pre_upload(
service: OperatorService = Depends(get_operator_service),
db = Depends(get_db),
):
"""预上传"""
req_id = await service.pre_upload(db)
await db.commit()
return StandardResponse(
code="0",
message="success",
data=req_id,
)
@router.post(
"/upload/chunk",
response_model=StandardResponse[dict],
summary="分块上传",
description="分块上传算子文件"
)
async def chunk_upload(
req_id: str = Form(..., alias="reqId", description="预上传ID"),
file_no: int = Form(1, alias="fileNo", description="文件编号"),
file_name: str = Form(..., alias="fileName", description="文件名"),
total_chunk_num: int = Form(1, alias="totalChunkNum", description="总分块数"),
chunk_no: int = Form(1, alias="chunkNo", description="当前分块号"),
file: UploadFile = File(...),
check_sum_hex: Optional[str] = Form(None, alias="checkSumHex", description="校验和"),
service: OperatorService = Depends(get_operator_service),
db = Depends(get_db),
):
"""分块上传"""
file_content = await file.read()
result = await service.chunk_upload(
req_id=req_id,
file_no=file_no,
file_name=file_name,
total_chunk_num=total_chunk_num,
chunk_no=chunk_no,
check_sum_hex=check_sum_hex,
file_content=file_content,
db=db
)
await db.commit()
return StandardResponse(code="0", message="success", data=result.dict())
@router.delete(
"/{operator_id}",
response_model=StandardResponse[None],
summary="删除算子",
description="删除算子"
)
async def delete_operator(
operator_id: str,
service: OperatorService = Depends(get_operator_service),
db = Depends(get_db),
):
"""删除算子"""
await service.delete_operator(operator_id, db)
await db.commit()
return StandardResponse(code="0", message="success", data=None)
@router.get(
"/examples/download",
response_class=FileResponse,
summary="下载示例算子",
description="下载示例算子文件"
)
async def download_example_operator(
service: OperatorService = Depends(get_operator_service),
):
"""下载示例算子"""
from app.module.operator.constants import EXAMPLE_OPERATOR_PATH
example_path = EXAMPLE_OPERATOR_PATH
file_path = service.download_example_operator(example_path)
return FileResponse(
path=str(file_path),
filename=file_path.name,
media_type="application/octet-stream"
)
runtime/deepdata-python/app/module/operator/parsers/__init__.py
+15
View File
@@ -0,0 +1,15 @@
"""
Operator File Parsers
算子文件解析器
"""
from .abstract_parser import AbstractParser
from .tar_parser import TarParser
from .zip_parser import ZipParser
from .parser_holder import ParserHolder
__all__ = [
"AbstractParser",
"TarParser",
"ZipParser",
"ParserHolder",
]
runtime/deepdata-python/app/module/operator/parsers/abstract_parser.py
+129
View File
@@ -0,0 +1,129 @@
"""
Abstract Parser
抽象解析器基类
"""
import json
import yaml
from abc import ABC, abstractmethod
from typing import Dict, Any, Optional
from app.module.operator.schema import OperatorDto, OperatorReleaseDto
from app.module.operator.constants import CATEGORY_MAP, CATEGORY_OTHER_VENDOR_ID, CATEGORY_CUSTOMIZED_ID, \
CATEGORY_CLEANING_ID
from app.module.operator.exceptions import FieldNotFoundError
class AbstractParser(ABC):
"""算子文件解析器抽象基类"""
@abstractmethod
def parse_yaml_from_archive(
self,
archive_path: str,
entry_path: str,
file_name: Optional[str] = None,
file_size: Optional[int] = None
) -> OperatorDto:
"""
从压缩包内读取指定路径的 yaml 文件并解析为 OperatorDto
Args:
archive_path: 压缩包路径（zip 或 tar）
entry_path: 压缩包内部的文件路径，例如 "config/app.yaml"
Returns:
解析后的 OperatorDto
"""
pass
@abstractmethod
def extract_to(self, archive_path: str, target_dir: str) -> None:
"""
将压缩包解压到目标目录（保持相对路径）
Args:
archive_path: 压缩包路径
target_dir: 目标目录
"""
pass
def parse_yaml(
self,
yaml_content: str,
file_name: Optional[str] = None,
file_size: Optional[int] = None
) -> OperatorDto:
"""解析 YAML 内容为 OperatorDto"""
content: Dict[str, Any] = yaml.safe_load(yaml_content)
operator = OperatorDto(
id=self._to_string(content.get("raw_id")),
name=self._to_string(content.get("name")),
description=self._to_string(content.get("description")),
version=self._to_string(content.get("version")),
inputs=self._to_json(content.get("inputs")),
outputs=self._to_json(content.get("outputs")),
runtime=self._to_json(content.get("runtime")),
settings=self._to_json(content.get("settings")),
metrics=self._to_json(content.get("metrics")),
file_name=file_name,
file_size=file_size,
)
# Handle changelog
changelog = content.get("release")
if isinstance(changelog, list):
operator_release = OperatorReleaseDto(
id=operator.id,
version=operator.version,
changelog=changelog
)
else:
operator_release = OperatorReleaseDto(
id=operator.id,
version=operator.version,
changelog=[]
)
operator.releases = [operator_release]
# Build categories
categories = []
types = content.get("types")
if isinstance(types, list):
for t in types:
if self._to_lower(t) in CATEGORY_MAP:
categories.append(CATEGORY_MAP[self._to_lower(t)])
if len(categories) == 0:
categories.append(CATEGORY_CLEANING_ID)
categories.extend([
CATEGORY_MAP.get(self._to_lower(content.get("language")), ""),
CATEGORY_MAP.get(self._to_lower(content.get("modal")), ""),
CATEGORY_MAP.get(self._to_lower(content.get("vendor")), CATEGORY_OTHER_VENDOR_ID),
CATEGORY_CUSTOMIZED_ID,
])
operator.categories = categories
return operator
def _to_string(self, obj: Any) -> str:
"""转换为字符串"""
if obj is None:
raise FieldNotFoundError("field")
return str(obj)
def _to_lower(self, obj: Any) -> str:
"""转换为小写字符串"""
if obj is None:
raise FieldNotFoundError("field")
return str(obj).lower()
def _to_json(self, obj: Any) -> Optional[str]:
"""转换为 JSON 字符串"""
if obj is None:
return None
try:
return json.dumps(obj).strip('"').strip("'")
except (TypeError, ValueError) as e:
raise ValueError(f"Failed to serialize to JSON: {e}")
runtime/deepdata-python/app/module/operator/parsers/parser_holder.py
+59
View File
@@ -0,0 +1,59 @@
"""
Parser Holder
解析器持有者，根据文件类型选择合适的解析器
"""
import os
from typing import Dict, Type, Optional
from app.module.operator.parsers.abstract_parser import AbstractParser
from app.module.operator.parsers.tar_parser import TarParser
from app.module.operator.parsers.zip_parser import ZipParser
from app.module.operator.schema import OperatorDto
class ParserHolder:
"""解析器持有者，根据文件类型选择解析器"""
def __init__(self):
self._parsers: Dict[str, AbstractParser] = {
"tar": TarParser(),
"gz": TarParser(),
"tgz": TarParser(),
"zip": ZipParser(),
}
def get_parser(self, file_path: str) -> AbstractParser:
"""根据文件扩展名获取解析器"""
_, ext = os.path.splitext(file_path)
file_type = ext.lstrip('.').lower()
if file_type not in self._parsers:
raise ValueError(f"Unsupported file type: {file_type}")
return self._parsers[file_type]
def parse_yaml_from_archive(
self,
file_type: str,
archive_path: str,
entry_path: str,
file_name: Optional[str] = None,
file_size: Optional[int] = None
) -> OperatorDto:
"""从压缩包解析 YAML"""
if file_type not in self._parsers:
raise ValueError(f"Unsupported file type: {file_type}")
return self._parsers[file_type].parse_yaml_from_archive(
archive_path,
entry_path,
file_name,
file_size
)
def extract_to(self, file_type: str, archive_path: str, target_dir: str) -> None:
"""解压文件到目标目录"""
if file_type not in self._parsers:
raise ValueError(f"Unsupported file type: {file_type}")
self._parsers[file_type].extract_to(archive_path, target_dir)
runtime/deepdata-python/app/module/operator/parsers/tar_parser.py
+47
View File
@@ -0,0 +1,47 @@
"""
Tar File Parser
TAR 文件解析器
"""
import tarfile
import os
from typing import Optional
from app.module.operator.parsers.abstract_parser import AbstractParser
from app.module.operator.schema import OperatorDto
class TarParser(AbstractParser):
"""TAR 压缩包解析器"""
def parse_yaml_from_archive(
self,
archive_path: str,
entry_path: str,
file_name: Optional[str] = None,
file_size: Optional[int] = None
) -> OperatorDto:
"""从 TAR 文件中解析 YAML"""
try:
with tarfile.open(archive_path, 'r:*') as tar:
for member in tar.getmembers():
if member.name == entry_path or member.name.endswith(f"/{entry_path}"):
file = tar.extractfile(member)
if file:
content = file.read().decode('utf-8')
return self.parse_yaml(content, file_name, file_size)
raise FileNotFoundError(f"File '{entry_path}' not found in archive")
except (tarfile.TarError, EOFError) as e:
raise ValueError(f"Failed to parse TAR file: {e}")
def extract_to(self, archive_path: str, target_dir: str) -> None:
"""解压 TAR 文件到目标目录"""
try:
os.makedirs(target_dir, exist_ok=True)
with tarfile.open(archive_path, 'r:*') as tar:
# Safety check: prevent path traversal
for member in tar.getmembers():
if os.path.isabs(member.name) or ".." in member.name.split("/"):
raise ValueError(f"Unsafe path in archive: {member.name}")
tar.extractall(target_dir)
except (tarfile.TarError, EOFError) as e:
raise ValueError(f"Failed to extract TAR file: {e}")
runtime/deepdata-python/app/module/operator/parsers/zip_parser.py
+46
View File
@@ -0,0 +1,46 @@
"""
Zip File Parser
ZIP 文件解析器
"""
import zipfile
import os
from typing import Optional
from app.module.operator.parsers.abstract_parser import AbstractParser
from app.module.operator.schema import OperatorDto
class ZipParser(AbstractParser):
"""ZIP 压缩包解析器"""
def parse_yaml_from_archive(
self,
archive_path: str,
entry_path: str,
file_name: Optional[str] = None,
file_size: Optional[int] = None
) -> OperatorDto:
"""从 ZIP 文件中解析 YAML"""
try:
with zipfile.ZipFile(archive_path, 'r') as zf:
for name in zf.namelist():
if name == entry_path or name.endswith(f"/{entry_path}"):
with zf.open(name) as file:
content = file.read().decode('utf-8')
return self.parse_yaml(content, file_name, file_size)
raise FileNotFoundError(f"File '{entry_path}' not found in archive")
except (zipfile.BadZipFile, zipfile.LargeZipFile) as e:
raise ValueError(f"Failed to parse ZIP file: {e}")
def extract_to(self, archive_path: str, target_dir: str) -> None:
"""解压 ZIP 文件到目标目录"""
try:
os.makedirs(target_dir, exist_ok=True)
with zipfile.ZipFile(archive_path, 'r') as zf:
# Safety check: prevent path traversal
for name in zf.namelist():
if os.path.isabs(name) or ".." in name.split("/"):
raise ValueError(f"Unsafe path in archive: {name}")
zf.extractall(target_dir)
except (zipfile.BadZipFile, zipfile.LargeZipFile) as e:
raise ValueError(f"Failed to extract ZIP file: {e}")
runtime/deepdata-python/app/module/operator/repository/__init__.py
+15
View File
@@ -0,0 +1,15 @@
"""
Operator Market Repositories
算子市场数据访问层
"""
from .operator_repository import OperatorRepository
from .category_repository import CategoryRepository
from .category_relation_repository import CategoryRelationRepository
from .operator_release_repository import OperatorReleaseRepository
__all__ = [
"OperatorRepository",
"CategoryRepository",
"CategoryRelationRepository",
"OperatorReleaseRepository",
]
runtime/deepdata-python/app/module/operator/repository/category_relation_repository.py
+77
View File
@@ -0,0 +1,77 @@
"""
Category Relation Repository
分类关系数据访问层
"""
from typing import List
from sqlalchemy import select, delete, and_
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models.operator import CategoryRelation
from app.module.operator.constants import CATEGORY_PREDEFINED_ID
class CategoryRelationRepository:
"""分类关系数据访问层"""
def __init__(self, model: CategoryRelation):
self.model = model
async def find_all(self, db: AsyncSession) -> List[CategoryRelation]:
"""查询所有分类关系"""
result = await db.execute(select(CategoryRelation))
return result.scalars().all()
async def batch_insert(
self,
operator_id: str,
category_ids: List[str],
db: AsyncSession
) -> None:
"""批量插入分类关系"""
for category_id in category_ids:
entity = CategoryRelation(
category_id=category_id,
operator_id=operator_id
)
db.add(entity)
async def batch_update(
self,
operator_id: str,
category_ids: List[str],
db: AsyncSession
) -> None:
"""批量更新分类关系（先删除后插入）"""
# Delete existing relations
await db.execute(
delete(CategoryRelation)
.where(CategoryRelation.operator_id == operator_id)
)
# Insert new relations
for category_id in category_ids:
entity = CategoryRelation(
category_id=category_id,
operator_id=operator_id
)
db.add(entity)
async def delete_by_operator_id(self, operator_id: str, db: AsyncSession) -> None:
"""根据算子ID删除分类关系"""
await db.execute(
delete(CategoryRelation)
.where(CategoryRelation.operator_id == operator_id)
)
async def operator_is_predefined(self, operator_id: str, db: AsyncSession) -> bool:
"""检查算子是否为预定义算子"""
result = await db.execute(
select(CategoryRelation)
.where(
and_(
CategoryRelation.operator_id == operator_id,
CategoryRelation.category_id == CATEGORY_PREDEFINED_ID
)
)
)
return result.first() is not None
runtime/deepdata-python/app/module/operator/repository/category_repository.py
+23
View File
@@ -0,0 +1,23 @@
"""
Category Repository
分类数据访问层
"""
from typing import List
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models.operator import Category
from app.module.operator.schema import CategoryDto
class CategoryRepository:
"""分类数据访问层"""
def __init__(self, model: Category):
self.model = model
async def find_all(self, db: AsyncSession) -> List[Category]:
"""查询所有分类"""
result = await db.execute(select(Category))
return result.scalars().all()
runtime/deepdata-python/app/module/operator/repository/operator_release_repository.py
+72
View File
@@ -0,0 +1,72 @@
"""
Operator Release Repository
算子发布版本数据访问层
"""
from typing import List
from sqlalchemy import select, delete, and_
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models.operator import OperatorRelease
from app.module.operator.schema import OperatorReleaseDto
class OperatorReleaseRepository:
"""算子发布版本数据访问层"""
def __init__(self, model: OperatorRelease):
self.model = model
async def find_all_by_operator_id(
self,
operator_id: str,
db: AsyncSession
) -> List[OperatorRelease]:
"""查询算子的所有发布版本"""
result = await db.execute(
select(OperatorRelease)
.where(OperatorRelease.id == operator_id)
.order_by(OperatorRelease.release_date.desc())
)
return result.scalars().all()
async def insert(
self,
dto: OperatorReleaseDto,
db: AsyncSession
) -> None:
"""插入发布版本"""
entity = OperatorRelease(
id=dto.id,
version=dto.version,
release_date=dto.release_date,
changelog=dto.changelog
)
db.add(entity)
async def update(
self,
dto: OperatorReleaseDto,
db: AsyncSession
) -> None:
"""更新发布版本"""
result = await db.execute(
select(OperatorRelease)
.where(
and_(
OperatorRelease.id == dto.id,
OperatorRelease.version == dto.version
)
)
)
entity = result.scalar_one_or_none()
if entity:
entity.changelog = dto.changelog
entity.release_date = dto.release_date
async def delete(self, operator_id: str, db: AsyncSession) -> None:
"""删除算子的所有发布版本"""
await db.execute(
delete(OperatorRelease)
.where(OperatorRelease.id == operator_id)
)
runtime/deepdata-python/app/module/operator/repository/operator_repository.py
+121
View File
@@ -0,0 +1,121 @@
"""
Operator Repository
算子数据访问层
"""
import json
from typing import List, Optional
from datetime import datetime, timezone
from sqlalchemy import select, text, update, func
from sqlalchemy.ext.asyncio import AsyncSession
from app.db.models.operator import Operator
from app.module.operator.schema import OperatorDto
class OperatorRepository:
"""算子数据访问层"""
def __init__(self, model: Operator):
self.model = model
async def find_all(self, db: AsyncSession) -> List[Operator]:
"""查询所有算子"""
result = await db.execute(select(Operator))
return result.scalars().all()
async def insert(self, dto: OperatorDto, db: AsyncSession) -> None:
"""插入算子"""
entity = Operator(
id=dto.id,
name=dto.name,
description=dto.description,
version=dto.version,
inputs=dto.inputs,
outputs=dto.outputs,
runtime=dto.runtime,
settings=dto.settings,
file_name=dto.file_name,
file_size=dto.file_size,
metrics=dto.metrics,
usage_count=dto.usage_count or 0,
is_star=dto.is_star or False,
)
db.add(entity)
async def update(self, dto: OperatorDto, db: AsyncSession) -> None:
"""更新算子"""
await db.execute(
update(Operator)
.where(Operator.id == dto.id)
.values(
name=dto.name,
description=dto.description,
version=dto.version,
inputs=dto.inputs,
outputs=dto.outputs,
runtime=dto.runtime,
settings=dto.settings,
file_name=dto.file_name,
file_size=dto.file_size,
metrics=dto.metrics,
is_star=dto.is_star,
updated_at=datetime.utcnow(),
)
)
async def delete(self, operator_id: str, db: AsyncSession) -> None:
"""删除算子"""
entity = await db.get(Operator, operator_id)
if entity:
await db.delete(entity)
async def count_by_star(self, is_star: bool, db: AsyncSession) -> int:
"""统计收藏算子数量"""
result = await db.execute(
select(text("COUNT(*)"))
.select_from(Operator)
.where(Operator.is_star == is_star)
)
return result.scalar() or 0
async def operator_in_template(self, operator_id: str, db: AsyncSession) -> bool:
"""检查算子是否在模板中"""
result = await db.execute(
text("""
SELECT COUNT(*) FROM t_operator_instance oi
JOIN t_clean_template t ON oi.instance_id = t.id
WHERE oi.operator_id = :operator_id
"""),
{"operator_id": operator_id}
)
return (result.scalar() or 0) > 0
async def operator_in_unstop_task(self, operator_id: str, db: AsyncSession) -> bool:
"""检查算子是否在未完成的任务中"""
result = await db.execute(
text("""
SELECT COUNT(*) FROM t_operator_instance oi
JOIN t_clean_task t ON oi.instance_id = t.id
WHERE oi.operator_id = :operator_id AND t.status != 'COMPLETED'
"""),
{"operator_id": operator_id}
)
return (result.scalar() or 0) > 0
async def increment_usage_count(
self,
operator_ids: List[str],
db: AsyncSession
) -> None:
"""增加算子使用次数"""
if not operator_ids:
return
await db.execute(
update(Operator)
.where(Operator.id.in_(operator_ids))
.values(
usage_count=func.coalesce(Operator.usage_count, 0) + 1,
updated_at=datetime.utcnow(),
)
)
runtime/deepdata-python/app/module/operator/schema/__init__.py
+29
View File
@@ -0,0 +1,29 @@
"""
Operator Market Schemas
算子市场 Schema 定义
"""
from .operator import (
OperatorDto,
OperatorListRequest,
PreUploadResponse,
OperatorUpdateDto,
)
from .category import (
CategoryDto,
CategoryTreeResponse,
CategoryTreePagedResponse,
CategoryRelationDto,
)
from .release import OperatorReleaseDto
__all__ = [
"OperatorDto",
"OperatorListRequest",
"PreUploadResponse",
"CategoryDto",
"CategoryTreeResponse",
"CategoryTreePagedResponse",
"CategoryRelationDto",
"OperatorReleaseDto",
"OperatorUpdateDto",
]
runtime/deepdata-python/app/module/operator/schema/category.py
+44
View File
@@ -0,0 +1,44 @@
"""
Category Schemas
分类 Schema 定义
"""
from typing import List, Optional
from datetime import datetime
from pydantic import BaseModel, Field
from app.module.shared.schema import BaseResponseModel, PaginatedData
class CategoryDto(BaseResponseModel):
"""分类 DTO"""
id: str = Field(..., description="分类ID")
name: str = Field(..., description="分类名称")
value: Optional[str] = Field(None, description="分类值")
type: Optional[str] = Field(None, description="分类类型")
parent_id: Optional[str] = Field(None, description="父分类ID")
count: Optional[int] = Field(0, description="算子数量")
created_at: Optional[datetime] = Field(None, description="创建时间")
class CategoryTreeResponse(BaseResponseModel):
"""分类树响应"""
id: str = Field(..., description="分类ID")
name: str = Field(..., description="分类名称")
count: int = Field(0, description="算子总数")
categories: List[CategoryDto] = Field(default_factory=list, description="子分类列表")
class CategoryTreePagedResponse(BaseResponseModel):
"""分类树分页响应"""
star_count: int = Field(0, description="收藏的算子数量")
categories: List[CategoryTreeResponse] = Field(default_factory=list, description="分类树列表")
class PaginatedCategoryTree(PaginatedData):
star_count: int = Field(0, description="收藏的算子数量")
class CategoryRelationDto(BaseResponseModel):
"""分类关系 DTO"""
category_id: str = Field(..., description="分类ID")
operator_id: str = Field(..., description="算子ID")
runtime/deepdata-python/app/module/operator/schema/operator.py
+72
View File
@@ -0,0 +1,72 @@
"""
Operator Schemas
算子 Schema 定义
"""
from __future__ import annotations
from typing import List, Optional, Dict, Any
from datetime import datetime
from pydantic import BaseModel, Field
from app.module.shared.schema import BaseResponseModel
from .release import OperatorReleaseDto
class OperatorDto(BaseResponseModel):
"""算子 DTO"""
id: str = Field(..., description="算子ID")
name: str = Field(..., description="算子名称")
description: Optional[str] = Field(None, description="算子描述")
version: str = Field(..., description="算子版本")
inputs: Optional[str] = Field(None, description="输入定义（JSON）")
outputs: Optional[str] = Field(None, description="输出定义（JSON）")
runtime: Optional[str] = Field(None, description="运行时配置（JSON）")
settings: Optional[str] = Field(None, description="算子设置（JSON）")
file_name: Optional[str] = Field(None, description="文件名")
file_size: Optional[int] = Field(None, description="文件大小（字节）")
metrics: Optional[str] = Field(None, description="算子指标（JSON）")
usage_count: Optional[int] = Field(None, description="使用次数")
is_star: Optional[bool] = Field(None, description="是否收藏")
categories: Optional[List[str]] = Field(None, description="分类ID列表")
overrides: Optional[Dict[str, Any]] = Field(None, description="设置覆盖值")
requirements: Optional[List[str]] = Field(None, description="Python 依赖列表")
readme: Optional[str] = Field(None, description="README 内容")
releases: Optional[List[OperatorReleaseDto]] = Field(None, description="发布版本列表")
created_at: Optional[datetime] = Field(None, description="创建时间")
updated_at: Optional[datetime] = Field(None, description="更新时间")
class OperatorListRequest(BaseResponseModel):
"""算子列表查询请求"""
page: int = Field(1, ge=0, description="页码（从0开始）")
size: int = Field(10, ge=1, description="页大小")
categories: List[List[str]] = Field(default_factory=list, description="分类ID列表（每个父分类下的id放到一个列表中）")
keyword: Optional[str] = Field(None, description="搜索关键词")
label_name: Optional[str] = Field(None, description="标签名称（暂不支持）")
is_star: Optional[bool] = Field(None, description="是否收藏")
class PreUploadResponse(BaseResponseModel):
"""预上传响应"""
req_id: str = Field(..., description="请求ID")
class OperatorUpdateDto(BaseResponseModel):
"""算子更新 DTO（所有字段可选）"""
name: Optional[str] = Field(None, description="算子名称")
description: Optional[str] = Field(None, description="算子描述")
version: Optional[str] = Field(None, description="算子版本")
inputs: Optional[str] = Field(None, description="输入定义（JSON）")
outputs: Optional[str] = Field(None, description="输出定义（JSON）")
runtime: Optional[str] = Field(None, description="运行时配置（JSON）")
settings: Optional[str] = Field(None, description="算子设置（JSON）")
file_name: Optional[str] = Field(None, description="文件名")
file_size: Optional[int] = Field(None, description="文件大小（字节）")
metrics: Optional[str] = Field(None, description="算子指标（JSON）")
usage_count: Optional[int] = Field(None, description="使用次数")
is_star: Optional[bool] = Field(None, description="是否收藏")
categories: Optional[List[str]] = Field(None, description="分类ID列表")
overrides: Optional[Dict[str, Any]] = Field(None, description="设置覆盖值")
requirements: Optional[List[str]] = Field(None, description="Python 依赖列表")
readme: Optional[str] = Field(None, description="README 内容")
releases: Optional[List[OperatorReleaseDto]] = Field(None, description="发布版本列表")
runtime/deepdata-python/app/module/operator/schema/release.py
+22
View File
@@ -0,0 +1,22 @@
"""
Operator Release Schemas
算子发布版本 Schema 定义
"""
from __future__ import annotations
from typing import List, Optional
from datetime import datetime
from pydantic import BaseModel, Field
from app.module.shared.schema import BaseResponseModel
class OperatorReleaseDto(BaseResponseModel):
"""算子发布版本 DTO"""
id: str = Field(..., description="算子ID")
version: str = Field(..., description="版本号")
release_date: Optional[datetime] = Field(None, description="发布时间")
changelog: Optional[List[str]] = Field(None, description="更新日志列表")
__all__ = ["OperatorReleaseDto"]
runtime/deepdata-python/app/module/operator/service/__init__.py
+11
View File
@@ -0,0 +1,11 @@
"""
Operator Market Services
算子市场服务层
"""
from .operator_service import OperatorService
from .category_service import CategoryService
__all__ = [
"OperatorService",
"CategoryService",
]
runtime/deepdata-python/app/module/operator/service/category_service.py
+110
View File
@@ -0,0 +1,110 @@
"""
Category Service
分类服务层
"""
from typing import List, Optional
from sqlalchemy.ext.asyncio import AsyncSession
from app.module.operator.repository import (
CategoryRepository,
CategoryRelationRepository,
)
from app.module.operator.schema import (
CategoryDto,
CategoryTreeResponse,
CategoryTreePagedResponse,
)
from app.db.models.operator import Operator
from app.module.operator.repository.operator_repository import OperatorRepository
class CategoryService:
"""分类服务"""
def __init__(
self,
category_repo: CategoryRepository,
category_relation_repo: CategoryRelationRepository,
operator_repo: OperatorRepository,
):
self.category_repo = category_repo
self.category_relation_repo = category_relation_repo
self.operator_repo = operator_repo
async def get_all_categories(
self, db: AsyncSession, locale: Optional[str] = None
) -> CategoryTreePagedResponse:
"""获取所有分类（树状结构）
Args:
db: 数据库会话
locale: 语言代码 (zh/en)，None 时返回中文
"""
# Helper function to get localized name
def get_localized_name(cat) -> str:
if locale == "en" and getattr(cat, "name_en", None):
return cat.name_en
return cat.name
# Get all categories
all_categories = await self.category_repo.find_all(db)
category_map = {c.id: c for c in all_categories}
# Get all relations and count operators per category
all_relations = await self.category_relation_repo.find_all(db)
relation_map = {}
for rel in all_relations:
if rel.category_id not in relation_map:
relation_map[rel.category_id] = 0
relation_map[rel.category_id] += 1
# Group by parent_id
grouped_by_parent = {}
for cat in all_categories:
if cat.parent_id != "0":
if cat.parent_id not in grouped_by_parent:
grouped_by_parent[cat.parent_id] = []
grouped_by_parent[cat.parent_id].append(cat)
# Build category trees
parent_ids = sorted(grouped_by_parent.keys(), key=lambda pid: pid)
category_trees = []
for parent_id in parent_ids:
group = grouped_by_parent[parent_id]
parent_category = category_map[parent_id]
# Build DTOs for children
child_dtos = []
total_count = 0
for cat in sorted(group, key=lambda c: c.created_at or 0):
cat_dto = CategoryDto(
id=cat.id,
name=get_localized_name(cat),
value=cat.value,
type=cat.type,
parent_id=cat.parent_id,
count=relation_map.get(cat.id, 0),
created_at=cat.created_at,
)
child_dtos.append(cat_dto)
total_count += cat_dto.count
tree = CategoryTreeResponse(
id=parent_id,
name=get_localized_name(parent_category),
count=total_count,
categories=child_dtos,
)
category_trees.append(tree)
# Get star count
star_count = await self.operator_repo.count_by_star(True, db)
return CategoryTreePagedResponse(
star_count=star_count,
categories=category_trees,
)
runtime/deepdata-python/app/module/operator/service/operator_service.py
+637
View File
@@ -0,0 +1,637 @@
"""
Operator Service
算子服务层
"""
import json
import os
import uuid
import shutil
from datetime import datetime
from pathlib import Path
from typing import List, Optional, Dict, Any, TYPE_CHECKING
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, text, func
from app.core.logging import get_logger
from app.core.exception import BusinessError, ErrorCodes
from app.module.operator.repository import (
OperatorRepository,
CategoryRelationRepository,
OperatorReleaseRepository,
)
from app.module.operator.schema import (
OperatorDto,
OperatorUpdateDto,
OperatorReleaseDto,
)
from app.module.operator.parsers import ParserHolder
from app.module.operator.constants import (
OPERATOR_BASE_PATH,
UPLOAD_DIR,
EXTRACT_DIR,
YAML_PATH,
SERVICE_ID,
)
from app.module.shared.file_service import FileService
from app.module.shared.file_models import (
ChunkUploadRequestDto,
FileUploadResult,
)
logger = get_logger(__name__)
class OperatorService:
"""算子服务"""
def __init__(
self,
operator_repo: OperatorRepository,
category_relation_repo: CategoryRelationRepository,
operator_release_repo: OperatorReleaseRepository,
parser_holder: ParserHolder,
file_service: FileService,
):
self.operator_repo = operator_repo
self.category_relation_repo = category_relation_repo
self.operator_release_repo = operator_release_repo
self.parser_holder = parser_holder
self.file_service = file_service
async def get_operators(
self,
page: int,
size: int,
categories: List[List[str]],
keyword: Optional[str],
is_star: Optional[bool],
db: AsyncSession
) -> List[OperatorDto]:
"""查询算子列表（分页）"""
offset = page * size
# Build query with categories filter
conditions = []
params = {"limit": size, "offset": offset}
if is_star is not None:
conditions.append("ov.is_star = :is_star")
params["is_star"] = is_star
if keyword:
conditions.append(
"(ov.operator_name ILIKE :keyword OR ov.description ILIKE :keyword)"
)
params["keyword"] = f"%{keyword}%"
where_clause = ""
if conditions:
where_clause = "WHERE " + " AND ".join(conditions)
# Handle categories grouping
group_by = "GROUP BY ov.operator_id, ov.operator_name, ov.description, ov.version, " \
"ov.inputs, ov.outputs, ov.runtime, ov.settings, ov.is_star, " \
"ov.file_size, ov.usage_count, ov.created_at, ov.updated_at, ov.created_by, ov.updated_by"
having_clause = ""
if categories:
# Flatten all category IDs for IN clause
all_category_ids = [cat_id for sublist in categories for cat_id in sublist]
if all_category_ids:
where_clause += " AND category_id = ANY(:category_ids)" if where_clause else "WHERE category_id = ANY(:category_ids)"
params["category_ids"] = all_category_ids
# Build HAVING clause for category groups
having_clauses = []
for i, cat_group in enumerate(categories):
cat_list = ", ".join([f"'{cat_id}'" for cat_id in cat_group])
having_clauses.append(
f"SUM(CASE WHEN category_id IN ({cat_list}) THEN 1 ELSE 0 END) > 0"
)
having_clause = "HAVING " + " AND ".join(having_clauses)
query = f"""
SELECT
ov.operator_id AS id,
ov.operator_name AS name,
ov.description,
ov.version,
ov.inputs,
ov.outputs,
ov.runtime,
ov.settings,
ov.is_star,
ov.file_size,
ov.usage_count,
ov.created_at,
ov.updated_at,
string_agg(ov.category_id, ',' ORDER BY ov.created_at DESC) AS categories
FROM v_operator ov
{where_clause}
{group_by}
{having_clause}
ORDER BY ov.created_at DESC
LIMIT :limit OFFSET :offset
"""
result = await db.execute(text(query), params)
rows = result.fetchall()
# Convert to DTOs
operators = []
for row in rows:
categories_list = []
if row.categories:
categories_list = [cat_id for cat_id in row.categories.split(',') if cat_id]
operators.append(OperatorDto(
id=row.id,
name=row.name,
description=row.description,
version=row.version,
inputs=row.inputs,
outputs=row.outputs,
runtime=row.runtime,
settings=row.settings,
file_name=None,
file_size=row.file_size,
metrics=None,
usage_count=row.usage_count,
is_star=row.is_star,
categories=categories_list,
created_at=row.created_at,
updated_at=row.updated_at,
))
return operators
async def count_operators(
self,
categories: List[List[str]],
keyword: Optional[str],
is_star: Optional[bool],
db: AsyncSession
) -> int:
"""统计算子数量"""
conditions = []
params = {}
if is_star is not None:
conditions.append("is_star = :is_star")
params["is_star"] = is_star
if keyword:
conditions.append(
"(operator_name ILIKE :keyword OR description ILIKE :keyword)"
)
params["keyword"] = f"%{keyword}%"
where_clause = ""
if conditions:
where_clause = "WHERE " + " AND ".join(conditions)
# Handle categories grouping
group_by = "GROUP BY operator_id, operator_name, description, version, inputs, outputs, " \
"runtime, settings, is_star, file_size, usage_count, created_at, updated_at, " \
"created_by, updated_by"
having_clause = ""
if categories:
# Flatten all category IDs for IN clause
all_category_ids = [cat_id for sublist in categories for cat_id in sublist]
if all_category_ids:
where_clause += " AND category_id = ANY(:category_ids)" if where_clause else "WHERE category_id = ANY(:category_ids)"
params["category_ids"] = all_category_ids
# Build HAVING clause for category groups
having_clauses = []
for i, cat_group in enumerate(categories):
cat_list = ", ".join([f"'{cat_id}'" for cat_id in cat_group])
having_clauses.append(
f"SUM(CASE WHEN category_id IN ({cat_list}) THEN 1 ELSE 0 END) > 0"
)
having_clause = "HAVING " + " AND ".join(having_clauses)
query = f"""
SELECT COUNT(*) as count
FROM (
SELECT operator_id
FROM v_operator
{where_clause}
{group_by}
{having_clause}
) AS t
"""
result = await db.execute(text(query), params)
return result.scalar() or 0
async def get_operator_by_id(
self,
operator_id: str,
db: AsyncSession,
locale: Optional[str] = None
) -> OperatorDto:
"""根据 ID 获取算子详情"""
attr = "category_name"
if locale == "en":
attr = "category_name_en"
result = await db.execute(
text(f"""
SELECT
operator_id, operator_name, description, version, inputs, outputs, runtime,
settings, is_star, file_name, file_size, usage_count, metrics,
created_at, updated_at, created_by, updated_by,
string_agg({attr}, ',' ORDER BY created_at DESC) AS categories
FROM v_operator
WHERE operator_id = :operator_id
GROUP BY operator_id, operator_name, description, version, inputs, outputs, runtime,
settings, is_star, file_name, file_size, usage_count, metrics,
created_at, updated_at, created_by, updated_by
"""),
{"operator_id": operator_id}
)
row = result.fetchone()
if not row:
raise BusinessError(ErrorCodes.OPERATOR_NOT_FOUND, operator_id)
# Parse categories from comma-separated string
categories_str = row.categories if hasattr(row, 'categories') and row.categories else ""
categories = [c.strip() for c in categories_str.split(",")] if categories_str else []
# Build DTO
operator = OperatorDto(
id=row.operator_id,
name=row.operator_name,
description=row.description,
version=row.version,
inputs=row.inputs,
outputs=row.outputs,
runtime=row.runtime,
settings=row.settings,
file_name=row.file_name,
file_size=row.file_size,
metrics=row.metrics,
usage_count=row.usage_count,
is_star=row.is_star,
created_at=row.created_at,
updated_at=row.updated_at,
categories=categories,
)
# Read requirements and readme if file exists
if row.file_name:
extract_path = self._get_extract_path(
self._get_stem(row.file_name)
)
operator.requirements = self._read_requirements(extract_path)
operator.readme = self._get_readme_content(extract_path)
# Load releases
releases = await self.operator_release_repo.find_all_by_operator_id(
operator_id, db
)
operator.releases = [
OperatorReleaseDto(
id=release.id,
version=release.version,
release_date=release.release_date,
changelog=release.changelog
)
for release in releases
]
return operator
async def create_operator(
self,
req: OperatorDto,
db: AsyncSession
) -> OperatorDto:
"""创建算子"""
# Generate ID if not provided
if not req.id:
req.id = str(uuid.uuid4())
# Override settings
self._override_settings(req)
# Insert operator
await self.operator_repo.insert(req, db)
await db.flush()
# Insert category relations
if req.categories:
await self.category_relation_repo.batch_insert(
req.id, req.categories, db
)
# Insert release
if req.releases:
release = req.releases[0]
release.id = req.id
release.version = req.version
release.release_date = datetime.now()
await self.operator_release_repo.insert(release, db)
# Extract files
if req.file_name:
self.parser_holder.extract_to(
self._get_file_type(req.file_name),
self._get_upload_path(req.file_name),
self._get_extract_path(self._get_stem(req.file_name))
)
return req
async def update_operator(
self,
operator_id: str,
req: OperatorUpdateDto,
db: AsyncSession
) -> OperatorDto:
"""更新算子"""
# Get existing operator
existing = await self.get_operator_by_id(operator_id, db)
# Save original version for release comparison
original_version = existing.version
# Merge update request into existing operator
# Only update fields that are provided (not None)
if req.name is not None:
existing.name = req.name
if req.description is not None:
existing.description = req.description
if req.version is not None:
existing.version = req.version
if req.inputs is not None:
existing.inputs = req.inputs
if req.outputs is not None:
existing.outputs = req.outputs
if req.runtime is not None:
existing.runtime = req.runtime
if req.settings is not None:
existing.settings = req.settings
if req.file_name is not None:
existing.file_name = req.file_name
if req.file_size is not None:
existing.file_size = req.file_size
if req.metrics is not None:
existing.metrics = req.metrics
if req.usage_count is not None:
existing.usage_count = req.usage_count
if req.is_star is not None:
existing.is_star = req.is_star
if req.categories is not None:
existing.categories = req.categories
if req.overrides is not None:
existing.overrides = req.overrides
# Override settings
self._override_settings(existing)
# Update operator
await self.operator_repo.update(existing, db)
# Update category relations
if req.file_name is not None and req.categories is not None:
await self.category_relation_repo.batch_update(
operator_id, req.categories, db
)
# Update release
if req.releases is not None and len(req.releases) > 0:
release = req.releases[0]
release.id = operator_id
release.version = req.version
release.release_date = datetime.now()
if original_version == release.version:
await self.operator_release_repo.update(release, db)
else:
await self.operator_release_repo.insert(release, db)
# Extract files
if req.file_name is not None:
self.parser_holder.extract_to(
self._get_file_type(req.file_name),
self._get_upload_path(req.file_name),
self._get_extract_path(self._get_stem(req.file_name))
)
await db.flush()
return await self.get_operator_by_id(operator_id, db)
async def delete_operator(
self,
operator_id: str,
db: AsyncSession
) -> None:
"""删除算子"""
# Check if operator is in use
in_template = await self.operator_repo.operator_in_template(operator_id, db)
in_unstop_task = await self.operator_repo.operator_in_unstop_task(operator_id, db)
if in_template or in_unstop_task:
raise BusinessError(ErrorCodes.OPERATOR_IN_INSTANCE)
# Check if operator is predefined
is_predefined = await self.category_relation_repo.operator_is_predefined(
operator_id, db
)
if is_predefined:
raise BusinessError(ErrorCodes.OPERATOR_CANNOT_DELETE_PREDEFINED)
# Get operator for file cleanup
operator = await self.get_operator_by_id(operator_id, db)
# Delete from database
await self.operator_repo.delete(operator_id, db)
await self.category_relation_repo.delete_by_operator_id(operator_id, db)
await self.operator_release_repo.delete(operator_id, db)
# Delete extracted files
if operator.file_name:
extract_path = self._get_extract_path(self._get_stem(operator.file_name))
shutil.rmtree(extract_path, ignore_errors=True)
async def upload_operator(
self,
file_name: str,
db: AsyncSession
) -> OperatorDto:
"""上传算子文件并解析元数据"""
file_path = self._get_upload_path(file_name)
file_size = os.path.getsize(file_path) if os.path.exists(file_path) else None
return self.parser_holder.parse_yaml_from_archive(
self._get_file_type(file_name),
file_path,
YAML_PATH,
file_name,
file_size
)
async def pre_upload(self, db: AsyncSession) -> str:
"""预上传，返回请求 ID"""
from app.module.operator.constants import OPERATOR_BASE_PATH, UPLOAD_DIR
upload_path = os.path.join(OPERATOR_BASE_PATH, UPLOAD_DIR)
req_id = await self.file_service.pre_upload(
upload_path=upload_path,
service_id=SERVICE_ID,
db_session=db,
check_info=None
)
return req_id
async def chunk_upload(
self,
req_id: str,
file_no: int,
file_name: str,
total_chunk_num: int,
chunk_no: int,
check_sum_hex: Optional[str],
file_content: bytes,
db: AsyncSession
) -> FileUploadResult:
"""分块上传文件"""
from app.module.operator.constants import OPERATOR_BASE_PATH, UPLOAD_DIR
upload_path = os.path.join(OPERATOR_BASE_PATH, UPLOAD_DIR)
chunk_request = ChunkUploadRequestDto(
req_id=req_id,
file_no=file_no,
file_name=file_name,
total_chunk_num=total_chunk_num,
chunk_no=chunk_no,
check_sum_hex=check_sum_hex,
)
return await self.file_service.chunk_upload(
chunk_request, upload_path, file_content, db
)
def download_example_operator(self, file_path: str) -> Path:
"""下载示例算子文件"""
path = Path(file_path)
if not path.exists():
raise FileNotFoundError(f"File not found: {file_path}")
return path
def _override_settings(self, operator: OperatorDto) -> None:
"""用 overrides 值覆盖 settings 的 defaultVal"""
if not operator.settings or not operator.overrides:
return
try:
settings = json.loads(operator.settings)
for key, value in operator.overrides.items():
if key not in settings:
continue
setting = settings[key]
setting_type = setting.get("type")
match setting_type:
case "slider" | "switch" | "select" | "input" | "radio":
setting["defaultVal"] = value
case "checkbox":
setting["defaultVal"] = self._convert_to_list_string(value)
case "range":
self._update_properties(setting, value)
settings[key] = setting
operator.settings = json.dumps(settings)
except json.JSONDecodeError as e:
raise BusinessError(ErrorCodes.OPERATOR_PARSE_FAILED, str(e))
def _convert_to_list_string(self, value: Any) -> str:
"""转换为逗号分隔的字符串"""
if value is None:
return ""
if isinstance(value, list):
return ",".join(str(v) for v in value)
return str(value)
def _update_properties(self, setting: Dict[str, Any], value: Any) -> None:
"""更新 range 类型的 properties"""
if not isinstance(value, list):
return
properties = setting.get("properties", [])
if not isinstance(properties, list) or len(properties) != len(value):
return
for i, prop in enumerate(properties):
if isinstance(prop, dict):
prop["defaultVal"] = value[i]
setting["properties"] = properties
def _read_requirements(self, extract_path: str) -> List[str]:
"""读取 requirements.txt"""
requirements_path = Path(extract_path) / "requirements.txt"
if not requirements_path.exists():
return []
requirements = []
try:
with open(requirements_path, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if line and not line.startswith('#'):
requirements.append(line)
except Exception as e:
logger.warning(f"Failed to read requirements: {e}")
return requirements
def _get_readme_content(self, extract_path: str) -> str:
"""读取 README 内容"""
dir_path = Path(extract_path)
if not dir_path.exists() or not dir_path.is_dir():
logger.info(f"Directory does not exist or is not a directory: {extract_path}")
return ""
candidates = ["README.md", "readme.md", "Readme.md"]
for filename in candidates:
readme_path = dir_path / filename
if readme_path.exists() and readme_path.is_file():
try:
content = readme_path.read_text(encoding='utf-8')
logger.info(f"Successfully read README from: {readme_path}")
return content
except Exception as e:
logger.warning(f"Failed to read README from {readme_path}: {e}")
logger.info(f"No README found in: {extract_path}")
return ""
def _get_file_type(self, file_name: str) -> str:
"""获取文件类型（扩展名）"""
return file_name.rsplit('.', 1)[-1].lower() if '.' in file_name else ""
def _get_stem(self, file_name: str) -> str:
"""获取文件名不含扩展名"""
return file_name.rsplit('.', 1)[0] if '.' in file_name else file_name
def _get_upload_path(self, file_name: str) -> str:
"""获取上传文件路径"""
return os.path.join(OPERATOR_BASE_PATH, UPLOAD_DIR, file_name)
async def increment_usage_count(
self,
operator_ids: List[str],
db: AsyncSession
) -> None:
"""增加算子使用次数"""
await self.operator_repo.increment_usage_count(operator_ids, db)
def _get_extract_path(self, file_stem: str) -> str:
"""获取解压路径"""
return os.path.join(OPERATOR_BASE_PATH, EXTRACT_DIR, file_stem)