deepdata: delivery stack (docker compose, helm, CI, build scripts)
CodeQL Advanced / Analyze (actions) (push) Canceled after 0s
CodeQL Advanced / Analyze (java-kotlin) (push) Canceled after 0s
CodeQL Advanced / Analyze (javascript-typescript) (push) Canceled after 0s
CodeQL Advanced / Analyze (python) (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Deer Flow Docker Image CI / build-and-push-amd (push) Canceled after 0s
Deer Flow Docker Image CI / build-and-push-arm (push) Canceled after 0s
Deer Flow Docker Image CI / manifest (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Python Backend Docker Image CI / Build and Push Backend Python Docker Image (push) Canceled after 0s
Backend Docker Image CI / Build and Push Backend Docker Image (push) Canceled after 0s
Database Docker Image CI / Build and Push Database Docker Image (push) Canceled after 0s
Frontend Docker Image CI / Build and Push Frontend Docker Image (push) Canceled after 0s
Gateway Docker Image CI / Build and Push Gateway Docker Image (push) Canceled after 0s
Label Studio Docker Image CI / Build and Push Label Studio Docker Image (push) Canceled after 0s
Runtime Docker Image CI / Build and Push Runtime Docker Image (push) Canceled after 0s
CodeQL Advanced / Analyze (actions) (push) Canceled after 0s
CodeQL Advanced / Analyze (java-kotlin) (push) Canceled after 0s
CodeQL Advanced / Analyze (javascript-typescript) (push) Canceled after 0s
CodeQL Advanced / Analyze (python) (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Deer Flow Docker Image CI / build-and-push-amd (push) Canceled after 0s
Deer Flow Docker Image CI / build-and-push-arm (push) Canceled after 0s
Deer Flow Docker Image CI / manifest (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Docker Image Build & Push / manifest (push) Canceled after 0s
Docker Image Build & Push / build-and-push-arm (push) Canceled after 0s
Docker Image Build & Push / build-and-push-amd (push) Canceled after 0s
Python Backend Docker Image CI / Build and Push Backend Python Docker Image (push) Canceled after 0s
Backend Docker Image CI / Build and Push Backend Docker Image (push) Canceled after 0s
Database Docker Image CI / Build and Push Database Docker Image (push) Canceled after 0s
Frontend Docker Image CI / Build and Push Frontend Docker Image (push) Canceled after 0s
Gateway Docker Image CI / Build and Push Gateway Docker Image (push) Canceled after 0s
Label Studio Docker Image CI / Build and Push Label Studio Docker Image (push) Canceled after 0s
Runtime Docker Image CI / Build and Push Runtime Docker Image (push) Canceled after 0s
This commit is contained in:
figmar
@@ -0,0 +1,4 @@
|
||||
\set ON_ERROR_STOP 0
|
||||
CREATE DATABASE deepdata;
|
||||
CREATE DATABASE labelstudio;
|
||||
\set ON_ERROR_STOP 1
|
||||
@@ -0,0 +1,451 @@
|
||||
-- 使用现有的deepdata数据库
|
||||
\c deepdata;
|
||||
|
||||
-- 标注配置模板表
|
||||
CREATE TABLE IF NOT EXISTS t_dm_annotation_templates (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
description VARCHAR(500),
|
||||
data_type VARCHAR(50) NOT NULL,
|
||||
labeling_type VARCHAR(50) NOT NULL,
|
||||
configuration JSONB NOT NULL,
|
||||
style VARCHAR(32) NOT NULL,
|
||||
category VARCHAR(50) DEFAULT 'custom',
|
||||
built_in BOOLEAN DEFAULT FALSE,
|
||||
version VARCHAR(20) DEFAULT '1.0',
|
||||
created_by VARCHAR(256),
|
||||
updated_by VARCHAR(256),
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
deleted_at TIMESTAMP
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_dm_annotation_templates IS '标注配置模板表';
|
||||
COMMENT ON COLUMN t_dm_annotation_templates.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_dm_annotation_templates.name IS '模板名称';
|
||||
COMMENT ON COLUMN t_dm_annotation_templates.description IS '模板描述';
|
||||
COMMENT ON COLUMN t_dm_annotation_templates.data_type IS '数据类型: image/text/audio/video/timeseries';
|
||||
COMMENT ON COLUMN t_dm_annotation_templates.labeling_type IS '标注类型: classification/detection/segmentation/ner/relation/etc';
|
||||
COMMENT ON COLUMN t_dm_annotation_templates.configuration IS '标注配置（包含labels定义等）';
|
||||
COMMENT ON COLUMN t_dm_annotation_templates.style IS '样式配置: horizontal/vertical';
|
||||
COMMENT ON COLUMN t_dm_annotation_templates.category IS '模板分类: medical/general/custom/system';
|
||||
COMMENT ON COLUMN t_dm_annotation_templates.built_in IS '是否系统内置模板';
|
||||
COMMENT ON COLUMN t_dm_annotation_templates.version IS '模板版本';
|
||||
COMMENT ON COLUMN t_dm_annotation_templates.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_dm_annotation_templates.updated_by IS '更新者';
|
||||
COMMENT ON COLUMN t_dm_annotation_templates.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_dm_annotation_templates.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_dm_annotation_templates.deleted_at IS '删除时间（软删除）';
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_ann_tpl_data_type ON t_dm_annotation_templates(data_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_ann_tpl_labeling_type ON t_dm_annotation_templates(labeling_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_ann_tpl_category ON t_dm_annotation_templates(category);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_ann_tpl_built_in ON t_dm_annotation_templates(built_in);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_ann_tpl_deleted_at ON t_dm_annotation_templates(deleted_at);
|
||||
|
||||
-- 标注项目表
|
||||
CREATE TABLE IF NOT EXISTS t_dm_labeling_projects (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
dataset_id VARCHAR(36) NOT NULL,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
labeling_project_id VARCHAR(8) NOT NULL,
|
||||
template_id VARCHAR(36),
|
||||
configuration JSONB,
|
||||
progress JSONB,
|
||||
created_by VARCHAR(256),
|
||||
updated_by VARCHAR(256),
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
deleted_at TIMESTAMP
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_dm_labeling_projects IS '标注项目表';
|
||||
COMMENT ON COLUMN t_dm_labeling_projects.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_dm_labeling_projects.dataset_id IS '数据集ID';
|
||||
COMMENT ON COLUMN t_dm_labeling_projects.name IS '项目名称';
|
||||
COMMENT ON COLUMN t_dm_labeling_projects.labeling_project_id IS 'Label Studio项目ID';
|
||||
COMMENT ON COLUMN t_dm_labeling_projects.template_id IS '使用的模板ID';
|
||||
COMMENT ON COLUMN t_dm_labeling_projects.configuration IS '项目配置（可能包含对模板的自定义修改）';
|
||||
COMMENT ON COLUMN t_dm_labeling_projects.progress IS '项目进度信息';
|
||||
COMMENT ON COLUMN t_dm_labeling_projects.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_dm_labeling_projects.updated_by IS '更新者';
|
||||
COMMENT ON COLUMN t_dm_labeling_projects.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_dm_labeling_projects.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_dm_labeling_projects.deleted_at IS '删除时间（软删除）';
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_lp_dataset_id ON t_dm_labeling_projects(dataset_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_lp_template_id ON t_dm_labeling_projects(template_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_lp_labeling_project_id ON t_dm_labeling_projects(labeling_project_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_lp_deleted_at ON t_dm_labeling_projects(deleted_at);
|
||||
|
||||
-- 外键约束
|
||||
ALTER TABLE t_dm_labeling_projects ADD CONSTRAINT fk_dm_labeling_projects_template
|
||||
FOREIGN KEY (template_id) REFERENCES t_dm_annotation_templates(id) ON DELETE SET NULL;
|
||||
|
||||
-- 自动标注任务表
|
||||
CREATE TABLE IF NOT EXISTS t_dm_auto_annotation_tasks (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
dataset_id VARCHAR(36) NOT NULL,
|
||||
dataset_name VARCHAR(255),
|
||||
config JSONB NOT NULL,
|
||||
file_ids JSONB,
|
||||
status VARCHAR(50) NOT NULL DEFAULT 'pending',
|
||||
progress INTEGER DEFAULT 0,
|
||||
total_images INTEGER DEFAULT 0,
|
||||
processed_images INTEGER DEFAULT 0,
|
||||
detected_objects INTEGER DEFAULT 0,
|
||||
output_path VARCHAR(500),
|
||||
error_message TEXT,
|
||||
created_by VARCHAR(256),
|
||||
updated_by VARCHAR(256),
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
completed_at TIMESTAMP,
|
||||
deleted_at TIMESTAMP
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_dm_auto_annotation_tasks IS '自动标注任务表';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.name IS '任务名称';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.dataset_id IS '数据集ID';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.dataset_name IS '数据集名称（冗余字段，方便查询）';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.config IS '任务配置（模型规模、置信度等）';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.file_ids IS '要处理的文件ID列表，为空则处理数据集所有图像';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.status IS '任务状态: pending/running/completed/failed';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.progress IS '任务进度 0-100';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.total_images IS '总图片数';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.processed_images IS '已处理图片数';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.detected_objects IS '检测到的对象总数';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.output_path IS '输出路径';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.error_message IS '错误信息';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.updated_by IS '更新者';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.completed_at IS '完成时间';
|
||||
COMMENT ON COLUMN t_dm_auto_annotation_tasks.deleted_at IS '删除时间（软删除）';
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_aat_dataset_id ON t_dm_auto_annotation_tasks(dataset_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_aat_status ON t_dm_auto_annotation_tasks(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_aat_created_at ON t_dm_auto_annotation_tasks(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_aat_deleted_at ON t_dm_auto_annotation_tasks(deleted_at);
|
||||
|
||||
-- 创建触发器用于自动更新 updated_at
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
-- 为各个表创建触发器
|
||||
DROP TRIGGER IF EXISTS update_dm_annotation_templates_updated_at ON t_dm_annotation_templates;
|
||||
CREATE TRIGGER update_dm_annotation_templates_updated_at
|
||||
BEFORE UPDATE ON t_dm_annotation_templates
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
DROP TRIGGER IF EXISTS update_dm_labeling_projects_updated_at ON t_dm_labeling_projects;
|
||||
CREATE TRIGGER update_dm_labeling_projects_updated_at
|
||||
BEFORE UPDATE ON t_dm_labeling_projects
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
DROP TRIGGER IF EXISTS update_dm_auto_annotation_tasks_updated_at ON t_dm_auto_annotation_tasks;
|
||||
CREATE TRIGGER update_dm_auto_annotation_tasks_updated_at
|
||||
BEFORE UPDATE ON t_dm_auto_annotation_tasks
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- 内置标注模板初始化数据
|
||||
|
||||
-- 1. 图像分类模板
|
||||
INSERT INTO t_dm_annotation_templates (id, name, description, data_type, labeling_type, configuration, style, category, built_in, version)
|
||||
VALUES ('tpl-image-classification-001', '图像分类', '简单的多标签图像分类模板', '图像', '分类', '{
|
||||
"labels": [
|
||||
{
|
||||
"fromName": "choice",
|
||||
"toName": "image",
|
||||
"type": "Choices",
|
||||
"options": ["Cat", "Dog", "Bird", "Other"],
|
||||
"required": true,
|
||||
"description": "选择最符合图像内容的标签"
|
||||
}
|
||||
],
|
||||
"objects": [
|
||||
{
|
||||
"name": "image",
|
||||
"type": "Image",
|
||||
"value": "$image"
|
||||
}
|
||||
]
|
||||
}',
|
||||
'horizontal',
|
||||
'计算机视觉',
|
||||
TRUE,
|
||||
'1.0.0'
|
||||
),
|
||||
(
|
||||
'tpl-object-detection-001',
|
||||
'目标检测（边界框）',
|
||||
'使用矩形边界框进行目标检测',
|
||||
'图像',
|
||||
'目标检测',
|
||||
'{
|
||||
"labels": [
|
||||
{
|
||||
"fromName": "label",
|
||||
"toName": "image",
|
||||
"type": "RectangleLabels",
|
||||
"labels": ["Person", "Vehicle", "Animal", "Object"],
|
||||
"required": false,
|
||||
"description": "在图像中框出目标并标注类别"
|
||||
}
|
||||
],
|
||||
"objects": [
|
||||
{
|
||||
"name": "image",
|
||||
"type": "Image",
|
||||
"value": "$image"
|
||||
}
|
||||
]
|
||||
}',
|
||||
'horizontal',
|
||||
'计算机视觉',
|
||||
TRUE,
|
||||
'1.0.0'
|
||||
),
|
||||
(
|
||||
'tpl-image-segmentation-001',
|
||||
'图像分割（多边形）',
|
||||
'使用多边形标注进行语义分割',
|
||||
'图像',
|
||||
'分割',
|
||||
'{
|
||||
"labels": [
|
||||
{
|
||||
"fromName": "label",
|
||||
"toName": "image",
|
||||
"type": "PolygonLabels",
|
||||
"labels": ["Background", "Foreground", "Person", "Car"],
|
||||
"required": false,
|
||||
"description": "使用多边形框选需要分割的区域"
|
||||
}
|
||||
],
|
||||
"objects": [
|
||||
{
|
||||
"name": "image",
|
||||
"type": "Image",
|
||||
"value": "$image"
|
||||
}
|
||||
]
|
||||
}',
|
||||
'horizontal',
|
||||
'计算机视觉',
|
||||
TRUE,
|
||||
'1.0.0'
|
||||
),
|
||||
(
|
||||
'tpl-text-classification-001',
|
||||
'文本情感分类',
|
||||
'将文本中表达的情感划分到预定义的类别',
|
||||
'文本',
|
||||
'分类',
|
||||
'{
|
||||
"labels": [
|
||||
{
|
||||
"fromName": "choice",
|
||||
"toName": "text",
|
||||
"type": "Choices",
|
||||
"options": ["Positive", "Negative", "Neutral"],
|
||||
"required": true,
|
||||
"description": "对文本的情感或类别进行选择"
|
||||
}
|
||||
],
|
||||
"objects": [
|
||||
{
|
||||
"name": "text",
|
||||
"type": "Text",
|
||||
"value": "$text"
|
||||
}
|
||||
]
|
||||
}',
|
||||
'vertical',
|
||||
'自然语言处理',
|
||||
TRUE,
|
||||
'1.0.0'
|
||||
),
|
||||
(
|
||||
'tpl-ner-001',
|
||||
'命名实体识别',
|
||||
'从文本中抽取并标注命名实体',
|
||||
'文本',
|
||||
'实体识别',
|
||||
'{
|
||||
"labels": [
|
||||
{
|
||||
"fromName": "label",
|
||||
"toName": "text",
|
||||
"type": "Labels",
|
||||
"labels": ["PERSON", "ORG", "LOC", "DATE", "MISC"],
|
||||
"required": false,
|
||||
"description": "在文本中标注人物、地点等实体"
|
||||
}
|
||||
],
|
||||
"objects": [
|
||||
{
|
||||
"name": "text",
|
||||
"type": "Text",
|
||||
"value": "$text"
|
||||
}
|
||||
]
|
||||
}',
|
||||
'vertical',
|
||||
'自然语言处理',
|
||||
TRUE,
|
||||
'1.0.0'
|
||||
),
|
||||
(
|
||||
'tpl-audio-classification-001',
|
||||
'音频分类',
|
||||
'将音频片段划分到不同类别',
|
||||
'音频',
|
||||
'分类',
|
||||
'{
|
||||
"labels": [
|
||||
{
|
||||
"fromName": "choice",
|
||||
"toName": "audio",
|
||||
"type": "Choices",
|
||||
"options": ["Speech", "Music", "Noise", "Silence"],
|
||||
"required": true,
|
||||
"description": "选择音频片段对应的类别"
|
||||
}
|
||||
],
|
||||
"objects": [
|
||||
{
|
||||
"name": "audio",
|
||||
"type": "Audio",
|
||||
"value": "$audio"
|
||||
}
|
||||
]
|
||||
}',
|
||||
'horizontal',
|
||||
'音频',
|
||||
TRUE,
|
||||
'1.0.0'
|
||||
),
|
||||
(
|
||||
'tpl-text-multilabel-001',
|
||||
'文本多标签分类',
|
||||
'可为文本选择多个标签，适用于主题、内容类别等多标签任务',
|
||||
'文本',
|
||||
'分类',
|
||||
'{
|
||||
"labels": [
|
||||
{
|
||||
"fromName": "labels",
|
||||
"toName": "text",
|
||||
"type": "Choices",
|
||||
"options": ["Sports", "Politics", "Tech", "Entertainment"],
|
||||
"required": true,
|
||||
"choice": "multiple",
|
||||
"description": "可选择多个标签"
|
||||
}
|
||||
],
|
||||
"objects": [
|
||||
{
|
||||
"name": "text",
|
||||
"type": "Text",
|
||||
"value": "$text"
|
||||
}
|
||||
]
|
||||
}',
|
||||
'vertical',
|
||||
'自然语言处理',
|
||||
TRUE,
|
||||
'1.0.0'
|
||||
),
|
||||
(
|
||||
'tpl-text-summarization-001',
|
||||
'文本摘要',
|
||||
'根据原文撰写简要摘要',
|
||||
'文本',
|
||||
'摘要',
|
||||
'{
|
||||
"labels": [
|
||||
{
|
||||
"fromName": "summary",
|
||||
"toName": "text",
|
||||
"type": "TextArea",
|
||||
"required": true,
|
||||
"description": "在此填写摘要内容"
|
||||
}
|
||||
],
|
||||
"objects": [
|
||||
{
|
||||
"name": "text",
|
||||
"type": "Text",
|
||||
"value": "$text"
|
||||
}
|
||||
]
|
||||
}',
|
||||
'vertical',
|
||||
'自然语言处理',
|
||||
TRUE,
|
||||
'1.0.0'
|
||||
),
|
||||
(
|
||||
'tpl-keyword-extract-001',
|
||||
'关键词抽取',
|
||||
'从文本中选出关键词或关键短语',
|
||||
'文本',
|
||||
'实体识别',
|
||||
'{
|
||||
"labels": [
|
||||
{
|
||||
"fromName": "kw",
|
||||
"toName": "text",
|
||||
"type": "Labels",
|
||||
"labels": ["Keyword"],
|
||||
"required": false,
|
||||
"description": "高亮文本并标注关键词"
|
||||
}
|
||||
],
|
||||
"objects": [
|
||||
{
|
||||
"name": "text",
|
||||
"type": "Text",
|
||||
"value": "$text"
|
||||
}
|
||||
]
|
||||
}',
|
||||
'vertical',
|
||||
'自然语言处理',
|
||||
TRUE,
|
||||
'1.0.0'
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
description = EXCLUDED.description,
|
||||
data_type = EXCLUDED.data_type,
|
||||
labeling_type = EXCLUDED.labeling_type,
|
||||
configuration = EXCLUDED.configuration,
|
||||
style = EXCLUDED.style,
|
||||
category = EXCLUDED.category,
|
||||
built_in = EXCLUDED.built_in,
|
||||
version = EXCLUDED.version,
|
||||
created_by = COALESCE(EXCLUDED.created_by, 'system'),
|
||||
updated_by = COALESCE(EXCLUDED.updated_by, 'system'),
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
|
||||
-- Ensure all annotation templates have created_by/updated_by set for data scope filtering
|
||||
UPDATE t_dm_annotation_templates SET created_by = 'system', updated_by = 'system' WHERE created_by IS NULL OR created_by = '';
|
||||
@@ -0,0 +1,198 @@
|
||||
-- 使用现有的deepdata数据库
|
||||
\c deepdata;
|
||||
|
||||
-- 清洗模板表
|
||||
CREATE TABLE IF NOT EXISTS t_clean_template
|
||||
(
|
||||
id VARCHAR(64) PRIMARY KEY,
|
||||
name VARCHAR(64) UNIQUE,
|
||||
description VARCHAR(256),
|
||||
created_by VARCHAR(256),
|
||||
updated_by VARCHAR(256),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
COMMENT ON TABLE t_clean_template IS '清洗模板表';
|
||||
COMMENT ON COLUMN t_clean_template.id IS '主键ID';
|
||||
COMMENT ON COLUMN t_clean_template.name IS '模板名称';
|
||||
COMMENT ON COLUMN t_clean_template.description IS '模板描述';
|
||||
COMMENT ON COLUMN t_clean_template.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_clean_template.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_clean_template.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_clean_template.updated_by IS '更新者';
|
||||
|
||||
-- 清洗任务表
|
||||
CREATE TABLE IF NOT EXISTS t_clean_task
|
||||
(
|
||||
id VARCHAR(64) PRIMARY KEY,
|
||||
name VARCHAR(64) UNIQUE,
|
||||
description VARCHAR(256),
|
||||
status VARCHAR(256),
|
||||
src_dataset_id VARCHAR(64),
|
||||
src_dataset_name VARCHAR(64),
|
||||
dest_dataset_id VARCHAR(64),
|
||||
dest_dataset_name VARCHAR(64),
|
||||
before_size BIGINT,
|
||||
after_size BIGINT,
|
||||
file_count INTEGER,
|
||||
retry_count INTEGER,
|
||||
started_at TIMESTAMP,
|
||||
finished_at TIMESTAMP,
|
||||
created_by VARCHAR(256),
|
||||
updated_by VARCHAR(256),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
COMMENT ON TABLE t_clean_task IS '清洗任务表';
|
||||
COMMENT ON COLUMN t_clean_task.id IS '主键ID';
|
||||
COMMENT ON COLUMN t_clean_task.name IS '任务名称';
|
||||
COMMENT ON COLUMN t_clean_task.description IS '任务描述';
|
||||
COMMENT ON COLUMN t_clean_task.status IS '任务状态';
|
||||
COMMENT ON COLUMN t_clean_task.src_dataset_id IS '源数据集ID';
|
||||
COMMENT ON COLUMN t_clean_task.src_dataset_name IS '源数据集名称';
|
||||
COMMENT ON COLUMN t_clean_task.dest_dataset_id IS '目标数据集ID';
|
||||
COMMENT ON COLUMN t_clean_task.dest_dataset_name IS '目标数据集名称';
|
||||
COMMENT ON COLUMN t_clean_task.before_size IS '清洗前大小';
|
||||
COMMENT ON COLUMN t_clean_task.after_size IS '清洗后大小';
|
||||
COMMENT ON COLUMN t_clean_task.file_count IS '文件数量';
|
||||
COMMENT ON COLUMN t_clean_task.retry_count IS '重试次数';
|
||||
COMMENT ON COLUMN t_clean_task.started_at IS '开始时间';
|
||||
COMMENT ON COLUMN t_clean_task.finished_at IS '完成时间';
|
||||
COMMENT ON COLUMN t_clean_task.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_clean_task.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_clean_task.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_clean_task.updated_by IS '更新者';
|
||||
|
||||
-- 操作员实例表
|
||||
CREATE TABLE IF NOT EXISTS t_operator_instance
|
||||
(
|
||||
instance_id VARCHAR(256),
|
||||
operator_id VARCHAR(256),
|
||||
op_index INTEGER,
|
||||
settings_override TEXT,
|
||||
PRIMARY KEY (instance_id, operator_id, op_index)
|
||||
);
|
||||
|
||||
COMMENT ON TABLE t_operator_instance IS '操作员实例表';
|
||||
COMMENT ON COLUMN t_operator_instance.instance_id IS '实例ID';
|
||||
COMMENT ON COLUMN t_operator_instance.operator_id IS '操作员ID';
|
||||
COMMENT ON COLUMN t_operator_instance.op_index IS '操作序号';
|
||||
COMMENT ON COLUMN t_operator_instance.settings_override IS '设置覆盖';
|
||||
|
||||
-- 清洗结果表
|
||||
CREATE TABLE IF NOT EXISTS t_clean_result
|
||||
(
|
||||
instance_id VARCHAR(64),
|
||||
src_file_id VARCHAR(64),
|
||||
dest_file_id VARCHAR(64),
|
||||
src_name VARCHAR(256),
|
||||
dest_name VARCHAR(256),
|
||||
src_type VARCHAR(256),
|
||||
dest_type VARCHAR(256),
|
||||
src_size BIGINT,
|
||||
dest_size BIGINT,
|
||||
status VARCHAR(256),
|
||||
result TEXT,
|
||||
PRIMARY KEY (instance_id, dest_file_id)
|
||||
);
|
||||
|
||||
COMMENT ON TABLE t_clean_result IS '清洗结果表';
|
||||
COMMENT ON COLUMN t_clean_result.instance_id IS '实例ID';
|
||||
COMMENT ON COLUMN t_clean_result.src_file_id IS '源文件ID';
|
||||
COMMENT ON COLUMN t_clean_result.dest_file_id IS '目标文件ID';
|
||||
COMMENT ON COLUMN t_clean_result.src_name IS '源文件名';
|
||||
COMMENT ON COLUMN t_clean_result.dest_name IS '目标文件名';
|
||||
COMMENT ON COLUMN t_clean_result.src_type IS '源文件类型';
|
||||
COMMENT ON COLUMN t_clean_result.dest_type IS '目标文件类型';
|
||||
COMMENT ON COLUMN t_clean_result.src_size IS '源文件大小';
|
||||
COMMENT ON COLUMN t_clean_result.dest_size IS '目标文件大小';
|
||||
COMMENT ON COLUMN t_clean_result.status IS '状态';
|
||||
COMMENT ON COLUMN t_clean_result.result IS '结果';
|
||||
|
||||
-- 创建触发器用于自动更新 updated_at
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
-- 为 t_clean_template 表创建触发器
|
||||
DROP TRIGGER IF EXISTS update_clean_template_updated_at ON t_clean_template;
|
||||
CREATE TRIGGER update_clean_template_updated_at
|
||||
BEFORE UPDATE ON t_clean_template
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- 插入初始数据 - 清洗模板
|
||||
INSERT INTO t_clean_template (id, name, description, created_by, updated_by)
|
||||
VALUES
|
||||
('550e8400-e29b-41d4-a716-446655440001', '安全与隐私合规处理模板', '针对敏感数据进行严格清洗，移除PII（个人身份信息）、政治敏感、暴力色情内容，适用于模型对外发布前的安全合规检查。', 'system', 'system'),
|
||||
('661f9500-f3ac-52e5-b827-557766550002', 'LLM SFT高质量文本清洗模板', '旨在生成高质量、低噪声的训练数据。包含去除乱码、重复内容、繁简转换、全角转半角以及格式标准化处理。', 'system', 'system'),
|
||||
('772a0611-a4bd-63f6-c938-668877660003', 'RAG知识库构建预处理模板', '专为RAG场景设计。重点去除目录、图注、XML/HTML标签等对向量检索无意义的噪声，并进行段落级去重以优化切片质量。', 'system', 'system'),
|
||||
('883b1722-b5ce-7407-d049-779988770004', '原始Web爬虫数据清洗模板', '针对互联网爬取的脏数据进行清洗。重点去除Emoji表情、URL链接、HTML标签以及不可见字符。', 'system', 'system'),
|
||||
('994c2833-c6df-8518-e150-880099880005', '多模态/CV模型训练预处理模板', '针对图像数据集处理。包含去除模糊/重复/相似图片，图片方向校正，目标检测预标注，以及尺寸和格式的统一化。', 'system', 'system')
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
-- Also update existing templates that may have empty created_by
|
||||
UPDATE t_clean_template SET created_by = 'system', updated_by = 'system' WHERE created_by IS NULL OR created_by = '';
|
||||
|
||||
INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override)
|
||||
VALUES
|
||||
('550e8400-e29b-41d4-a716-446655440001', 'PoliticalWordCleaner', 1, NULL),
|
||||
('550e8400-e29b-41d4-a716-446655440001', 'SexualAndViolentWordCleaner', 2, NULL),
|
||||
('550e8400-e29b-41d4-a716-446655440001', 'PiiDetector', 3, NULL),
|
||||
('550e8400-e29b-41d4-a716-446655440001', 'AnonymizedIdNumber', 4, NULL),
|
||||
('550e8400-e29b-41d4-a716-446655440001', 'AnonymizedCreditCardNumber', 5, NULL),
|
||||
('550e8400-e29b-41d4-a716-446655440001', 'AnonymizedPhoneNumber', 6, NULL),
|
||||
('550e8400-e29b-41d4-a716-446655440001', 'EmailNumberCleaner', 7, NULL),
|
||||
('550e8400-e29b-41d4-a716-446655440001', 'AnonymizedIpAddress', 8, NULL)
|
||||
ON CONFLICT (instance_id, operator_id, op_index) DO NOTHING;
|
||||
|
||||
INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override)
|
||||
VALUES
|
||||
('661f9500-f3ac-52e5-b827-557766550002', 'GrableCharactersCleaner', 1, NULL),
|
||||
('661f9500-f3ac-52e5-b827-557766550002', 'InvisibleCharactersCleaner', 2, NULL),
|
||||
('661f9500-f3ac-52e5-b827-557766550002', 'FullWidthCharacterCleaner', 3, NULL),
|
||||
('661f9500-f3ac-52e5-b827-557766550002', 'TraditionalChineseCleaner', 4, NULL),
|
||||
('661f9500-f3ac-52e5-b827-557766550002', 'FileWithShortOrLongLengthFilter', 5, '{"fileLength": [50, 8192]}'),
|
||||
('661f9500-f3ac-52e5-b827-557766550002', 'FileWithHighRepeatPhraseRateFilter', 6, NULL),
|
||||
('661f9500-f3ac-52e5-b827-557766550002', 'FileWithHighSpecialCharRateFilter', 7, NULL),
|
||||
('661f9500-f3ac-52e5-b827-557766550002', 'DuplicateFilesFilter', 8, NULL)
|
||||
ON CONFLICT (instance_id, operator_id, op_index) DO NOTHING;
|
||||
|
||||
|
||||
INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override)
|
||||
VALUES
|
||||
('772a0611-a4bd-63f6-c938-668877660003', 'HtmlTagCleaner', 1, '{"removeTableTags": "false"}'), -- 表格对RAG可能有价值，暂不去除表格
|
||||
('772a0611-a4bd-63f6-c938-668877660003', 'ContentCleaner', 2, NULL),
|
||||
('772a0611-a4bd-63f6-c938-668877660003', 'LegendCleaner', 3, NULL),
|
||||
('772a0611-a4bd-63f6-c938-668877660003', 'XMLTagCleaner', 4, NULL),
|
||||
('772a0611-a4bd-63f6-c938-668877660003', 'UnicodeSpaceCleaner', 5, NULL),
|
||||
('772a0611-a4bd-63f6-c938-668877660003', 'ExtraSpaceCleaner', 6, NULL),
|
||||
('772a0611-a4bd-63f6-c938-668877660003', 'DuplicateSentencesFilter', 7, NULL),
|
||||
('772a0611-a4bd-63f6-c938-668877660003', 'FileWithShortOrLongLengthFilter', 8, '{"fileLength": [20, 100000]}')
|
||||
ON CONFLICT (instance_id, operator_id, op_index) DO NOTHING;
|
||||
|
||||
INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override)
|
||||
VALUES
|
||||
('883b1722-b5ce-7407-d049-779988770004', 'HtmlTagCleaner', 1, '{"removeTableTags": "true"}'),
|
||||
('883b1722-b5ce-7407-d049-779988770004', 'AnonymizedUrlCleaner', 2, NULL),
|
||||
('883b1722-b5ce-7407-d049-779988770004', 'EmojiCleaner', 3, NULL),
|
||||
('883b1722-b5ce-7407-d049-779988770004', 'InvisibleCharactersCleaner', 4, NULL),
|
||||
('883b1722-b5ce-7407-d049-779988770004', 'ExtraSpaceCleaner', 5, NULL),
|
||||
('883b1722-b5ce-7407-d049-779988770004', 'DuplicateFilesFilter', 6, '{"fileDuplicateThreshold": 0.6}')
|
||||
ON CONFLICT (instance_id, operator_id, op_index) DO NOTHING;
|
||||
|
||||
INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override)
|
||||
VALUES
|
||||
('994c2833-c6df-8518-e150-880099880005', 'ImgBlurredImagesCleaner', 1, NULL),
|
||||
('994c2833-c6df-8518-e150-880099880005', 'ImgDuplicatedImagesCleaner', 2, NULL),
|
||||
('994c2833-c6df-8518-e150-880099880005', 'ImgSimilarImagesCleaner', 3, NULL),
|
||||
('994c2833-c6df-8518-e150-880099880005', 'ImgDirectionCorrect', 4, NULL),
|
||||
('994c2833-c6df-8518-e150-880099880005', 'ImgResize', 5, '{"widthSize": 512, "heightSize": 512}'),
|
||||
('994c2833-c6df-8518-e150-880099880005', 'ImgTypeUnify', 6, '{"imgType": "jpg"}')
|
||||
ON CONFLICT (instance_id, operator_id, op_index) DO NOTHING;
|
||||
@@ -0,0 +1,248 @@
|
||||
-- 使用现有的deepdata数据库
|
||||
\c deepdata;
|
||||
|
||||
-- 数据归集任务表
|
||||
CREATE TABLE IF NOT EXISTS t_dc_collection_tasks (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
sync_mode VARCHAR(20) DEFAULT 'ONCE',
|
||||
template_id VARCHAR(36) NOT NULL,
|
||||
template_name VARCHAR(255) NOT NULL,
|
||||
target_path VARCHAR(1000) DEFAULT '',
|
||||
config JSONB NOT NULL,
|
||||
schedule_expression VARCHAR(255),
|
||||
status VARCHAR(20) DEFAULT 'DRAFT',
|
||||
retry_count INTEGER DEFAULT 3,
|
||||
timeout_seconds INTEGER DEFAULT 3600,
|
||||
last_execution_id VARCHAR(36),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_dc_collection_tasks IS '数据归集任务表';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.id IS '任务ID（UUID）';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.name IS '任务名称';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.description IS '任务描述';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.sync_mode IS '同步模式：ONCE/SCHEDULED';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.template_id IS '归集模板ID';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.template_name IS '归集模板名称';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.target_path IS '目标存储路径';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.config IS '归集配置（DataX配置），包含源端和目标端配置信息';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.schedule_expression IS 'Cron调度表达式';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.status IS '任务状态：DRAFT/READY/RUNNING/SUCCESS/FAILED/STOPPED';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.retry_count IS '重试次数';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.timeout_seconds IS '超时时间（秒）';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.last_execution_id IS '最后执行ID（UUID）';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_dc_collection_tasks.updated_by IS '更新者';
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_dc_tasks_status ON t_dc_collection_tasks(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_dc_tasks_created_at ON t_dc_collection_tasks(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_dc_tasks_template_id ON t_dc_collection_tasks(template_id);
|
||||
|
||||
-- 任务执行明细表
|
||||
CREATE TABLE IF NOT EXISTS t_dc_task_executions (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
task_id VARCHAR(36) NOT NULL,
|
||||
task_name VARCHAR(255) NOT NULL,
|
||||
status VARCHAR(20) DEFAULT 'RUNNING',
|
||||
log_path VARCHAR(1000) NOT NULL,
|
||||
started_at TIMESTAMP,
|
||||
completed_at TIMESTAMP,
|
||||
duration_seconds INTEGER DEFAULT 0,
|
||||
error_message TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_dc_task_executions IS '任务执行明细表';
|
||||
COMMENT ON COLUMN t_dc_task_executions.id IS '执行记录ID（UUID）';
|
||||
COMMENT ON COLUMN t_dc_task_executions.task_id IS '任务ID';
|
||||
COMMENT ON COLUMN t_dc_task_executions.task_name IS '任务名称';
|
||||
COMMENT ON COLUMN t_dc_task_executions.status IS '执行状态：RUNNING/SUCCESS/FAILED/STOPPED';
|
||||
COMMENT ON COLUMN t_dc_task_executions.log_path IS '日志文件路径';
|
||||
COMMENT ON COLUMN t_dc_task_executions.started_at IS '开始时间';
|
||||
COMMENT ON COLUMN t_dc_task_executions.completed_at IS '完成时间';
|
||||
COMMENT ON COLUMN t_dc_task_executions.duration_seconds IS '执行时长（秒）';
|
||||
COMMENT ON COLUMN t_dc_task_executions.error_message IS '错误信息';
|
||||
COMMENT ON COLUMN t_dc_task_executions.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_dc_task_executions.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_dc_task_executions.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_dc_task_executions.updated_by IS '更新者';
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_dc_executions_task_id ON t_dc_task_executions(task_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_dc_executions_status ON t_dc_task_executions(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_dc_executions_started_at ON t_dc_task_executions(started_at);
|
||||
|
||||
-- 外键约束
|
||||
ALTER TABLE t_dc_task_executions ADD CONSTRAINT fk_dc_executions_task FOREIGN KEY (task_id)
|
||||
REFERENCES t_dc_collection_tasks(id) ON DELETE CASCADE;
|
||||
|
||||
-- 数据归集模板配置表
|
||||
CREATE TABLE IF NOT EXISTS t_dc_collection_templates (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL UNIQUE,
|
||||
description TEXT,
|
||||
source_type VARCHAR(64) NOT NULL,
|
||||
source_name VARCHAR(64) NOT NULL,
|
||||
target_type VARCHAR(64) NOT NULL,
|
||||
target_name VARCHAR(64) NOT NULL,
|
||||
template_content JSONB NOT NULL,
|
||||
built_in BOOLEAN DEFAULT FALSE,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_dc_collection_templates IS '数据归集模板配置表';
|
||||
COMMENT ON COLUMN t_dc_collection_templates.id IS '模板ID（UUID）';
|
||||
COMMENT ON COLUMN t_dc_collection_templates.name IS '模板名称';
|
||||
COMMENT ON COLUMN t_dc_collection_templates.description IS '模板描述';
|
||||
COMMENT ON COLUMN t_dc_collection_templates.source_type IS '源数据源类型';
|
||||
COMMENT ON COLUMN t_dc_collection_templates.source_name IS '源数据源名称';
|
||||
COMMENT ON COLUMN t_dc_collection_templates.target_type IS '目标数据源类型';
|
||||
COMMENT ON COLUMN t_dc_collection_templates.target_name IS '目标数据源名称';
|
||||
COMMENT ON COLUMN t_dc_collection_templates.template_content IS '模板内容';
|
||||
COMMENT ON COLUMN t_dc_collection_templates.built_in IS '是否系统内置';
|
||||
COMMENT ON COLUMN t_dc_collection_templates.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_dc_collection_templates.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_dc_collection_templates.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_dc_collection_templates.updated_by IS '更新者';
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_dc_templates_source_target ON t_dc_collection_templates(source_type, target_type);
|
||||
|
||||
-- 创建触发器用于自动更新 updated_at
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
-- 为各个表创建触发器
|
||||
DROP TRIGGER IF EXISTS update_dc_collection_tasks_updated_at ON t_dc_collection_tasks;
|
||||
CREATE TRIGGER update_dc_collection_tasks_updated_at
|
||||
BEFORE UPDATE ON t_dc_collection_tasks
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
DROP TRIGGER IF EXISTS update_dc_task_executions_updated_at ON t_dc_task_executions;
|
||||
CREATE TRIGGER update_dc_task_executions_updated_at
|
||||
BEFORE UPDATE ON t_dc_task_executions
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
DROP TRIGGER IF EXISTS update_dc_collection_templates_updated_at ON t_dc_collection_templates;
|
||||
CREATE TRIGGER update_dc_collection_templates_updated_at
|
||||
BEFORE UPDATE ON t_dc_collection_templates
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- 插入初始数据 - 数据归集模板
|
||||
INSERT INTO t_dc_collection_templates (
|
||||
id, name, description, source_type, source_name, target_type, target_name,
|
||||
template_content, built_in, created_by, updated_by
|
||||
) VALUES
|
||||
(
|
||||
'1',
|
||||
'NAS归集模板',
|
||||
'将NAS存储上的文件归集到平台上。',
|
||||
'nfsreader',
|
||||
'nfsreader',
|
||||
'nfswriter',
|
||||
'nfswriter',
|
||||
'{"parameter": {"ip": {"name": "NAS地址","description": "NAS服务的地址，可以为IP或者域名。","type": "input", "required": true, "index": 1}, "path": {"name": "共享路径","description": "NAS服务的共享路径。","type": "input", "required": true, "index": 2}, "files": {"name": "文件列表","description": "指定文件列表进行归集。","type": "selectTag", "required": false, "index": 3}}, "reader": {}, "writer": {}}',
|
||||
TRUE,
|
||||
'system',
|
||||
'system'
|
||||
),
|
||||
(
|
||||
'2',
|
||||
'S3存储归集模板',
|
||||
'将S3对象存储（如OBS、MinIO、Ceph等）上的文件归集到平台上。',
|
||||
's3reader',
|
||||
's3reader',
|
||||
's3writer',
|
||||
's3writer',
|
||||
'{"parameter": {"endpoint": {"name": "服务地址","description": "S3兼容存储的服务地址（如http://minio.example.com:9000）。","type": "input", "required": true, "index": 1}, "bucket": {"name": "存储桶名称","description": "S3存储桶名称。","type": "input", "required": true, "index": 2}, "accessKey": {"name": "Access Key","description": "S3访问密钥（Access Key ID）。","type": "input", "required": true, "index": 3}, "secretKey": {"name": "Secret Key","description": "S3密钥（Secret Access Key）。","type": "password", "required": true, "index": 4}, "prefix": {"name": "匹配前缀","description": "按照匹配前缀选中S3中的文件进行归集。","type": "input", "required": false, "index": 5}, "region": {"name": "区域","description": "S3区域（默认us-east-1）。","type": "input", "required": false, "index": 6}}, "reader": {}, "writer": {}}',
|
||||
TRUE,
|
||||
'system',
|
||||
'system'
|
||||
),
|
||||
(
|
||||
'4',
|
||||
'StarRocks归集模板',
|
||||
'将StarRocks中的数据以csv文件的形式归集到DeepData平台上。',
|
||||
'starrocksreader',
|
||||
'starrocksreader',
|
||||
'txtfilewriter',
|
||||
'txtfilewriter',
|
||||
'{"parameter": {}, "reader": {"username": {"name": "用户名","description": "数据库的用户名。","type": "input", "required": true, "index": 2}, "password": {"name": "密码","description": "数据库的密码。","type": "password", "required": true, "index": 3}, "connection": {"name": "数据库连接信息", "description": "数据库连接信息。", "type": "multipleList", "size": 1, "index": 1, "properties": {"jdbcUrl": {"type": "inputList", "name": "数据库连接", "description": "数据库连接url。", "required": true, "index": 1}, "querySql": {"type": "inputList", "name": "查询sql", "description": "输入符合语法的sql查询语句。", "required": true, "index": 2}}}}, "writer": {"header": {"name": "列名","description": "查询结果的列名，最终会体现为csv文件的表头。","type": "selectTag", "required": false}}}',
|
||||
TRUE,
|
||||
'system',
|
||||
'system'
|
||||
),
|
||||
(
|
||||
'5',
|
||||
'GlusterFS归集模板',
|
||||
'将GlusterFS分布式文件系统上的文件归集到平台上。',
|
||||
'glusterfsreader',
|
||||
'glusterfsreader',
|
||||
'glusterfswriter',
|
||||
'glusterfswriter',
|
||||
'{"parameter": {"ip": {"name": "服务器地址","description": "GlusterFS服务器的IP地址或域名。","type": "input", "required": true, "index": 1}, "volume": {"name": "卷名称","description": "GlusterFS卷名称。","type": "input", "required": true, "index": 2}, "path": {"name": "子路径","description": "卷内的子目录路径（可选）。","type": "input", "required": false, "index": 3}, "files": {"name": "文件列表","description": "指定文件列表进行归集。","type": "selectTag", "required": false, "index": 4}}, "reader": {}, "writer": {}}',
|
||||
TRUE,
|
||||
'system',
|
||||
'system'),
|
||||
(
|
||||
'6',
|
||||
'API归集模板',
|
||||
'将指定API返回的数据以csv文件的形式归集到平台上。',
|
||||
'apireader',
|
||||
'apireader',
|
||||
'txtfilewriter',
|
||||
'txtfilewriter',
|
||||
'{"parameter": {}, "reader": {"api": {"name": "接口地址","description": "API的访问地址。","type": "input", "required": true, "index": 1}, "method": {"name": "请求方式","description": "API的请求方式（默认为GET）。","type": "option", "options": ["GET", "POST"], "required": false, "index": 2}, "body": {"name": "请求体参数","description": "请求体参数，主要针对POST请求。示例：\n {\"query\": \"value\"}","type": "jsonobject", "required": false, "index": 3}, "headers": {"name": "请求头参数","description": "需要设置的请求头参数。示例：\n {\"Authorization\": \"***\"}","type": "jsonobject", "required": false, "index": 4}, "schema": {"name": "数据解析schema","description": "会依据schema解析API返回结果。示例：\n {\"dataPath\": \"返回体内定位到具体数据的路径，形如$.data\", \"fields\": [{\"name\": \"属性名\", \"alias\": \"别名\", \"path\": \"属性的路径\"}]}","type": "jsonobject", "required": true, "index": 5}}, "writer": {}}',
|
||||
TRUE,
|
||||
'system',
|
||||
'system'),
|
||||
(
|
||||
'7',
|
||||
'通用关系型数据库归集模板',
|
||||
'将关系型数据库中的数据以csv文件的形式归集到平台上。当前支持postgres、opengauss、sqlserver、mysql、达梦、db2格式、PPAS格式等关系型数据库。',
|
||||
'rdbmsreader',
|
||||
'rdbmsreader',
|
||||
'txtfilewriter',
|
||||
'txtfilewriter',
|
||||
'{"parameter": {}, "reader": {"username": {"name": "用户名","description": "数据库的用户名。","type": "input", "required": true, "index": 2}, "password": {"name": "密码","description": "数据库的密码。","type": "password", "required": true, "index": 3}, "connection": {"name": "数据库连接信息", "description": "数据库连接信息。", "type": "multipleList", "size": 1, "index": 1, "properties": {"jdbcUrl": {"type": "inputList", "name": "数据库连接", "description": "数据库连接url。", "required": true, "index": 1}, "querySql": {"type": "inputList", "name": "查询sql", "description": "输入符合语法的sql查询语句。", "required": true, "index": 2}}}}, "writer": {"header": {"name": "列名","description": "查询结果的列名，最终会体现为csv文件的表头。","type": "selectTag", "required": false}}}',
|
||||
TRUE,
|
||||
'system',
|
||||
'system'
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
description = EXCLUDED.description,
|
||||
source_type = EXCLUDED.source_type,
|
||||
source_name = EXCLUDED.source_name,
|
||||
target_type = EXCLUDED.target_type,
|
||||
target_name = EXCLUDED.target_name,
|
||||
template_content = EXCLUDED.template_content,
|
||||
built_in = EXCLUDED.built_in,
|
||||
created_by = EXCLUDED.created_by,
|
||||
updated_by = EXCLUDED.updated_by,
|
||||
updated_at = CURRENT_TIMESTAMP;
|
||||
@@ -0,0 +1,85 @@
|
||||
-- 使用现有的deepdata数据库
|
||||
\c deepdata;
|
||||
|
||||
-- 文件切片上传请求表
|
||||
CREATE TABLE IF NOT EXISTS t_chunk_upload_request
|
||||
(
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
total_file_num INTEGER,
|
||||
uploaded_file_num INTEGER,
|
||||
upload_path VARCHAR(256),
|
||||
timeout TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
service_id VARCHAR(64),
|
||||
check_info TEXT
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_chunk_upload_request IS '文件切片上传请求表';
|
||||
COMMENT ON COLUMN t_chunk_upload_request.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_chunk_upload_request.total_file_num IS '总文件数';
|
||||
COMMENT ON COLUMN t_chunk_upload_request.uploaded_file_num IS '已上传文件数';
|
||||
COMMENT ON COLUMN t_chunk_upload_request.upload_path IS '文件路径';
|
||||
COMMENT ON COLUMN t_chunk_upload_request.timeout IS '上传请求超时时间';
|
||||
COMMENT ON COLUMN t_chunk_upload_request.service_id IS '上传请求所属服务：DATA-MANAGEMENT(数据管理);';
|
||||
COMMENT ON COLUMN t_chunk_upload_request.check_info IS '业务信息';
|
||||
|
||||
-- 如果需要索引可以添加
|
||||
CREATE INDEX IF NOT EXISTS idx_chunk_upload_service ON t_chunk_upload_request(service_id);
|
||||
|
||||
-- =========================
|
||||
-- 数据血缘：节点表
|
||||
-- 节点表示实体对象（归集来源、数据集、知识库、模型等）
|
||||
-- =========================
|
||||
CREATE TABLE IF NOT EXISTS t_lineage_node
|
||||
(
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
graph_id VARCHAR(36) NOT NULL,
|
||||
node_type VARCHAR(64) NOT NULL,
|
||||
name VARCHAR(256) NOT NULL,
|
||||
description TEXT,
|
||||
node_metadata TEXT
|
||||
);
|
||||
|
||||
COMMENT ON TABLE t_lineage_node IS '数据血缘节点表（实体对象）';
|
||||
COMMENT ON COLUMN t_lineage_node.id IS '主键ID';
|
||||
COMMENT ON COLUMN t_lineage_node.graph_id IS '图ID';
|
||||
COMMENT ON COLUMN t_lineage_node.node_type IS '节点类型：DATASOURCE/DATASET/KNOWLEDGE_BASE/MODEL等';
|
||||
COMMENT ON COLUMN t_lineage_node.name IS '节点名称';
|
||||
COMMENT ON COLUMN t_lineage_node.description IS '节点描述';
|
||||
COMMENT ON COLUMN t_lineage_node.node_metadata IS '节点扩展信息（JSON）';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_lineage_node_id ON t_lineage_node(id);
|
||||
CREATE INDEX IF NOT EXISTS idx_lineage_graph_id ON t_lineage_node(graph_id);
|
||||
|
||||
-- =========================
|
||||
-- 数据血缘：边表
|
||||
-- 边表示处理流程（归集任务、数据清洗、数据标注、数据合成、数据配比等）
|
||||
-- =========================
|
||||
CREATE TABLE IF NOT EXISTS t_lineage_edge
|
||||
(
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
process_id VARCHAR(36) NOT NULL,
|
||||
graph_id VARCHAR(36) NOT NULL,
|
||||
edge_type VARCHAR(64) NOT NULL,
|
||||
name VARCHAR(256),
|
||||
description TEXT,
|
||||
edge_metadata TEXT,
|
||||
from_node_id VARCHAR(36) NOT NULL,
|
||||
to_node_id VARCHAR(36) NOT NULL
|
||||
);
|
||||
|
||||
COMMENT ON TABLE t_lineage_edge IS '数据血缘边表（处理流程）';
|
||||
COMMENT ON COLUMN t_lineage_edge.id IS '边ID';
|
||||
COMMENT ON COLUMN t_lineage_edge.graph_id IS '图ID';
|
||||
COMMENT ON COLUMN t_lineage_edge.process_id IS '处理流程ID';
|
||||
COMMENT ON COLUMN t_lineage_edge.edge_type IS '边类型：DATA_COLLECTION/DATA_CLEANING/DATA_LABELING/DATA_SYNTHESIS/DATA_RATIO等';
|
||||
COMMENT ON COLUMN t_lineage_edge.name IS '边名称';
|
||||
COMMENT ON COLUMN t_lineage_edge.description IS '边描述';
|
||||
COMMENT ON COLUMN t_lineage_edge.edge_metadata IS '边扩展信息（JSON）';
|
||||
COMMENT ON COLUMN t_lineage_edge.from_node_id IS '源节点ID';
|
||||
COMMENT ON COLUMN t_lineage_edge.to_node_id IS '目标节点ID';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_lineage_process_id ON t_lineage_edge(process_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_lineage_edge_graph_id ON t_lineage_edge(graph_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_lineage_edge_from ON t_lineage_edge(from_node_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_lineage_edge_to ON t_lineage_edge(to_node_id);
|
||||
@@ -0,0 +1,151 @@
|
||||
-- 使用现有的deepdata数据库
|
||||
\c deepdata;
|
||||
|
||||
-- 评估任务表
|
||||
CREATE TABLE IF NOT EXISTS t_de_eval_task (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
task_type VARCHAR(50) NOT NULL,
|
||||
source_type VARCHAR(36),
|
||||
source_id VARCHAR(36),
|
||||
source_name VARCHAR(255),
|
||||
status VARCHAR(50) DEFAULT 'PENDING',
|
||||
eval_method VARCHAR(50) DEFAULT 'AUTO',
|
||||
eval_process DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
eval_prompt TEXT,
|
||||
eval_config TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_de_eval_task IS '评估任务表（UUID 主键）';
|
||||
COMMENT ON COLUMN t_de_eval_task.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_de_eval_task.name IS '评估任务名称';
|
||||
COMMENT ON COLUMN t_de_eval_task.description IS '评估任务描述';
|
||||
COMMENT ON COLUMN t_de_eval_task.task_type IS '评估任务类型：QA';
|
||||
COMMENT ON COLUMN t_de_eval_task.source_type IS '待评估对象类型：DATASET/SYNTHESIS';
|
||||
COMMENT ON COLUMN t_de_eval_task.source_id IS '待评估对象ID';
|
||||
COMMENT ON COLUMN t_de_eval_task.source_name IS '待评估对象名称';
|
||||
COMMENT ON COLUMN t_de_eval_task.status IS '状态：PENDING/RUNNING/COMPLETED/STOPPED/FAILED';
|
||||
COMMENT ON COLUMN t_de_eval_task.eval_method IS '评估方式：AUTO/MANUAL';
|
||||
COMMENT ON COLUMN t_de_eval_task.eval_process IS '评估进度';
|
||||
COMMENT ON COLUMN t_de_eval_task.eval_prompt IS '评估提示词';
|
||||
COMMENT ON COLUMN t_de_eval_task.eval_config IS '评估配置';
|
||||
COMMENT ON COLUMN t_de_eval_task.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_de_eval_task.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_de_eval_task.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_de_eval_task.updated_by IS '更新者';
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_de_eval_task_status ON t_de_eval_task(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_de_eval_task_created_at ON t_de_eval_task(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_de_eval_task_source_id ON t_de_eval_task(source_id);
|
||||
|
||||
-- 评估文件表
|
||||
CREATE TABLE IF NOT EXISTS t_de_eval_file (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
task_id VARCHAR(36) NOT NULL,
|
||||
file_id VARCHAR(36),
|
||||
file_name VARCHAR(255),
|
||||
total_count INTEGER DEFAULT 0,
|
||||
evaluated_count INTEGER DEFAULT 0,
|
||||
error_message TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_de_eval_file IS '评估文件表（UUID 主键）';
|
||||
COMMENT ON COLUMN t_de_eval_file.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_de_eval_file.task_id IS '评估任务ID';
|
||||
COMMENT ON COLUMN t_de_eval_file.file_id IS '文件ID';
|
||||
COMMENT ON COLUMN t_de_eval_file.file_name IS '文件名';
|
||||
COMMENT ON COLUMN t_de_eval_file.total_count IS '总数';
|
||||
COMMENT ON COLUMN t_de_eval_file.evaluated_count IS '已评估数';
|
||||
COMMENT ON COLUMN t_de_eval_file.error_message IS '错误信息';
|
||||
COMMENT ON COLUMN t_de_eval_file.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_de_eval_file.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_de_eval_file.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_de_eval_file.updated_by IS '更新者';
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_de_eval_file_task_id ON t_de_eval_file(task_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_de_eval_file_file_id ON t_de_eval_file(file_id);
|
||||
|
||||
-- 评估条目表
|
||||
CREATE TABLE IF NOT EXISTS t_de_eval_item (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
task_id VARCHAR(36) NOT NULL,
|
||||
file_id VARCHAR(36),
|
||||
item_id VARCHAR(36) NOT NULL,
|
||||
eval_content TEXT,
|
||||
eval_score DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
eval_result TEXT,
|
||||
status VARCHAR(50) DEFAULT 'PENDING',
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_de_eval_item IS '评估条目表（UUID 主键）';
|
||||
COMMENT ON COLUMN t_de_eval_item.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_de_eval_item.task_id IS '评估任务ID';
|
||||
COMMENT ON COLUMN t_de_eval_item.file_id IS '文件ID';
|
||||
COMMENT ON COLUMN t_de_eval_item.item_id IS '评估条目ID';
|
||||
COMMENT ON COLUMN t_de_eval_item.eval_content IS '评估内容';
|
||||
COMMENT ON COLUMN t_de_eval_item.eval_score IS '评估分数';
|
||||
COMMENT ON COLUMN t_de_eval_item.eval_result IS '评估结果';
|
||||
COMMENT ON COLUMN t_de_eval_item.status IS '状态：PENDING/EVALUATED';
|
||||
COMMENT ON COLUMN t_de_eval_item.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_de_eval_item.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_de_eval_item.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_de_eval_item.updated_by IS '更新者';
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_de_eval_item_task_id ON t_de_eval_item(task_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_de_eval_item_file_id ON t_de_eval_item(file_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_de_eval_item_status ON t_de_eval_item(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_de_eval_item_item_id ON t_de_eval_item(item_id);
|
||||
|
||||
-- 创建外键约束
|
||||
ALTER TABLE t_de_eval_file ADD CONSTRAINT fk_de_eval_file_task FOREIGN KEY (task_id)
|
||||
REFERENCES t_de_eval_task(id) ON DELETE CASCADE;
|
||||
|
||||
ALTER TABLE t_de_eval_item ADD CONSTRAINT fk_de_eval_item_task FOREIGN KEY (task_id)
|
||||
REFERENCES t_de_eval_task(id) ON DELETE CASCADE;
|
||||
|
||||
-- 创建触发器用于自动更新 updated_at
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
-- 为各个表创建触发器
|
||||
DROP TRIGGER IF EXISTS update_de_eval_task_updated_at ON t_de_eval_task;
|
||||
CREATE TRIGGER update_de_eval_task_updated_at
|
||||
BEFORE UPDATE ON t_de_eval_task
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
DROP TRIGGER IF EXISTS update_de_eval_file_updated_at ON t_de_eval_file;
|
||||
CREATE TRIGGER update_de_eval_file_updated_at
|
||||
BEFORE UPDATE ON t_de_eval_file
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
DROP TRIGGER IF EXISTS update_de_eval_item_updated_at ON t_de_eval_item;
|
||||
CREATE TRIGGER update_de_eval_item_updated_at
|
||||
BEFORE UPDATE ON t_de_eval_item
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
@@ -0,0 +1,315 @@
|
||||
-- 使用现有的deepdata数据库
|
||||
\c deepdata;
|
||||
-- 数据集表
|
||||
CREATE TABLE IF NOT EXISTS t_dm_datasets (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
dataset_type VARCHAR(50) NOT NULL,
|
||||
category VARCHAR(100),
|
||||
path VARCHAR(500),
|
||||
format VARCHAR(50),
|
||||
schema_info JSONB,
|
||||
size_bytes BIGINT DEFAULT 0,
|
||||
file_count BIGINT DEFAULT 0,
|
||||
record_count BIGINT DEFAULT 0,
|
||||
retention_days INTEGER DEFAULT 0,
|
||||
tags JSONB,
|
||||
metadata JSONB,
|
||||
status VARCHAR(50) DEFAULT 'DRAFT',
|
||||
is_public BOOLEAN DEFAULT FALSE,
|
||||
is_featured BOOLEAN DEFAULT FALSE,
|
||||
version BIGINT NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
COMMENT ON TABLE t_dm_datasets IS '数据集表（UUID 主键）';
|
||||
COMMENT ON COLUMN t_dm_datasets.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_dm_datasets.name IS '数据集名称';
|
||||
COMMENT ON COLUMN t_dm_datasets.description IS '数据集描述';
|
||||
COMMENT ON COLUMN t_dm_datasets.dataset_type IS '数据集类型：IMAGE/TEXT/QA/MULTIMODAL/OTHER';
|
||||
COMMENT ON COLUMN t_dm_datasets.category IS '数据集分类：医学影像/问答/文献等';
|
||||
COMMENT ON COLUMN t_dm_datasets.path IS '数据存储路径';
|
||||
COMMENT ON COLUMN t_dm_datasets.format IS '数据格式：DCM/JPG/JSON/CSV等';
|
||||
COMMENT ON COLUMN t_dm_datasets.schema_info IS '数据结构信息';
|
||||
COMMENT ON COLUMN t_dm_datasets.size_bytes IS '数据大小(字节)';
|
||||
COMMENT ON COLUMN t_dm_datasets.file_count IS '文件数量';
|
||||
COMMENT ON COLUMN t_dm_datasets.record_count IS '记录数量';
|
||||
COMMENT ON COLUMN t_dm_datasets.retention_days IS '数据保留天数（0表示长期保留）';
|
||||
COMMENT ON COLUMN t_dm_datasets.tags IS '标签列表';
|
||||
COMMENT ON COLUMN t_dm_datasets.metadata IS '元数据信息';
|
||||
COMMENT ON COLUMN t_dm_datasets.status IS '状态：DRAFT/ACTIVE/ARCHIVED';
|
||||
COMMENT ON COLUMN t_dm_datasets.is_public IS '是否公开';
|
||||
COMMENT ON COLUMN t_dm_datasets.is_featured IS '是否推荐';
|
||||
COMMENT ON COLUMN t_dm_datasets.version IS '版本号';
|
||||
COMMENT ON COLUMN t_dm_datasets.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_dm_datasets.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_dm_datasets.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_dm_datasets.updated_by IS '更新者';
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_dataset_type ON t_dm_datasets(dataset_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_category ON t_dm_datasets(category);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_format ON t_dm_datasets(format);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_status ON t_dm_datasets(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_public ON t_dm_datasets(is_public);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_featured ON t_dm_datasets(is_featured);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_created_at ON t_dm_datasets(created_at);
|
||||
|
||||
-- 创建触发器用于自动更新 updated_at
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
DROP TRIGGER IF EXISTS update_t_dm_datasets_updated_at ON t_dm_datasets;
|
||||
CREATE TRIGGER update_t_dm_datasets_updated_at
|
||||
BEFORE UPDATE ON t_dm_datasets
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- 数据集文件表
|
||||
CREATE TABLE IF NOT EXISTS t_dm_dataset_files (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
dataset_id VARCHAR(36) NOT NULL,
|
||||
file_name VARCHAR(255) NOT NULL,
|
||||
file_path VARCHAR(1000) NOT NULL,
|
||||
file_type VARCHAR(50),
|
||||
file_size BIGINT DEFAULT 0,
|
||||
check_sum VARCHAR(64),
|
||||
tags JSONB,
|
||||
tags_updated_at TIMESTAMP,
|
||||
annotation JSONB,
|
||||
metadata JSONB,
|
||||
status VARCHAR(50) DEFAULT 'ACTIVE',
|
||||
upload_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
last_access_time TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
COMMENT ON TABLE t_dm_dataset_files IS '数据集文件表（UUID 主键）';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.dataset_id IS '所属数据集ID（UUID）';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.file_name IS '文件名';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.file_path IS '文件路径';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.file_type IS '文件格式：JPG/PNG/DCM/TXT等';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.file_size IS '文件大小(字节)';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.check_sum IS '文件校验和';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.tags IS '文件标签信息（结构化标签/标注概要）';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.tags_updated_at IS '标签最后更新时间';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.annotation IS '完整标注结果（原始JSON）';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.metadata IS '文件元数据';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.status IS '文件状态：ACTIVE/DELETED/PROCESSING';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.upload_time IS '上传时间';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.last_access_time IS '最后访问时间';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_dm_dataset_files.updated_at IS '更新时间';
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_dataset ON t_dm_dataset_files(dataset_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_file_type ON t_dm_dataset_files(file_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_file_status ON t_dm_dataset_files(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_upload_time ON t_dm_dataset_files(upload_time);
|
||||
|
||||
-- 外键约束
|
||||
ALTER TABLE t_dm_dataset_files
|
||||
ADD CONSTRAINT fk_dm_dataset_files_dataset
|
||||
FOREIGN KEY (dataset_id)
|
||||
REFERENCES t_dm_datasets(id)
|
||||
ON DELETE CASCADE;
|
||||
|
||||
-- 创建触发器
|
||||
DROP TRIGGER IF EXISTS update_t_dm_dataset_files_updated_at ON t_dm_dataset_files;
|
||||
CREATE TRIGGER update_t_dm_dataset_files_updated_at
|
||||
BEFORE UPDATE ON t_dm_dataset_files
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- 数据集统计信息表
|
||||
CREATE TABLE IF NOT EXISTS t_dm_dataset_statistics (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
dataset_id VARCHAR(36) NOT NULL,
|
||||
stat_date DATE NOT NULL,
|
||||
total_files BIGINT DEFAULT 0,
|
||||
total_size BIGINT DEFAULT 0,
|
||||
processed_files BIGINT DEFAULT 0,
|
||||
error_files BIGINT DEFAULT 0,
|
||||
download_count BIGINT DEFAULT 0,
|
||||
view_count BIGINT DEFAULT 0,
|
||||
quality_metrics JSONB,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
COMMENT ON TABLE t_dm_dataset_statistics IS '数据集统计信息表（UUID 主键）';
|
||||
COMMENT ON COLUMN t_dm_dataset_statistics.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_dm_dataset_statistics.dataset_id IS '数据集ID（UUID）';
|
||||
COMMENT ON COLUMN t_dm_dataset_statistics.stat_date IS '统计日期';
|
||||
COMMENT ON COLUMN t_dm_dataset_statistics.total_files IS '总文件数';
|
||||
COMMENT ON COLUMN t_dm_dataset_statistics.total_size IS '总大小(字节)';
|
||||
COMMENT ON COLUMN t_dm_dataset_statistics.processed_files IS '已处理文件数';
|
||||
COMMENT ON COLUMN t_dm_dataset_statistics.error_files IS '错误文件数';
|
||||
COMMENT ON COLUMN t_dm_dataset_statistics.download_count IS '下载次数';
|
||||
COMMENT ON COLUMN t_dm_dataset_statistics.view_count IS '查看次数';
|
||||
COMMENT ON COLUMN t_dm_dataset_statistics.quality_metrics IS '质量指标';
|
||||
COMMENT ON COLUMN t_dm_dataset_statistics.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_dm_dataset_statistics.updated_at IS '更新时间';
|
||||
|
||||
-- 创建唯一约束和索引
|
||||
ALTER TABLE t_dm_dataset_statistics
|
||||
ADD CONSTRAINT uk_dm_dataset_date
|
||||
UNIQUE (dataset_id, stat_date);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_stat_date ON t_dm_dataset_statistics(stat_date);
|
||||
|
||||
-- 外键约束
|
||||
ALTER TABLE t_dm_dataset_statistics
|
||||
ADD CONSTRAINT fk_dm_dataset_statistics_dataset
|
||||
FOREIGN KEY (dataset_id)
|
||||
REFERENCES t_dm_datasets(id)
|
||||
ON DELETE CASCADE;
|
||||
|
||||
-- 创建触发器
|
||||
DROP TRIGGER IF EXISTS update_t_dm_dataset_statistics_updated_at ON t_dm_dataset_statistics;
|
||||
CREATE TRIGGER update_t_dm_dataset_statistics_updated_at
|
||||
BEFORE UPDATE ON t_dm_dataset_statistics
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- 标签表
|
||||
CREATE TABLE IF NOT EXISTS t_dm_tags (
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
name VARCHAR(100) NOT NULL UNIQUE,
|
||||
description TEXT,
|
||||
category VARCHAR(50),
|
||||
color VARCHAR(7),
|
||||
usage_count BIGINT DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
COMMENT ON TABLE t_dm_tags IS '标签表（UUID 主键）';
|
||||
COMMENT ON COLUMN t_dm_tags.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_dm_tags.name IS '标签名称';
|
||||
COMMENT ON COLUMN t_dm_tags.description IS '标签描述';
|
||||
COMMENT ON COLUMN t_dm_tags.category IS '标签分类';
|
||||
COMMENT ON COLUMN t_dm_tags.color IS '标签颜色(十六进制)';
|
||||
COMMENT ON COLUMN t_dm_tags.usage_count IS '使用次数';
|
||||
COMMENT ON COLUMN t_dm_tags.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_dm_tags.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_dm_tags.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_dm_tags.updated_by IS '更新者';
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_tag_category ON t_dm_tags(category);
|
||||
CREATE INDEX IF NOT EXISTS idx_dm_tag_usage_count ON t_dm_tags(usage_count);
|
||||
|
||||
-- 创建触发器
|
||||
DROP TRIGGER IF EXISTS update_t_dm_tags_updated_at ON t_dm_tags;
|
||||
CREATE TRIGGER update_t_dm_tags_updated_at
|
||||
BEFORE UPDATE ON t_dm_tags
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- 数据集标签关联表
|
||||
CREATE TABLE IF NOT EXISTS t_dm_dataset_tags (
|
||||
dataset_id VARCHAR(36) NOT NULL,
|
||||
tag_id VARCHAR(36) NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (dataset_id, tag_id)
|
||||
);
|
||||
|
||||
COMMENT ON TABLE t_dm_dataset_tags IS '数据集标签关联表（UUID 外键）';
|
||||
COMMENT ON COLUMN t_dm_dataset_tags.dataset_id IS '数据集ID（UUID）';
|
||||
COMMENT ON COLUMN t_dm_dataset_tags.tag_id IS '标签ID（UUID）';
|
||||
COMMENT ON COLUMN t_dm_dataset_tags.created_at IS '创建时间';
|
||||
|
||||
-- 外键约束
|
||||
ALTER TABLE t_dm_dataset_tags
|
||||
ADD CONSTRAINT fk_dm_dataset_tags_dataset
|
||||
FOREIGN KEY (dataset_id)
|
||||
REFERENCES t_dm_datasets(id)
|
||||
ON DELETE CASCADE;
|
||||
|
||||
ALTER TABLE t_dm_dataset_tags
|
||||
ADD CONSTRAINT fk_dm_dataset_tags_tag
|
||||
FOREIGN KEY (tag_id)
|
||||
REFERENCES t_dm_tags(id)
|
||||
ON DELETE CASCADE;
|
||||
|
||||
-- ===========================================
|
||||
-- 非数据管理表
|
||||
-- ===========================================
|
||||
|
||||
-- 用户表
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
username VARCHAR(255) NOT NULL UNIQUE,
|
||||
email VARCHAR(255) NOT NULL UNIQUE,
|
||||
password_hash VARCHAR(255) NOT NULL,
|
||||
full_name VARCHAR(255),
|
||||
avatar_url VARCHAR(500),
|
||||
role VARCHAR(50) NOT NULL DEFAULT 'USER',
|
||||
organization VARCHAR(255),
|
||||
user_groups JSON,
|
||||
enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
last_login_at TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
COMMENT ON TABLE users IS '用户表';
|
||||
COMMENT ON COLUMN users.id IS '主键ID';
|
||||
COMMENT ON COLUMN users.username IS '用户名';
|
||||
COMMENT ON COLUMN users.email IS '邮箱';
|
||||
COMMENT ON COLUMN users.password_hash IS '密码哈希';
|
||||
COMMENT ON COLUMN users.full_name IS '真实姓名';
|
||||
COMMENT ON COLUMN users.avatar_url IS '头像URL';
|
||||
COMMENT ON COLUMN users.role IS '角色：ADMIN/USER（M2 起组驱动用户存解析后的主角色键 admin/dev/annotator/viewer）';
|
||||
COMMENT ON COLUMN users.organization IS '所属机构';
|
||||
COMMENT ON COLUMN users.user_groups IS 'Casdoor 组列表(JSON)，如 ["deepdata-admin","deepdata-dev"]；组驱动角色解析源(M2)';
|
||||
COMMENT ON COLUMN users.enabled IS '是否启用';
|
||||
COMMENT ON COLUMN users.last_login_at IS '最后登录时间';
|
||||
COMMENT ON COLUMN users.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN users.updated_at IS '更新时间';
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_users_username ON users(username);
|
||||
CREATE INDEX IF NOT EXISTS idx_users_email ON users(email);
|
||||
CREATE INDEX IF NOT EXISTS idx_users_role ON users(role);
|
||||
CREATE INDEX IF NOT EXISTS idx_users_enabled ON users(enabled);
|
||||
|
||||
-- 创建触发器
|
||||
DROP TRIGGER IF EXISTS update_users_updated_at ON users;
|
||||
CREATE TRIGGER update_users_updated_at
|
||||
BEFORE UPDATE ON users
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- 插入初始数据
|
||||
INSERT INTO users (username, email, password_hash, full_name, role, organization)
|
||||
VALUES
|
||||
('admin', 'admin@deepdataengine.com', '$2a$10$/esawo436yxg2eodpl/JJ.3Xu6y9m91/ihXRHie9al.LUoNQR5fF.', '系统管理员', 'ADMIN', 'DeepData')
|
||||
ON CONFLICT (username) DO NOTHING;
|
||||
|
||||
-- 创建视图：数据集统计摘要
|
||||
CREATE OR REPLACE VIEW v_dm_dataset_summary AS
|
||||
SELECT
|
||||
COUNT(*) as total_datasets,
|
||||
SUM(CASE WHEN status = 'ACTIVE' THEN 1 ELSE 0 END) as active_datasets,
|
||||
SUM(CASE WHEN is_public = TRUE THEN 1 ELSE 0 END) as public_datasets,
|
||||
SUM(CASE WHEN is_featured = TRUE THEN 1 ELSE 0 END) as featured_datasets,
|
||||
COALESCE(SUM(file_count), 0) as total_files,
|
||||
COALESCE(SUM(record_count), 0) as total_records,
|
||||
COUNT(DISTINCT dataset_type) as dataset_types,
|
||||
COUNT(DISTINCT category) as categories
|
||||
FROM t_dm_datasets;
|
||||
@@ -0,0 +1,526 @@
|
||||
-- 使用现有的deepdata数据库
|
||||
\c deepdata;
|
||||
|
||||
-- 算子表
|
||||
CREATE TABLE IF NOT EXISTS t_operator
|
||||
(
|
||||
id VARCHAR(64) PRIMARY KEY,
|
||||
name VARCHAR(64) UNIQUE,
|
||||
description VARCHAR(255),
|
||||
version VARCHAR(255),
|
||||
inputs VARCHAR(255),
|
||||
outputs VARCHAR(255),
|
||||
runtime TEXT,
|
||||
settings TEXT,
|
||||
file_name TEXT,
|
||||
file_size BIGINT,
|
||||
metrics TEXT,
|
||||
is_star BOOLEAN,
|
||||
usage_count INT DEFAULT 0,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
COMMENT ON TABLE t_operator IS '算子表';
|
||||
COMMENT ON COLUMN t_operator.id IS '主键ID';
|
||||
COMMENT ON COLUMN t_operator.name IS '算子名称';
|
||||
COMMENT ON COLUMN t_operator.description IS '描述';
|
||||
COMMENT ON COLUMN t_operator.version IS '版本';
|
||||
COMMENT ON COLUMN t_operator.inputs IS '输入类型';
|
||||
COMMENT ON COLUMN t_operator.outputs IS '输出类型';
|
||||
COMMENT ON COLUMN t_operator.runtime IS '运行时信息';
|
||||
COMMENT ON COLUMN t_operator.settings IS '设置信息';
|
||||
COMMENT ON COLUMN t_operator.file_name IS '文件名';
|
||||
COMMENT ON COLUMN t_operator.file_size IS '文件大小';
|
||||
COMMENT ON COLUMN t_operator.metrics IS '性能指标';
|
||||
COMMENT ON COLUMN t_operator.usage_count IS '使用次数';
|
||||
COMMENT ON COLUMN t_operator.is_star IS '是否收藏';
|
||||
COMMENT ON COLUMN t_operator.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_operator.updated_by IS '更新者';
|
||||
COMMENT ON COLUMN t_operator.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_operator.updated_at IS '更新时间';
|
||||
|
||||
-- 算子版本表
|
||||
CREATE TABLE IF NOT EXISTS t_operator_release
|
||||
(
|
||||
id VARCHAR(64),
|
||||
version VARCHAR(255),
|
||||
release_date TIMESTAMP,
|
||||
changelog JSON,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id, version)
|
||||
);
|
||||
|
||||
-- 算子分类表
|
||||
CREATE TABLE IF NOT EXISTS t_operator_category
|
||||
(
|
||||
id VARCHAR(64) PRIMARY KEY,
|
||||
name VARCHAR(64) UNIQUE,
|
||||
name_en VARCHAR(64),
|
||||
value VARCHAR(64) UNIQUE,
|
||||
type VARCHAR(64),
|
||||
parent_id VARCHAR(64),
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
COMMENT ON TABLE t_operator_category IS '算子分类表';
|
||||
COMMENT ON COLUMN t_operator_category.id IS '主键ID';
|
||||
COMMENT ON COLUMN t_operator_category.name IS '分类名称';
|
||||
COMMENT ON COLUMN t_operator_category.value IS '分类值';
|
||||
COMMENT ON COLUMN t_operator_category.type IS '分类类型';
|
||||
COMMENT ON COLUMN t_operator_category.parent_id IS '父分类ID';
|
||||
COMMENT ON COLUMN t_operator_category.created_at IS '创建时间';
|
||||
|
||||
-- 算子分类关联表
|
||||
CREATE TABLE IF NOT EXISTS t_operator_category_relation
|
||||
(
|
||||
category_id VARCHAR(64),
|
||||
operator_id VARCHAR(64),
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (category_id, operator_id)
|
||||
);
|
||||
|
||||
COMMENT ON TABLE t_operator_category_relation IS '算子分类关联表';
|
||||
COMMENT ON COLUMN t_operator_category_relation.category_id IS '分类ID';
|
||||
COMMENT ON COLUMN t_operator_category_relation.operator_id IS '算子ID';
|
||||
|
||||
-- 外键约束
|
||||
ALTER TABLE t_operator_category_relation
|
||||
ADD CONSTRAINT fk_operator_category_relation_category
|
||||
FOREIGN KEY (category_id)
|
||||
REFERENCES t_operator_category(id)
|
||||
ON DELETE CASCADE;
|
||||
|
||||
ALTER TABLE t_operator_category_relation
|
||||
ADD CONSTRAINT fk_operator_category_relation_operator
|
||||
FOREIGN KEY (operator_id)
|
||||
REFERENCES t_operator(id)
|
||||
ON DELETE CASCADE;
|
||||
|
||||
-- 创建触发器用于自动更新 updated_at
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
DROP TRIGGER IF EXISTS update_t_operator_updated_at ON t_operator;
|
||||
CREATE TRIGGER update_t_operator_updated_at
|
||||
BEFORE UPDATE ON t_operator
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- 算子视图
|
||||
CREATE OR REPLACE VIEW v_operator AS
|
||||
SELECT
|
||||
o.id AS operator_id,
|
||||
o.name AS operator_name,
|
||||
o.description,
|
||||
o.version,
|
||||
o.inputs,
|
||||
o.outputs,
|
||||
o.runtime,
|
||||
o.settings,
|
||||
o.is_star,
|
||||
o.file_name,
|
||||
o.file_size,
|
||||
o.usage_count,
|
||||
o.metrics,
|
||||
o.created_at,
|
||||
o.updated_at,
|
||||
o.created_by,
|
||||
o.updated_by,
|
||||
toc.id AS category_id,
|
||||
toc.name AS category_name,
|
||||
toc.name_en AS category_name_en
|
||||
FROM t_operator_category_relation tocr
|
||||
LEFT JOIN t_operator o ON tocr.operator_id = o.id
|
||||
LEFT JOIN t_operator_category toc ON tocr.category_id = toc.id;
|
||||
|
||||
COMMENT ON VIEW v_operator IS '算子视图';
|
||||
|
||||
INSERT INTO t_operator_category(id, name, name_en, value, type, parent_id)
|
||||
VALUES ('64465bec-b46b-11f0-8291-00155d0e4808', '模态', 'Modal', 'modal', 'predefined', '0'),
|
||||
('873000a2-65b3-474b-8ccc-4813c08c76fb', '语言', 'Language', 'language', 'predefined', '0'),
|
||||
('4857cc9e-7b72-429e-b2a8-ddd1c48c4483', '功能', 'Function', 'function', 'predefined', '0'),
|
||||
('d8a5df7a-52a9-42c2-83c4-01062e60f597', '文本', 'Text', 'text', 'predefined', '64465bec-b46b-11f0-8291-00155d0e4808'),
|
||||
('de36b61c-9e8a-4422-8c31-d30585c7100f', '图片', 'Image', 'image', 'predefined', '64465bec-b46b-11f0-8291-00155d0e4808'),
|
||||
('42dd9392-73e4-458c-81ff-41751ada47b5', '音频', 'Audio', 'audio', 'predefined', '64465bec-b46b-11f0-8291-00155d0e4808'),
|
||||
('a233d584-73c8-4188-ad5d-8f7c8dda9c27', '视频', 'Video', 'video', 'predefined', '64465bec-b46b-11f0-8291-00155d0e4808'),
|
||||
('4d7dbd77-0a92-44f3-9056-2cd62d4a71e4', '多模态', 'Multimodal', 'multimodal', 'predefined', '64465bec-b46b-11f0-8291-00155d0e4808'),
|
||||
('9eda9d5d-072b-499b-916c-797a0a8750e1', 'Python', 'Python', 'python', 'predefined', '873000a2-65b3-474b-8ccc-4813c08c76fb'),
|
||||
('8c09476a-a922-418f-a908-733f8a0de521', '清洗', 'Cleaning', 'cleaning', 'predefined', '4857cc9e-7b72-429e-b2a8-ddd1c48c4483'),
|
||||
('cfa9d8e2-5b5f-4f1e-9f12-1234567890ab', '标注', 'Annotation', 'annotation', 'predefined', '4857cc9e-7b72-429e-b2a8-ddd1c48c4483'),
|
||||
('16e2d99e-eafb-44fc-acd0-f35a2bad28f8', '来源', 'Origin', 'origin', 'predefined', '0'),
|
||||
('96a3b07a-3439-4557-a835-525faad60ca3', '系统预置', 'System Preset', 'predefined', 'predefined', '16e2d99e-eafb-44fc-acd0-f35a2bad28f8'),
|
||||
('ec2cdd17-8b93-4a81-88c4-ac9e98d10757', '用户上传', 'User Upload', 'customized', 'predefined', '16e2d99e-eafb-44fc-acd0-f35a2bad28f8'),
|
||||
('0ed75eea-e20b-11f0-88e6-00155d5c9528', '归属', 'Vendor', 'vendor', 'predefined', '0'),
|
||||
('431e7798-5426-4e1a-aae6-b9905a836b34', 'DeepData', 'DeepData', 'deepdata', 'predefined', '0ed75eea-e20b-11f0-88e6-00155d5c9528'),
|
||||
('79b385b4-fde8-4617-bcba-02a176938996', 'DataJuicer', 'DataJuicer', 'data-juicer', 'predefined', '0ed75eea-e20b-11f0-88e6-00155d5c9528'),
|
||||
('f00eaa3e-96c1-4de4-96cd-9848ef5429ec', '其他', 'Others', 'others', 'predefined', '0ed75eea-e20b-11f0-88e6-00155d5c9528')
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO t_operator
|
||||
(id, name, description, version, inputs, outputs, runtime, settings, file_name, file_size, is_star, created_by, updated_by)
|
||||
VALUES
|
||||
('MineruFormatter', 'MinerU PDF文本抽取', '基于MinerU API，抽取PDF中的文本。', '1.0.0', 'text', 'text', null, '{"mineruApi":{"name":"Mineru Api地址","description":"指定mineru服务的api地址，默认为本地同一集群内地址。","type":"input","defaultVal":"http://deepdata-mineru:8000","required":false},"exportType":{"name":"导出类型","description":"指定清洗结果文件类型。若指定为md且后续存在其他清洗算子，可能导致文件格式错乱。","type":"select","defaultVal":"markdown","required":false,"options":[{"label":"markdown","value":"md"},{"label":"txt","value":"txt"}]}}', '', 12288, false, 'system', 'system'),
|
||||
('FileWithHighRepeatPhraseRateFilter', '文档词重复率检查', '去除重复词过多的文档。', '1.0.0', 'text', 'text', null, '{"repeatPhraseRatio": {"name": "文档词重复率", "description": "某个词的统计数/文档总词数 > 设定值，该文档被去除。", "type": "slider", "defaultVal": 0.5, "min": 0, "max": 1, "step": 0.1}, "hitStopwords": {"name": "去除停用词", "description": "统计重复词时，选择是否要去除停用词。", "type": "switch", "defaultVal": false, "required": true, "checkedLabel": "去除", "unCheckedLabel": "不去除"}}', '', 16384, false, 'system', 'system'),
|
||||
('FileWithHighRepeatWordRateFilter', '文档字重复率检查', '去除重复字过多的文档。', '1.0.0', 'text', 'text', null, '{"repeatWordRatio": {"name": "文档字重复率", "description": "某个字的统计数/文档总字数 > 设定值，该文档被去除。", "type": "slider", "defaultVal": 0.5, "min": 0, "max": 1, "step": 0.1}}', '', 8192, false, 'system', 'system'),
|
||||
('FileWithHighSpecialCharRateFilter', '文档特殊字符率检查', '去除特殊字符过多的文档。', '1.0.0', 'text', 'text', null, '{"specialCharRatio": {"name": "文档特殊字符率", "description": "特殊字符的统计数/文档总字数 > 设定值，该文档被去除。", "type": "slider", "defaultVal": 0.3, "min": 0, "max": 1, "step": 0.1}}', '', 5120, false, 'system', 'system'),
|
||||
('DuplicateFilesFilter', '相似文档去除', '相似文档去除。', '1.0.0', 'text', 'text', null, '{"fileDuplicateThreshold": {"name": "文档相似度", "description": "基于MinHash算法和Jaccard相似度，计算当前文档与数据集中其它文档相似性，超过设定值，该文档被去除。", "type": "slider", "defaultVal": 0.5, "min": 0, "max": 1, "step": 0.1}}', '', 13312, false, 'system', 'system'),
|
||||
('FileWithManySensitiveWordsFilter', '文档敏感词率检查', '去除敏感词过多的文档。', '1.0.0', 'text', 'text', null, '{"sensitiveWordsRate": {"name": "文档敏感词率", "description": "敏感词的字数/文档总字数 > 设定值，该文档被去除。", "type": "slider", "defaultVal": 0.01, "min": 0, "max": 1, "step": 0.01}}', '', 29696, false, 'system', 'system'),
|
||||
('FileWithShortOrLongLengthFilter', '文档字数检查', '字数不在指定范围会被过滤掉。', '1.0.0', 'text', 'text', null, '{"fileLength": {"name": "文档字数", "description": "过滤字数不在指定范围内的文档，如[10,10000000]。若输入为空，则不对字数上/下限做限制。", "type": "range", "defaultVal": [10, 10000000], "min": 0, "max": 10000000000000000, "step": 1}}', '', 8192, false, 'system', 'system'),
|
||||
('ContentCleaner', '文档目录去除', '去除文档中的目录。', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
||||
('AnonymizedCreditCardNumber', '信用卡号匿名化', '信用卡号匿名化', '1.0.0', 'text', 'text', null, null, '', 8192, false, 'system', 'system'),
|
||||
('EmailNumberCleaner', '邮件地址匿名化', '邮件地址匿名化', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
||||
('EmojiCleaner', '文档表情去除', '去除文档中表情字符或者emoji符号。', '1.0.0', 'text', 'text', null, null, '', 5120, false, 'system', 'system'),
|
||||
('ExtraSpaceCleaner', '多余空格去除', '移除文档首尾、句中或标点符号附近多余空格和 tab 等。', '1.0.0', 'text', 'text', null, null, '', 8192, false, 'system', 'system'),
|
||||
('FullWidthCharacterCleaner', '全角转半角', '将文档中的所有全角字符转换成半角字符。', '1.0.0', 'text', 'text', null, null, '', 8192, false, 'system', 'system'),
|
||||
('GrableCharactersCleaner', '文档乱码去除', '去除文档中的乱码和无意义的unicode。', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
||||
('HtmlTagCleaner', 'HTML标签去除', '移除文档中HTML标签，如 <html>、<dev>、<p> 等。', '1.0.0', 'text', 'text', null, '{"removeTableTags":{"name":"是否去除表格标签","description":"若为是，则会去除表格标签<tr><td>等。","type":"switch","defaultVal":"false","required":false,"checkedLabel":"是","unCheckedLabel":"否"}}', '', 12288, false, 'system', 'system'),
|
||||
('AnonymizedIdNumber', '身份证号匿名化', '身份证号匿名化。', '1.0.0', 'text', 'text', null, null, '', 36864, false, 'system', 'system'),
|
||||
('InvisibleCharactersCleaner', '不可见字符去除', '去除文档中的不可见字符，例如 0-31 号字符中的部分字符。', '1.0.0', 'text', 'text', null, null, '', 5120, false, 'system', 'system'),
|
||||
('AnonymizedIpAddress', 'IP地址匿名化', 'IP地址匿名化', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
||||
('LegendCleaner', '图注表注去除', '去除文档中的图注、表注等内容。', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
||||
('AnonymizedPhoneNumber', '电话号码匿名化', '电话号码匿名化', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
||||
('PoliticalWordCleaner', '政治文本匿名化', '将政治文本进行匿名化。', '1.0.0', 'text', 'text', null, null, '', 8192, false, 'system', 'system'),
|
||||
('DuplicateSentencesFilter', '文档局部内容去重', '文档局部内容去重。', '1.0.0', 'text', 'text', null, null, '', 5120, false, 'system', 'system'),
|
||||
('SexualAndViolentWordCleaner', '暴力色情文本匿名化', '将暴力、色情文本进行匿名化。', '1.0.0', 'text', 'text', null, null, '', 20480, false, 'system', 'system'),
|
||||
('TraditionalChineseCleaner', '繁体转简体', '将繁体转换为简体。', '1.0.0', 'text', 'text', null, null, '', 5120, false, 'system', 'system'),
|
||||
('UnicodeSpaceCleaner', '空格标准化', '将文档中不同的 unicode 空格，如 u2008，转换为正常空格\\u0020。', '1.0.0', 'text', 'text', null, null, '', 8192, false, 'system', 'system'),
|
||||
('AnonymizedUrlCleaner', 'URL网址匿名化', '将文档中的url网址匿名化。', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
||||
('XMLTagCleaner', 'XML标签去除', '去除XML中的标签。', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
||||
('ImgBlurredImagesCleaner', '模糊图片过滤', '去除模糊的图片。', '1.0.0', 'image', 'image', null, '{"blurredThreshold": {"name": "梯度函数值", "description": "梯度函数值取值越小，图片模糊度越高。", "type": "slider", "defaultVal": 1000, "min": 1, "max": 10000, "step": 1}}', '', 5120, false, 'system', 'system'),
|
||||
('ImgBrightness', '图片亮度增强', '自适应调节图片的亮度。', '1.0.0', 'image', 'image', null, null, '', 4096, false, 'system', 'system'),
|
||||
('ImgContrast', '图片对比度增强', '自适应调节图片的对比度。', '1.0.0', 'image', 'image', null, null, '', 4096, false, 'system', 'system'),
|
||||
('ImgDenoise', '图片噪点去除', '去除图片中的噪点，主要适用于自然场景。', '1.0.0', 'image', 'image', null, null, '', 4096, false, 'system', 'system'),
|
||||
('ImgDuplicatedImagesCleaner', '重复图片去除', '去除重复的图片。', '1.0.0', 'image', 'image', null, null, '', 8192, false, 'system', 'system'),
|
||||
('ImgPerspectiveTransformation', '图片透视变换', '自适应校正图片的视角，主要适用于文档校正场景。', '1.0.0', 'image', 'image', null, null, '', 8192, false, 'system', 'system'),
|
||||
('ImgResize', '图片重采样', '将图片放大或缩小到指定像素。', '1.0.0', 'image', 'image', null, '{"widthSize":{"name":"宽度","type":"inputNumber","description":"像素","defaultVal":256,"min":1,"max":4096,"step":1},"heightSize":{"type":"inputNumber","name":"高度","description":"像素","defaultVal":256,"min":1,"max":4096,"step":1}}', '', 8192, false, 'system', 'system'),
|
||||
('ImgSaturation', '图片饱和度增强', '自适应调节图片的饱和度，主要适用于自然场景图片。', '1.0.0', 'image', 'image', null, null, '', 4096, false, 'system', 'system'),
|
||||
('ImgShadowRemove', '图片阴影去除', '去除图片中的阴影，主要适用于文档场景。', '1.0.0', 'image', 'image', null, null, '', 4096, false, 'system', 'system'),
|
||||
('ImgSharpness', '图片锐度增强', '自适应调节图片的锐度，主要适用于自然场景图片。', '1.0.0', 'image', 'image', null, null, '', 4096, false, 'system', 'system'),
|
||||
('ImgSimilarImagesCleaner', '相似图片去除', '去除相似的图片。', '1.0.0', 'image', 'image', null, '{"similarThreshold": {"name": "相似度", "description": "相似度取值越大，图片相似度越高。", "type": "slider", "defaultVal": 0.8, "min": 0, "max": 1, "step": 0.01}}', '', 14336, false, 'system', 'system'),
|
||||
('ImgTypeUnify', '图片格式转换', '将图片编码格式统一为jpg、jpeg、png、bmp格式。', '1.0.0', 'image', 'image', null, '{"imgType": {"name": "图片编码格式", "type": "select", "defaultVal": "jpg", "options": [{"label": "jpg", "value": "jpg"}, {"label": "png", "value": "png"}, {"label": "jpeg", "value": "jpeg"}, {"label": "bmp", "value": "bmp"}]}}', '', 5120, false, 'system', 'system'),
|
||||
('ImgDirectionCorrect', '图片方向校正', '将含有文字的图片校正到文字水平方向，主要适用于文档场景。', '1.0.0', 'image', 'image', null, null, '', 8192, false, 'system', 'system'),
|
||||
('PiiDetector', '高级匿名化', '高级匿名化算子，检测命名实体并匿名化。', '1.0.0', 'text', 'text', null, null, '', 8192, false, 'system', 'system'),
|
||||
('ObjectDetectionRectangle', '图像目标检测与预标注', '基于 YOLOv8 的图像目标检测算子。对输入图像进行目标检测，输出带矩形框与类别标签的标注图像，并生成结构化标注 JSON（包含类别、置信度与边界框坐标）。支持将检测结果导出为 Label Studio 兼容的 predictions 预标注格式（rectanglelabels），可在标注任务中直接加载并进行人工校正，从而显著降低人工标注成本并提升标注效率。', '1.0.0', 'image', 'image,json', null, null, '', 12288, false, 'system', 'system')
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
||||
SELECT c.id, o.id
|
||||
FROM t_operator_category c
|
||||
CROSS JOIN t_operator o
|
||||
WHERE c.id IN ('d8a5df7a-52a9-42c2-83c4-01062e60f597', '9eda9d5d-072b-499b-916c-797a0a8750e1', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34', '8c09476a-a922-418f-a908-733f8a0de521')
|
||||
AND o.id IN ('FileWithShortOrLongLengthFilter', 'FileWithHighRepeatPhraseRateFilter',
|
||||
'FileWithHighRepeatWordRateFilter', 'FileWithHighSpecialCharRateFilter', 'FileWithManySensitiveWordsFilter',
|
||||
'DuplicateFilesFilter', 'DuplicateSentencesFilter', 'AnonymizedCreditCardNumber', 'AnonymizedIdNumber',
|
||||
'AnonymizedIpAddress', 'AnonymizedPhoneNumber', 'AnonymizedUrlCleaner', 'HtmlTagCleaner', 'XMLTagCleaner',
|
||||
'ContentCleaner', 'EmailNumberCleaner', 'EmojiCleaner', 'ExtraSpaceCleaner', 'FullWidthCharacterCleaner',
|
||||
'GrableCharactersCleaner', 'InvisibleCharactersCleaner', 'LegendCleaner', 'PoliticalWordCleaner',
|
||||
'SexualAndViolentWordCleaner', 'TraditionalChineseCleaner', 'UnicodeSpaceCleaner', 'MineruFormatter',
|
||||
'PiiDetector')
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
||||
SELECT c.id, o.id
|
||||
FROM t_operator_category c
|
||||
CROSS JOIN t_operator o
|
||||
WHERE c.id IN ('de36b61c-9e8a-4422-8c31-d30585c7100f', '9eda9d5d-072b-499b-916c-797a0a8750e1', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34', '8c09476a-a922-418f-a908-733f8a0de521')
|
||||
AND o.id IN ('ImgBlurredImagesCleaner', 'ImgBrightness', 'ImgContrast', 'ImgDenoise',
|
||||
'ImgDuplicatedImagesCleaner', 'ImgPerspectiveTransformation', 'ImgResize', 'ImgSaturation',
|
||||
'ImgShadowRemove', 'ImgSharpness', 'ImgSimilarImagesCleaner', 'ImgTypeUnify', 'ImgDirectionCorrect')
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
-- 图像目标检测与预标注算子：功能归类为「标注」而非「清洗」
|
||||
-- INSERT INTO t_operator_category_relation(category_id, operator_id)
|
||||
-- SELECT c.id, o.id
|
||||
-- FROM t_operator_category c
|
||||
-- CROSS JOIN t_operator o
|
||||
-- WHERE c.id IN ('de36b61c-9e8a-4422-8c31-d30585c7100f', 'cfa9d8e2-5b5f-4f1e-9f12-1234567890ab', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34','9eda9d5d-072b-499b-916c-797a0a8750e1')
|
||||
-- AND o.id IN ('ObjectDetectionRectangle')
|
||||
-- ON CONFLICT DO NOTHING;
|
||||
|
||||
|
||||
INSERT INTO t_operator
|
||||
(id, name, description, version, inputs, outputs, runtime, settings, file_name, is_star, created_by, updated_by)
|
||||
VALUES
|
||||
('entity_attribute_aggregator', '实体属性聚合器', 'Summarizes a given attribute of an entity from a set of documents. 汇总一组文档中实体的给定属性。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('meta_tags_aggregator', '元标签聚合器', 'Merge similar meta tags into a single, unified tag. 将类似的元标记合并到一个统一的标记中。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('most_relevant_entities_aggregator', '最相关实体聚合器', 'Extracts and ranks entities closely related to a given entity from provided texts. 从提供的文本中提取与给定实体密切相关的实体并对其进行排名。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('nested_aggregator', '嵌套聚合器', 'Aggregates nested content from multiple samples into a single summary. 将多个示例中的嵌套内容聚合到单个摘要中。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('document_deduplicator', '文档去重器', 'Deduplicates samples at the document level using exact matching. 使用完全匹配在文档级别删除重复的样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('document_minhash_deduplicator', '文档MinHash去重器', 'Deduplicates samples at the document level using MinHash LSH. 使用MinHash LSH在文档级别删除重复样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('document_simhash_deduplicator', '文档SimHash去重器', 'Deduplicates samples at the document level using SimHash. 使用SimHash在文档级别删除重复的样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_deduplicator', '图像去重器', 'Deduplicates samples at the document level by exact matching of images. 通过图像的精确匹配在文档级别删除重复的样本。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('ray_basic_deduplicator', 'Ray基础去重器', 'Backend for deduplicator. deduplicator的后端。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('ray_bts_minhash_deduplicator', 'Ray BTS MinHash去重器', 'A distributed implementation of Union-Find with load balancing. 具有负载平衡的Union-Find的分布式实现。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('ray_document_deduplicator', 'Ray文档去重器', 'Deduplicates samples at the document level using exact matching in Ray distributed mode. 在Ray分布式模式下使用精确匹配在文档级别删除重复的样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('ray_image_deduplicator', 'Ray图像去重器', 'Deduplicates samples at the document level using exact matching of images in Ray distributed mode. 在光线分布模式下使用图像的精确匹配在文档级别删除重复样本。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('ray_video_deduplicator', 'Ray视频去重器', 'Deduplicates samples at document-level using exact matching of videos in Ray distributed mode. 在Ray分布式模式下使用视频的精确匹配在文档级删除重复样本。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_deduplicator', '视频去重器', 'Deduplicates samples at the document level using exact matching of videos. 使用视频的精确匹配在文档级别删除重复的样本。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('alphanumeric_filter', '字母数字过滤器', 'Filter to keep samples with an alphabet/numeric ratio within a specific range. 过滤器，以保持具有特定范围内的字母/数字比率的样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('audio_duration_filter', '音频时长过滤器', 'Keep data samples whose audio durations are within a specified range. 保留音频持续时间在指定范围内的数据样本。', '1.4.4', 'audio', 'audio', NULL, NULL, '', false, 'system', 'system'),
|
||||
('audio_nmf_snr_filter', '音频NMF信噪比过滤器', 'Keep data samples whose audio Signal-to-Noise Ratios (SNRs) are within a specified range. 保留音频信噪比 (snr) 在指定范围内的数据样本。', '1.4.4', 'audio', 'audio', NULL, NULL, '', false, 'system', 'system'),
|
||||
('audio_size_filter', '音频大小过滤器', 'Keep data samples based on the size of their audio files. 根据音频文件的大小保留数据样本。', '1.4.4', 'audio', 'audio', NULL, NULL, '', false, 'system', 'system'),
|
||||
('average_line_length_filter', '平均行长过滤器', 'Filter to keep samples with average line length within a specific range. 过滤器，以保持平均线长度在特定范围内的样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('character_repetition_filter', '字符重复过滤器', 'Filter to keep samples with character-level n-gram repetition ratio within a specific range. 过滤器将具有字符级n-gram重复比的样本保持在特定范围内。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('flagged_words_filter', '标记词过滤器', 'Filter to keep samples with flagged-word ratio in a specified range. 过滤器将标记词比率的样本保留在指定范围内。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('general_field_filter', '通用字段过滤器', 'Filter to keep samples based on a general field filter condition. 根据常规字段筛选条件保留样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_aesthetics_filter', '图像美学过滤器', 'Filter to keep samples with aesthetics scores within a specific range. 过滤以保持美学分数在特定范围内的样品。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_aspect_ratio_filter', '图像长宽比过滤器', 'Filter to keep samples with image aspect ratio within a specific range. 过滤器，以保持样本的图像纵横比在特定范围内。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_face_count_filter', '图像人脸计数过滤器', 'Filter to keep samples with the number of faces within a specific range. 过滤以保持样本的面数在特定范围内。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_face_ratio_filter', '图像人脸占比过滤器', 'Filter to keep samples with face area ratios within a specific range. 过滤以保持面面积比在特定范围内的样本。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_nsfw_filter', '图像NSFW过滤器', 'Filter to keep samples whose images have nsfw scores in a specified range. 过滤器保留其图像的nsfw分数在指定范围内的样本。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_pair_similarity_filter', '图像对相似度过滤器', 'Filter to keep image pairs with similarities between images within a specific range. 过滤器将图像之间具有相似性的图像对保持在特定范围内。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_shape_filter', '图像形状过滤器', 'Filter to keep samples with image shape (width, height) within specific ranges. 过滤器，以保持样本的图像形状 (宽度，高度) 在特定的范围内。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_size_filter', '图像大小过滤器', 'Keep data samples whose image size (in Bytes/KB/MB/...) is within a specific range. 保留图像大小 (以字节/KB/MB/... 为单位) 在特定范围内的数据样本。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_text_matching_filter', '图文匹配过滤器', 'Filter to keep samples with image-text matching scores within a specific range. 过滤器将图像文本匹配分数的样本保持在特定范围内。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_text_similarity_filter', '图文相似度过滤器', 'Filter to keep samples with image-text similarity within a specified range. 过滤器将具有图像-文本相似性的样本保持在指定范围内。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_watermark_filter', '图像水印过滤器', 'Filter to keep samples whose images have no watermark with high probability. 过滤器以保持其图像没有水印的样本具有高概率。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('in_context_influence_filter', '上下文影响过滤器', 'Filter to keep texts based on their in-context influence on a validation set. 过滤以根据文本在上下文中对验证集的影响来保留文本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('instruction_following_difficulty_filter', '指令跟随难度过滤器', 'Filter to keep texts based on their instruction following difficulty (IFD, https://arxiv.org/abs/2308.12032) score. 过滤以保持文本基于他们的指令跟随难度 (IFD， https://arxiv.org/abs/ 2308.12032) 分数。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('language_id_score_filter', '语种识别得分过滤器', 'Filter to keep samples in a specific language with a confidence score above a threshold. 过滤器以保留置信度高于阈值的特定语言的样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('llm_analysis_filter', 'LLM分析过滤器', 'Base filter class for leveraging LLMs to analyze and filter data samples. 用于利用LLMs分析和过滤数据样本的基本筛选器类。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('llm_difficulty_score_filter', 'LLM难度得分过滤器', 'Filter to keep samples with high difficulty scores estimated by an LLM. 过滤器以保留由LLM估计的高难度分数的样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('llm_perplexity_filter', 'LLM困惑度过滤器', 'Filter to keep samples with perplexity scores within a specified range, computed using a specified LLM. 过滤器将困惑分数的样本保留在指定范围内，使用指定的LLM计算。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('llm_quality_score_filter', 'LLM质量得分过滤器', 'Filter to keep samples with a high quality score estimated by a language model. 过滤器，以保留具有语言模型估计的高质量分数的样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('llm_task_relevance_filter', 'LLM任务相关性过滤器', 'Filter to keep samples with high relevance scores to validation tasks estimated by an LLM. 过滤器以保留与LLM估计的验证任务具有高相关性分数的样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('maximum_line_length_filter', '最大行长过滤器', 'Filter to keep samples with a maximum line length within a specified range. 筛选器将最大行长度的样本保持在指定范围内。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('perplexity_filter', '困惑度过滤器', 'Filter to keep samples with perplexity score in a specified range. 过滤以保持困惑分数在指定范围内的样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('phrase_grounding_recall_filter', '短语定位召回过滤器', 'Filter to keep samples based on the phrase grounding recall of phrases extracted from text in images. 根据从图像中的文本中提取的短语接地召回来过滤以保留样本。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('special_characters_filter', '特殊字符过滤器', 'Filter to keep samples with special-character ratio within a specific range. 过滤器，以将具有特殊字符比率的样本保持在特定范围内。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('specified_field_filter', '指定字段过滤器', 'Filter samples based on the specified field information. 根据指定的字段信息筛选样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('specified_numeric_field_filter', '指定数值字段过滤器', 'Filter samples based on a specified numeric field value. 根据指定的数值字段值筛选样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('stopwords_filter', '停用词过滤器', 'Filter to keep samples with stopword ratio within a specified range. 过滤器将停止词比率的样本保持在指定范围内。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('suffix_filter', '后缀过滤器', 'Filter to keep samples with specified suffix. 过滤器以保留具有指定后缀的样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('text_action_filter', '文本动作过滤器', 'Filter to keep texts that contain a minimum number of actions. 过滤以保留包含最少数量操作的文本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('text_embd_similarity_filter', '文本嵌入相似度过滤器', 'Filter to keep texts whose average embedding similarity to a set of given validation texts falls within a specific range. 过滤器，以保留与一组给定验证文本的平均嵌入相似度在特定范围内的文本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('text_entity_dependency_filter', '文本实体依赖过滤器', 'Identify and filter text samples based on entity dependencies. 根据实体依赖关系识别和过滤文本样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('text_length_filter', '文本长度过滤器', 'Filter to keep samples with total text length within a specific range. 过滤以保持文本总长度在特定范围内的样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('text_pair_similarity_filter', '文本对相似度过滤器', 'Filter to keep text pairs with similarities within a specific range. 过滤以将具有相似性的文本对保持在特定范围内。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('token_num_filter', 'Token数量过滤器', 'Filter to keep samples with a total token number within a specified range. 筛选器将总令牌数的样本保留在指定范围内。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_aesthetics_filter', '视频美学过滤器', 'Filter to keep data samples with aesthetics scores for specified frames in the videos within a specific range. 过滤器将视频中指定帧的美学得分数据样本保留在特定范围内。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_aspect_ratio_filter', '视频长宽比过滤器', 'Filter to keep samples with video aspect ratio within a specific range. 过滤器将视频纵横比的样本保持在特定范围内。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_duration_filter', '视频时长过滤器', 'Keep data samples whose videos'' durations are within a specified range. 保留视频持续时间在指定范围内的数据样本。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_frames_text_similarity_filter', '视频帧文本相似度过滤器', 'Filter to keep samples based on the similarity between video frame images and text within a specific range. 根据视频帧图像和文本之间的相似性进行过滤，以保持样本在特定范围内。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_motion_score_filter', '视频运动得分过滤器', 'Filter to keep samples with video motion scores within a specific range. 过滤器将视频运动分数的样本保持在特定范围内。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_motion_score_raft_filter', '视频RAFT运动得分过滤器', 'Filter to keep samples with video motion scores within a specified range. 过滤器将视频运动分数的样本保持在指定范围内。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_nsfw_filter', '视频NSFW过滤器', 'Filter to keep samples whose videos have nsfw scores in a specified range. 过滤器以保留其视频的nsfw分数在指定范围内的样本。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_ocr_area_ratio_filter', '视频OCR面积占比过滤器', 'Keep data samples whose detected text area ratios for specified frames in the video are within a specified range. 保留检测到的视频中指定帧的文本面积比率在指定范围内的数据样本。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_resolution_filter', '视频分辨率过滤器', 'Keep data samples whose videos'' resolutions are within a specified range. 保留视频分辨率在指定范围内的数据样本。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_tagging_from_frames_filter', '视频帧标签过滤器', 'Filter to keep samples whose videos contain specified tags. 过滤器以保留其视频包含指定标签的样本。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_watermark_filter', '视频水印过滤器', 'Filter to keep samples whose videos have no watermark with high probability. 过滤器以保持其视频具有高概率没有水印的样本。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('word_repetition_filter', '单词重复过滤器', 'Filter to keep samples with word-level n-gram repetition ratio within a specific range. 过滤器将单词级n-gram重复比率的样本保持在特定范围内。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('words_num_filter', '词数过滤器', 'Filter to keep samples with a total word count within a specified range. 过滤器将样本的总字数保持在指定范围内。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('key_value_grouper', '键值分组器', 'Groups samples into batches based on values in specified keys. 根据指定键中的值将样本分组为批处理。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('naive_grouper', '朴素分组器', 'Group all samples in a dataset into a single batched sample. 将数据集中的所有样本分组为单个批处理样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('naive_reverse_grouper', '朴素反向分组器', 'Split batched samples into individual samples. 将批处理的样品分成单个样品。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('audio_add_gaussian_noise_mapper', '音频高斯噪声添加映射器', 'Mapper to add Gaussian noise to audio samples. 映射器将高斯噪声添加到音频样本。', '1.4.4', 'audio', 'audio', NULL, NULL, '', false, 'system', 'system'),
|
||||
('audio_ffmpeg_wrapped_mapper', '音频FFmpeg封装映射器', 'Wraps FFmpeg audio filters for processing audio files in a dataset. 包装FFmpeg音频过滤器，用于处理数据集中的音频文件。', '1.4.4', 'audio', 'audio', NULL, NULL, '', false, 'system', 'system'),
|
||||
('calibrate_qa_mapper', 'QA校准映射器', 'Calibrates question-answer pairs based on reference text using an API model. 使用API模型根据参考文本校准问答对。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('calibrate_query_mapper', '查询校准映射器', 'Calibrate query in question-answer pairs based on reference text. 基于参考文本校准问答对中的查询。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('calibrate_response_mapper', '回复校准映射器', 'Calibrate response in question-answer pairs based on reference text. 根据参考文本校准问答对中的回答。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('chinese_convert_mapper', '中文简繁转换映射器', 'Mapper to convert Chinese text between Traditional, Simplified, and Japanese Kanji. 映射器在繁体、简体和日文汉字之间转换中文文本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('clean_copyright_mapper', '版权清洗映射器', 'Cleans copyright comments at the beginning of text samples. 清除文本示例开头的版权注释。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('clean_email_mapper', '邮箱清洗映射器', 'Cleans email addresses from text samples using a regular expression. 使用正则表达式从文本示例中清除电子邮件地址。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('clean_html_mapper', 'HTML清洗映射器', 'Cleans HTML code from text samples, converting HTML to plain text. 从文本示例中清除HTML代码，将HTML转换为纯文本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('clean_ip_mapper', 'IP清洗映射器', 'Cleans IPv4 and IPv6 addresses from text samples. 从文本示例中清除IPv4和IPv6地址。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('clean_links_mapper', '链接清洗映射器', 'Mapper to clean links like http/https/ftp in text samples. 映射器来清理链接，如文本示例中的http/https/ftp。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('detect_character_attributes_mapper', '角色属性检测映射器', 'Takes an image, a caption, and main character names as input to extract the characters'' attributes. 根据给定的图像、图像描述信息和（多个）角色名称，提取图像中主要角色的属性。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('detect_character_locations_mapper', '角色位置检测映射器', 'Given an image and a list of main character names, extract the bounding boxes for each present character. 给定一张图像和主要角色的名称列表，提取每个在场角色的边界框。(YOLOE + MLLM)', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('detect_main_character_mapper', '主要角色检测映射器', 'Extract all main character names based on the given image and its caption. 根据给定的图像及其图像描述，提取所有主要角色的名字。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('dialog_intent_detection_mapper', '对话意图检测映射器', 'Generates user''s intent labels in a dialog by analyzing the history, query, and response. 通过分析历史记录、查询和响应，在对话框中生成用户的意图标签。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('dialog_sentiment_detection_mapper', '对话情感检测映射器', 'Generates sentiment labels and analysis for user queries in a dialog. 在对话框中为用户查询生成情绪标签和分析。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('dialog_sentiment_intensity_mapper', '对话情感强度映射器', 'Mapper to predict user''s sentiment intensity in a dialog, ranging from -5 to 5. Mapper预测用户在对话框中的情绪强度，范围从-5到5。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('dialog_topic_detection_mapper', '对话主题检测映射器', 'Generates user''s topic labels and analysis in a dialog. 在对话框中生成用户的主题标签和分析。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('download_file_mapper', '文件下载映射器', 'Mapper to download URL files to local files or load them into memory. 映射器将URL文件下载到本地文件或将其加载到内存中。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('expand_macro_mapper', '宏展开映射器', 'Expands macro definitions in the document body of LaTeX samples. 展开LaTeX示例文档主体中的宏定义。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('extract_entity_attribute_mapper', '实体属性提取映射器', 'Extracts attributes for given entities from the text and stores them in the sample''s metadata. 从文本中提取给定实体的属性，并将其存储在示例的元数据中。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('extract_entity_relation_mapper', '实体关系提取映射器', 'Extracts entities and relations from text to build a knowledge graph. 从文本中提取实体和关系以构建知识图谱。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('extract_event_mapper', '事件提取映射器', 'Extracts events and relevant characters from the text. 从文本中提取事件和相关字符。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('extract_keyword_mapper', '关键词提取映射器', 'Generate keywords for the text. 为文本生成关键字。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('extract_nickname_mapper', '昵称提取映射器', 'Extracts nickname relationships in the text using a language model. 使用语言模型提取文本中的昵称关系。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('extract_support_text_mapper', '支撑文本提取映射器', 'Extracts a supporting sub-text from the original text based on a given summary. 根据给定的摘要从原始文本中提取支持子文本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('extract_tables_from_html_mapper', 'HTML表格提取映射器', 'Extracts tables from HTML content and stores them in a specified field. 从HTML内容中提取表并将其存储在指定字段中。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('fix_unicode_mapper', 'Unicode修复映射器', 'Fixes unicode errors in text samples. 修复文本示例中的unicode错误。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('generate_qa_from_examples_mapper', '示例生成QA映射器', 'Generates question and answer pairs from examples using a Hugging Face model. 使用拥抱面部模型从示例生成问题和答案对。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('generate_qa_from_text_mapper', '文本生成QA映射器', 'Generates question and answer pairs from text using a specified model. 使用指定的模型从文本生成问题和答案对。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_blur_mapper', '图像模糊映射器', 'Blurs images in the dataset with a specified probability and blur type. 使用指定的概率和模糊类型对数据集中的图像进行模糊处理。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_captioning_from_gpt4v_mapper', 'GPT4V图像描述映射器', 'Generates text captions for images using the GPT-4 Vision model. 使用GPT-4视觉模型为图像生成文本标题。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_captioning_mapper', '图像描述映射器', 'Generates image captions using a Hugging Face model and appends them to samples. 使用拥抱面部模型生成图像标题，并将其附加到样本中。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_detection_yolo_mapper', 'YOLO图像检测映射器', 'Perform object detection using YOLO on images and return bounding boxes and class labels. 使用YOLO对图像执行对象检测，并返回边界框和类标签。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_diffusion_mapper', '图像扩散生成映射器', 'Generate images using a diffusion model based on provided captions. 使用基于提供的字幕的扩散模型生成图像。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_face_blur_mapper', '图像人脸模糊映射器', 'Mapper to blur faces detected in images. 映射器模糊图像中检测到的人脸。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_remove_background_mapper', '图像去背景映射器', 'Mapper to remove the background of images. 映射器删除图像的背景。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_segment_mapper', '图像分割映射器', 'Perform segment-anything on images and return the bounding boxes. 对图像执行segment-任何操作并返回边界框。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('image_tagging_mapper', '图像打标映射器', 'Generates image tags for each image in the sample. 为样本中的每个图像生成图像标记。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('imgdiff_difference_area_generator_mapper', 'ImgDiff差异区域生成映射器', 'Generates and filters bounding boxes for image pairs based on similarity, segmentation, and text matching. 根据相似性、分割和文本匹配生成和过滤图像对的边界框。', '1.4.4', 'image', 'image', NULL, NULL, '', false, 'system', 'system'),
|
||||
('imgdiff_difference_caption_generator_mapper', 'ImgDiff差异描述生成映射器', 'Generates difference captions for bounding box regions in two images. 为两个图像中的边界框区域生成差异字幕。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('mllm_mapper', 'MLLM视觉问答映射器', 'Mapper to use MLLMs for visual question answering tasks. Mapper使用MLLMs进行视觉问答任务。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('nlpaug_en_mapper', 'NLPAug英语增强映射器', 'Augments English text samples using various methods from the nlpaug library. 使用nlpaug库中的各种方法增强英语文本样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('nlpcda_zh_mapper', 'NLPCDA中文增强映射器', 'Augments Chinese text samples using the nlpcda library. 使用nlpcda库扩充中文文本样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('optimize_prompt_mapper', 'Prompt优化映射器', 'Optimize prompts based on existing ones in the same batch. 根据同一批次中的现有提示优化提示。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('optimize_qa_mapper', 'QA优化映射器', 'Mapper to optimize question-answer pairs. 映射器来优化问题-答案对。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('optimize_query_mapper', '查询优化映射器', 'Optimize queries in question-answer pairs to make them more specific and detailed. 优化问答对中的查询，使其更加具体和详细。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('optimize_response_mapper', '回复优化映射器', 'Optimize response in question-answer pairs to be more detailed and specific. 优化问答对中的响应，使其更加详细和具体。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('pair_preference_mapper', '配对偏好映射器', 'Mapper to construct paired preference samples by generating a rejected response and its reason. Mapper通过生成拒绝响应及其原因来构造成对的偏好样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('punctuation_normalization_mapper', '标点归一化映射器', 'Normalizes unicode punctuations to their English equivalents in text samples. 将unicode标点规范化为文本示例中的英语等效项。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('python_file_mapper', 'Python文件映射器', 'Executes a Python function defined in a file on input data. 对输入数据执行文件中定义的Python函数。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('python_lambda_mapper', 'Python Lambda映射器', 'Mapper for applying a Python lambda function to data samples. Mapper，用于将Python lambda函数应用于数据样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('query_intent_detection_mapper', '查询意图检测映射器', 'Predicts the user''s intent label and corresponding score for a given query. 为给定查询预测用户的意图标签和相应的分数。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('query_sentiment_detection_mapper', '查询情感检测映射器', 'Predicts user''s sentiment label (''negative'', ''neutral'', ''positive'') in a query. 在查询中预测用户的情绪标签 (“负面” 、 “中性” 、 “正面”)。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('query_topic_detection_mapper', '查询主题检测映射器', 'Predicts the topic label and its corresponding score for a given query. 预测给定查询的主题标签及其相应的分数。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('relation_identity_mapper', '关系识别映射器', 'Identify the relation between two entities in a given text. 确定给定文本中两个实体之间的关系。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('remove_bibliography_mapper', '参考书目移除映射器', 'Removes bibliography sections at the end of LaTeX documents. 删除LaTeX文档末尾的参考书目部分。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('remove_comments_mapper', '注释移除映射器', 'Removes comments from documents, currently supporting only ''tex'' format. 从文档中删除注释，当前仅支持 “文本” 格式。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('remove_header_mapper', '页眉移除映射器', 'Removes headers at the beginning of documents in LaTeX samples. 删除LaTeX示例中文档开头的标题。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('remove_long_words_mapper', '长词移除映射器', 'Mapper to remove long words within a specific range. 映射器删除特定范围内的长词。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('remove_non_chinese_character_mapper', '非中文字符移除映射器', 'Removes non-Chinese characters from text samples. 从文本样本中删除非中文字符。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('remove_repeat_sentences_mapper', '重复句移除映射器', 'Mapper to remove repeat sentences in text samples. 映射器删除文本样本中的重复句子。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('remove_specific_chars_mapper', '指定字符移除映射器', 'Removes specific characters from text samples. 从文本示例中删除特定字符。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('remove_table_text_mapper', '表格文本移除映射器', 'Mapper to remove table texts from text samples. 映射器从文本样本中删除表文本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('remove_words_with_incorrect_substrings_mapper', '错误子串单词移除映射器', 'Mapper to remove words containing specified incorrect substrings. 映射程序删除包含指定的不正确子字符串的单词。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('replace_content_mapper', '内容替换映射器', 'Replaces content in the text that matches a specific regular expression pattern with a designated replacement string. 用指定的替换字符串替换与特定正则表达式模式匹配的文本中的内容。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('sdxl_prompt2prompt_mapper', 'SDXL Prompt2Prompt映射器', 'Generates pairs of similar images using the SDXL model. 使用SDXL模型生成成对的相似图像。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('sentence_augmentation_mapper', '句子增强映射器', 'Augments sentences by generating enhanced versions using a Hugging Face model. 通过使用拥抱面部模型生成增强版本来增强句子。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('sentence_split_mapper', '句子切分映射器', 'Splits text samples into individual sentences based on the specified language. 根据指定的语言将文本样本拆分为单个句子。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('text_chunk_mapper', '文本分块映射器', 'Split input text into chunks based on specified criteria. 根据指定的条件将输入文本拆分为块。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('text_tagging_by_prompt_mapper', 'Prompt文本打标映射器', 'Mapper to generate text tags using prompt with LLM. Mapper使用带有LLM的prompt生成文本标记。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('vggt_mapper', 'VGGT视频提取映射器', 'Input a video of a single scene, and use VGGT to extract information including Camera Pose, Depth Maps, Point Maps, and 3D Point Tracks. 输入单个场景的视频，并使用VGGT提取包括相机姿态、深度图、点图和3D点轨迹的信息。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_captioning_from_audio_mapper', '音频生成视频描述映射器', 'Mapper to caption a video according to its audio streams based on Qwen-Audio model. 映射器根据基于qwen-audio模型的音频流为视频添加字幕。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_captioning_from_frames_mapper', '帧生成视频描述映射器', 'Generates video captions from sampled frames using an image-to-text model. 使用图像到文本模型从采样帧生成视频字幕。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_captioning_from_summarizer_mapper', '摘要生成视频描述映射器', 'Mapper to generate video captions by summarizing several kinds of generated texts (captions from video/audio/frames, tags from audio/frames, ...). 映射器通过总结几种生成的文本 (来自视频/音频/帧的字幕，来自音频/帧的标签，...) 来生成视频字幕。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_captioning_from_video_mapper', '视频生成视频描述映射器', 'Generates video captions using a Hugging Face video-to-text model and sampled video frames. 使用拥抱面部视频到文本模型和采样视频帧生成视频字幕。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_captioning_from_vlm_mapper', 'VLM视频描述映射器', 'Generates video captions using a VLM that accepts videos as inputs. 使用接受视频作为输入的VLM生成视频字幕。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_depth_estimation_mapper', '视频深度估计映射器', 'Perform depth estimation on the video. 对视频进行深度估计。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_extract_frames_mapper', '视频抽帧映射器', 'Mapper to extract frames from video files according to specified methods. 映射器根据指定的方法从视频文件中提取帧。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_face_blur_mapper', '视频人脸模糊映射器', 'Mapper to blur faces detected in videos. 映射器模糊在视频中检测到的人脸。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_ffmpeg_wrapped_mapper', '视频FFmpeg封装映射器', 'Wraps FFmpeg video filters for processing video files in a dataset. 包装FFmpeg视频过滤器，用于处理数据集中的视频文件。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_hand_reconstruction_mapper', '视频手部重建映射器', 'Use the WiLoR model for hand localization and reconstruction. 使用WiLoR模型进行手部定位和重建。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_object_segmenting_mapper', '视频对象分割映射器', 'Text-guided semantic segmentation of valid objects throughout the video (YOLOE + SAM2). 在整个视频中对有效对象进行文本引导的语义分割 (YOLOE SAM2)。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_remove_watermark_mapper', '视频去水印映射器', 'Remove watermarks from videos based on specified regions. 根据指定区域从视频中删除水印。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_resize_aspect_ratio_mapper', '视频宽高比调整映射器', 'Resizes videos to fit within a specified aspect ratio range. 调整视频大小以适应指定的宽高比范围。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_resize_resolution_mapper', '视频分辨率调整映射器', 'Resizes video resolution based on specified width and height constraints. 根据指定的宽度和高度限制调整视频分辨率。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_split_by_duration_mapper', '视频按时长切分映射器', 'Splits videos into segments based on a specified duration. 根据指定的持续时间将视频拆分为多个片段。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_split_by_key_frame_mapper', '视频关键帧切分映射器', 'Splits a video into segments based on key frames. 根据关键帧将视频分割为多个片段。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_split_by_scene_mapper', '视频场景切分映射器', 'Splits videos into scene clips based on detected scene changes. 根据检测到的场景变化将视频拆分为场景剪辑。', '1.4.4', 'multimodal', 'multimodal', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_tagging_from_audio_mapper', '音频视频打标映射器', 'Generates video tags from audio streams using the Audio Spectrogram Transformer. 使用音频频谱图转换器从音频流生成视频标签。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_tagging_from_frames_mapper', '帧视频打标映射器', 'Generates video tags from frames extracted from videos. 从视频中提取的帧生成视频标签。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('video_whole_body_pose_estimation_mapper', '视频全身姿态估计映射器', 'Input a video containing people, and use the DWPose model to extract the body, hand, feet, and face keypoints of the human subjects in the video, i.e., 2D Whole-body Pose Estimation. 输入包含人的视频，并使用DWPose模型来提取视频中人类主体的身体、手、脚和面部关键点，即2D全身姿态估计。', '1.4.4', 'video', 'video', NULL, NULL, '', false, 'system', 'system'),
|
||||
('whitespace_normalization_mapper', '空白字符归一化映射器', 'Normalizes various types of whitespace characters to standard spaces in text samples. 将文本样本中各种类型的空白字符规范化为标准空格。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('frequency_specified_field_selector', '频率指定字段选择器', 'Selector to filter samples based on the frequency of a specified field. 选择器根据指定字段的频率过滤样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('random_selector', '随机选择器', 'Randomly selects a subset of samples from the dataset. 从数据集中随机选择样本子集。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('range_specified_field_selector', '范围指定字段选择器', 'Selects a range of samples based on the sorted values of a specified field. 根据指定字段的排序值选择采样范围。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('tags_specified_field_selector', '标签指定字段选择器', 'Selector to filter samples based on the tags of a specified field. 选择器根据指定字段的标签过滤样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system'),
|
||||
('topk_specified_field_selector', 'TopK指定字段选择器', 'Selects top samples based on the sorted values of a specified field. 根据指定字段的排序值选择顶部样本。', '1.4.4', 'text', 'text', NULL, NULL, '', false, 'system', 'system')
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
||||
SELECT c.id, o.id
|
||||
FROM t_operator_category c
|
||||
CROSS JOIN t_operator o
|
||||
WHERE c.id IN ('d8a5df7a-52a9-42c2-83c4-01062e60f597', '9eda9d5d-072b-499b-916c-797a0a8750e1',
|
||||
'96a3b07a-3439-4557-a835-525faad60ca3', '79b385b4-fde8-4617-bcba-02a176938996', '8c09476a-a922-418f-a908-733f8a0de521')
|
||||
AND o.id IN
|
||||
('entity_attribute_aggregator', 'meta_tags_aggregator', 'most_relevant_entities_aggregator', 'nested_aggregator',
|
||||
'document_deduplicator', 'document_minhash_deduplicator', 'document_simhash_deduplicator',
|
||||
'ray_basic_deduplicator', 'ray_bts_minhash_deduplicator', 'ray_document_deduplicator', 'alphanumeric_filter',
|
||||
'average_line_length_filter', 'character_repetition_filter', 'flagged_words_filter', 'general_field_filter',
|
||||
'in_context_influence_filter', 'instruction_following_difficulty_filter', 'language_id_score_filter',
|
||||
'llm_analysis_filter', 'llm_difficulty_score_filter', 'llm_perplexity_filter', 'llm_quality_score_filter',
|
||||
'llm_task_relevance_filter', 'maximum_line_length_filter', 'perplexity_filter', 'special_characters_filter',
|
||||
'specified_field_filter', 'specified_numeric_field_filter', 'stopwords_filter', 'suffix_filter',
|
||||
'text_action_filter', 'text_embd_similarity_filter', 'text_entity_dependency_filter', 'text_length_filter',
|
||||
'text_pair_similarity_filter', 'token_num_filter', 'word_repetition_filter', 'words_num_filter',
|
||||
'key_value_grouper', 'naive_grouper', 'naive_reverse_grouper', 'calibrate_qa_mapper', 'calibrate_query_mapper',
|
||||
'calibrate_response_mapper', 'chinese_convert_mapper', 'clean_copyright_mapper', 'clean_email_mapper',
|
||||
'clean_html_mapper', 'clean_ip_mapper', 'clean_links_mapper', 'dialog_intent_detection_mapper',
|
||||
'dialog_sentiment_detection_mapper', 'dialog_sentiment_intensity_mapper', 'dialog_topic_detection_mapper',
|
||||
'download_file_mapper', 'expand_macro_mapper', 'extract_entity_attribute_mapper',
|
||||
'extract_entity_relation_mapper', 'extract_event_mapper', 'extract_keyword_mapper', 'extract_nickname_mapper',
|
||||
'extract_support_text_mapper', 'extract_tables_from_html_mapper', 'fix_unicode_mapper',
|
||||
'generate_qa_from_examples_mapper', 'generate_qa_from_text_mapper', 'nlpaug_en_mapper', 'nlpcda_zh_mapper',
|
||||
'optimize_prompt_mapper', 'optimize_qa_mapper', 'optimize_query_mapper', 'optimize_response_mapper',
|
||||
'pair_preference_mapper', 'punctuation_normalization_mapper', 'python_file_mapper', 'python_lambda_mapper',
|
||||
'query_intent_detection_mapper', 'query_sentiment_detection_mapper', 'query_topic_detection_mapper',
|
||||
'relation_identity_mapper', 'remove_bibliography_mapper', 'remove_comments_mapper', 'remove_header_mapper',
|
||||
'remove_long_words_mapper', 'remove_non_chinese_character_mapper', 'remove_repeat_sentences_mapper',
|
||||
'remove_specific_chars_mapper', 'remove_table_text_mapper', 'remove_words_with_incorrect_substrings_mapper',
|
||||
'replace_content_mapper', 'sdxl_prompt2prompt_mapper', 'sentence_augmentation_mapper', 'sentence_split_mapper',
|
||||
'text_chunk_mapper', 'text_tagging_by_prompt_mapper', 'whitespace_normalization_mapper',
|
||||
'frequency_specified_field_selector', 'random_selector', 'range_specified_field_selector',
|
||||
'tags_specified_field_selector', 'topk_specified_field_selector')
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
||||
SELECT c.id, o.id
|
||||
FROM t_operator_category c
|
||||
CROSS JOIN t_operator o
|
||||
WHERE c.id IN ('de36b61c-9e8a-4422-8c31-d30585c7100f', '9eda9d5d-072b-499b-916c-797a0a8750e1',
|
||||
'96a3b07a-3439-4557-a835-525faad60ca3', '79b385b4-fde8-4617-bcba-02a176938996', '8c09476a-a922-418f-a908-733f8a0de521')
|
||||
AND o.id IN ('image_deduplicator', 'ray_image_deduplicator', 'image_aesthetics_filter', 'image_aspect_ratio_filter',
|
||||
'image_face_count_filter', 'image_face_ratio_filter', 'image_nsfw_filter',
|
||||
'image_pair_similarity_filter', 'image_shape_filter', 'image_size_filter', 'image_watermark_filter',
|
||||
'image_blur_mapper', 'image_detection_yolo_mapper', 'image_face_blur_mapper',
|
||||
'image_remove_background_mapper', 'image_segment_mapper', 'image_tagging_mapper',
|
||||
'imgdiff_difference_area_generator_mapper')
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
||||
SELECT c.id, o.id
|
||||
FROM t_operator_category c
|
||||
CROSS JOIN t_operator o
|
||||
WHERE c.id IN ('42dd9392-73e4-458c-81ff-41751ada47b5', '9eda9d5d-072b-499b-916c-797a0a8750e1',
|
||||
'96a3b07a-3439-4557-a835-525faad60ca3', '79b385b4-fde8-4617-bcba-02a176938996', '8c09476a-a922-418f-a908-733f8a0de521')
|
||||
AND o.id IN ('audio_duration_filter', 'audio_nmf_snr_filter', 'audio_size_filter', 'audio_add_gaussian_noise_mapper',
|
||||
'audio_ffmpeg_wrapped_mapper')
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
||||
SELECT c.id, o.id
|
||||
FROM t_operator_category c
|
||||
CROSS JOIN t_operator o
|
||||
WHERE c.id IN ('a233d584-73c8-4188-ad5d-8f7c8dda9c27', '9eda9d5d-072b-499b-916c-797a0a8750e1',
|
||||
'96a3b07a-3439-4557-a835-525faad60ca3', '79b385b4-fde8-4617-bcba-02a176938996', '8c09476a-a922-418f-a908-733f8a0de521')
|
||||
AND o.id IN ('ray_video_deduplicator', 'video_deduplicator', 'video_aesthetics_filter', 'video_aspect_ratio_filter',
|
||||
'video_duration_filter', 'video_motion_score_filter', 'video_motion_score_raft_filter',
|
||||
'video_nsfw_filter', 'video_ocr_area_ratio_filter', 'video_resolution_filter',
|
||||
'video_tagging_from_frames_filter', 'video_watermark_filter', 'vggt_mapper',
|
||||
'video_depth_estimation_mapper', 'video_face_blur_mapper', 'video_ffmpeg_wrapped_mapper',
|
||||
'video_hand_reconstruction_mapper', 'video_object_segmenting_mapper', 'video_remove_watermark_mapper',
|
||||
'video_resize_aspect_ratio_mapper', 'video_resize_resolution_mapper', 'video_tagging_from_audio_mapper',
|
||||
'video_tagging_from_frames_mapper', 'video_whole_body_pose_estimation_mapper')
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
||||
SELECT c.id, o.id
|
||||
FROM t_operator_category c
|
||||
CROSS JOIN t_operator o
|
||||
WHERE c.id IN ('4d7dbd77-0a92-44f3-9056-2cd62d4a71e4', '9eda9d5d-072b-499b-916c-797a0a8750e1',
|
||||
'96a3b07a-3439-4557-a835-525faad60ca3', '79b385b4-fde8-4617-bcba-02a176938996', '8c09476a-a922-418f-a908-733f8a0de521')
|
||||
AND o.id IN ('image_text_matching_filter', 'image_text_similarity_filter', 'phrase_grounding_recall_filter',
|
||||
'video_frames_text_similarity_filter', 'detect_character_attributes_mapper',
|
||||
'detect_character_locations_mapper', 'detect_main_character_mapper',
|
||||
'image_captioning_from_gpt4v_mapper', 'image_captioning_mapper', 'image_diffusion_mapper',
|
||||
'imgdiff_difference_caption_generator_mapper', 'mllm_mapper', 'video_captioning_from_audio_mapper',
|
||||
'video_captioning_from_frames_mapper', 'video_captioning_from_summarizer_mapper',
|
||||
'video_captioning_from_video_mapper', 'video_captioning_from_vlm_mapper', 'video_extract_frames_mapper',
|
||||
'video_split_by_duration_mapper', 'video_split_by_key_frame_mapper', 'video_split_by_scene_mapper')
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
-- Ensure all operator categories have created_by/updated_by set for data scope filtering
|
||||
UPDATE t_operator_category SET created_by = 'system', updated_by = 'system' WHERE created_by IS NULL OR created_by = '';
|
||||
@@ -0,0 +1,97 @@
|
||||
-- 使用现有的deepdata数据库
|
||||
\c deepdata;
|
||||
|
||||
-- 配比实例表
|
||||
CREATE TABLE IF NOT EXISTS t_st_ratio_instances
|
||||
(
|
||||
id VARCHAR(64) PRIMARY KEY,
|
||||
name VARCHAR(64),
|
||||
description TEXT,
|
||||
target_dataset_id VARCHAR(64),
|
||||
ratio_parameters JSONB,
|
||||
merge_method VARCHAR(50),
|
||||
status VARCHAR(20),
|
||||
totals BIGINT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_st_ratio_instances IS '配比实例表（UUID 主键）';
|
||||
COMMENT ON COLUMN t_st_ratio_instances.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_st_ratio_instances.name IS '名称';
|
||||
COMMENT ON COLUMN t_st_ratio_instances.description IS '描述';
|
||||
COMMENT ON COLUMN t_st_ratio_instances.target_dataset_id IS '模板数据集ID';
|
||||
COMMENT ON COLUMN t_st_ratio_instances.ratio_parameters IS '配比参数';
|
||||
COMMENT ON COLUMN t_st_ratio_instances.merge_method IS '合并方式';
|
||||
COMMENT ON COLUMN t_st_ratio_instances.status IS '状态';
|
||||
COMMENT ON COLUMN t_st_ratio_instances.totals IS '总数';
|
||||
COMMENT ON COLUMN t_st_ratio_instances.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_st_ratio_instances.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_st_ratio_instances.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_st_ratio_instances.updated_by IS '更新者';
|
||||
|
||||
-- 配比关系表
|
||||
CREATE TABLE IF NOT EXISTS t_st_ratio_relations
|
||||
(
|
||||
id VARCHAR(64) PRIMARY KEY,
|
||||
ratio_instance_id VARCHAR(64),
|
||||
source_dataset_id VARCHAR(64),
|
||||
ratio_value VARCHAR(256),
|
||||
counts BIGINT,
|
||||
filter_conditions TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_st_ratio_relations IS '配比关系表（UUID 主键）';
|
||||
COMMENT ON COLUMN t_st_ratio_relations.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_st_ratio_relations.ratio_instance_id IS '配比实例ID';
|
||||
COMMENT ON COLUMN t_st_ratio_relations.source_dataset_id IS '源数据集ID';
|
||||
COMMENT ON COLUMN t_st_ratio_relations.ratio_value IS '配比值';
|
||||
COMMENT ON COLUMN t_st_ratio_relations.counts IS '条数';
|
||||
COMMENT ON COLUMN t_st_ratio_relations.filter_conditions IS '过滤条件';
|
||||
COMMENT ON COLUMN t_st_ratio_relations.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_st_ratio_relations.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_st_ratio_relations.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_st_ratio_relations.updated_by IS '更新者';
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_st_ratio_instances_target_dataset ON t_st_ratio_instances(target_dataset_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_st_ratio_instances_status ON t_st_ratio_instances(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_st_ratio_instances_created_at ON t_st_ratio_instances(created_at);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_st_ratio_relations_instance ON t_st_ratio_relations(ratio_instance_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_st_ratio_relations_source_dataset ON t_st_ratio_relations(source_dataset_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_st_ratio_relations_created_at ON t_st_ratio_relations(created_at);
|
||||
|
||||
-- 创建外键约束
|
||||
ALTER TABLE t_st_ratio_relations ADD CONSTRAINT fk_st_ratio_relations_instance FOREIGN KEY (ratio_instance_id)
|
||||
REFERENCES t_st_ratio_instances(id) ON DELETE CASCADE;
|
||||
|
||||
-- 创建触发器用于自动更新 updated_at
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
-- 为各个表创建触发器
|
||||
DROP TRIGGER IF EXISTS update_st_ratio_instances_updated_at ON t_st_ratio_instances;
|
||||
CREATE TRIGGER update_st_ratio_instances_updated_at
|
||||
BEFORE UPDATE ON t_st_ratio_instances
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
DROP TRIGGER IF EXISTS update_st_ratio_relations_updated_at ON t_st_ratio_relations;
|
||||
CREATE TRIGGER update_st_ratio_relations_updated_at
|
||||
BEFORE UPDATE ON t_st_ratio_relations
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
@@ -0,0 +1,158 @@
|
||||
-- 使用现有的deepdata数据库
|
||||
\c deepdata;
|
||||
|
||||
-- 数据合成任务表
|
||||
CREATE TABLE IF NOT EXISTS t_data_synth_instances
|
||||
(
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
status VARCHAR(20),
|
||||
synth_type VARCHAR(20) NOT NULL,
|
||||
progress INTEGER DEFAULT 0,
|
||||
synth_config JSONB NOT NULL,
|
||||
total_files INTEGER DEFAULT 0,
|
||||
processed_files INTEGER DEFAULT 0,
|
||||
total_chunks INTEGER DEFAULT 0,
|
||||
processed_chunks INTEGER DEFAULT 0,
|
||||
total_synth_data INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_data_synth_instances IS '数据合成任务表（UUID 主键）';
|
||||
COMMENT ON COLUMN t_data_synth_instances.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_data_synth_instances.name IS '任务名称';
|
||||
COMMENT ON COLUMN t_data_synth_instances.description IS '任务描述';
|
||||
COMMENT ON COLUMN t_data_synth_instances.status IS '任务状态';
|
||||
COMMENT ON COLUMN t_data_synth_instances.synth_type IS '合成类型';
|
||||
COMMENT ON COLUMN t_data_synth_instances.progress IS '任务进度(百分比)';
|
||||
COMMENT ON COLUMN t_data_synth_instances.synth_config IS '合成配置';
|
||||
COMMENT ON COLUMN t_data_synth_instances.total_files IS '总文件数';
|
||||
COMMENT ON COLUMN t_data_synth_instances.processed_files IS '已处理文件数';
|
||||
COMMENT ON COLUMN t_data_synth_instances.total_chunks IS '总文本块数';
|
||||
COMMENT ON COLUMN t_data_synth_instances.processed_chunks IS '已处理文本块数';
|
||||
COMMENT ON COLUMN t_data_synth_instances.total_synth_data IS '总合成数据量';
|
||||
COMMENT ON COLUMN t_data_synth_instances.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_data_synth_instances.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_data_synth_instances.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_data_synth_instances.updated_by IS '更新者';
|
||||
|
||||
-- 数据合成文件任务表
|
||||
CREATE TABLE IF NOT EXISTS t_data_synthesis_file_instances
|
||||
(
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
synthesis_instance_id VARCHAR(36),
|
||||
file_name VARCHAR(255) NOT NULL,
|
||||
source_file_id VARCHAR(255) NOT NULL,
|
||||
target_file_location VARCHAR(1000),
|
||||
status VARCHAR(20),
|
||||
total_chunks INTEGER DEFAULT 0,
|
||||
processed_chunks INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_data_synthesis_file_instances IS '数据合成文件任务表（UUID 主键）';
|
||||
COMMENT ON COLUMN t_data_synthesis_file_instances.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_data_synthesis_file_instances.synthesis_instance_id IS '数据合成任务ID';
|
||||
COMMENT ON COLUMN t_data_synthesis_file_instances.file_name IS '文件名';
|
||||
COMMENT ON COLUMN t_data_synthesis_file_instances.source_file_id IS '原始文件ID';
|
||||
COMMENT ON COLUMN t_data_synthesis_file_instances.target_file_location IS '目标文件存储位置';
|
||||
COMMENT ON COLUMN t_data_synthesis_file_instances.status IS '任务状态';
|
||||
COMMENT ON COLUMN t_data_synthesis_file_instances.total_chunks IS '总文本块数';
|
||||
COMMENT ON COLUMN t_data_synthesis_file_instances.processed_chunks IS '已处理文本块数';
|
||||
COMMENT ON COLUMN t_data_synthesis_file_instances.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_data_synthesis_file_instances.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_data_synthesis_file_instances.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_data_synthesis_file_instances.updated_by IS '更新者';
|
||||
|
||||
-- 数据合成分块任务表
|
||||
CREATE TABLE IF NOT EXISTS t_data_synthesis_chunk_instances
|
||||
(
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
synthesis_file_instance_id VARCHAR(36),
|
||||
chunk_index INTEGER,
|
||||
chunk_content TEXT,
|
||||
metadata JSONB
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_data_synthesis_chunk_instances IS '数据合成分块任务表（UUID 主键）';
|
||||
COMMENT ON COLUMN t_data_synthesis_chunk_instances.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_data_synthesis_chunk_instances.synthesis_file_instance_id IS '数据合成文件任务ID';
|
||||
COMMENT ON COLUMN t_data_synthesis_chunk_instances.chunk_index IS '分块索引';
|
||||
COMMENT ON COLUMN t_data_synthesis_chunk_instances.chunk_content IS '分块内容';
|
||||
COMMENT ON COLUMN t_data_synthesis_chunk_instances.metadata IS '分块元数据';
|
||||
|
||||
-- 数据合成任务队列表
|
||||
CREATE TABLE IF NOT EXISTS t_data_synthesis_data
|
||||
(
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
data JSONB,
|
||||
synthesis_file_instance_id VARCHAR(36),
|
||||
chunk_instance_id VARCHAR(36)
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_data_synthesis_data IS '数据合成任务队列表（UUID 主键）';
|
||||
COMMENT ON COLUMN t_data_synthesis_data.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_data_synthesis_data.data IS '合成的数据';
|
||||
COMMENT ON COLUMN t_data_synthesis_data.synthesis_file_instance_id IS '数据合成文件任务ID';
|
||||
COMMENT ON COLUMN t_data_synthesis_data.chunk_instance_id IS '分块任务ID';
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_synth_instances_status ON t_data_synth_instances(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_synth_instances_created_at ON t_data_synth_instances(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_synth_instances_synth_type ON t_data_synth_instances(synth_type);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_synth_file_instances_instance ON t_data_synthesis_file_instances(synthesis_instance_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_synth_file_instances_status ON t_data_synthesis_file_instances(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_synth_file_instances_source_file ON t_data_synthesis_file_instances(source_file_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_synth_chunk_instances_file ON t_data_synthesis_chunk_instances(synthesis_file_instance_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_synth_chunk_instances_index ON t_data_synthesis_chunk_instances(chunk_index);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_synth_data_file ON t_data_synthesis_data(synthesis_file_instance_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_synth_data_chunk ON t_data_synthesis_data(chunk_instance_id);
|
||||
|
||||
-- 创建外键约束
|
||||
ALTER TABLE t_data_synthesis_file_instances ADD CONSTRAINT fk_synth_file_instances_instance
|
||||
FOREIGN KEY (synthesis_instance_id) REFERENCES t_data_synth_instances(id) ON DELETE CASCADE;
|
||||
|
||||
ALTER TABLE t_data_synthesis_chunk_instances ADD CONSTRAINT fk_synth_chunk_instances_file
|
||||
FOREIGN KEY (synthesis_file_instance_id) REFERENCES t_data_synthesis_file_instances(id) ON DELETE CASCADE;
|
||||
|
||||
ALTER TABLE t_data_synthesis_data ADD CONSTRAINT fk_synth_data_file FOREIGN KEY (synthesis_file_instance_id)
|
||||
REFERENCES t_data_synthesis_file_instances(id) ON DELETE CASCADE;
|
||||
|
||||
ALTER TABLE t_data_synthesis_data ADD CONSTRAINT fk_synth_data_chunk FOREIGN KEY (chunk_instance_id)
|
||||
REFERENCES t_data_synthesis_chunk_instances(id) ON DELETE CASCADE;
|
||||
|
||||
-- 创建触发器用于自动更新 updated_at
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
-- 为有 updated_at 字段的表创建触发器
|
||||
DROP TRIGGER IF EXISTS update_synth_instances_updated_at ON t_data_synth_instances;
|
||||
CREATE TRIGGER update_synth_instances_updated_at
|
||||
BEFORE UPDATE ON t_data_synth_instances
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
DROP TRIGGER IF EXISTS update_synth_file_instances_updated_at ON t_data_synthesis_file_instances;
|
||||
CREATE TRIGGER update_synth_file_instances_updated_at
|
||||
BEFORE UPDATE ON t_data_synthesis_file_instances
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
@@ -0,0 +1,99 @@
|
||||
-- 使用现有的deepdata数据库
|
||||
\c deepdata;
|
||||
|
||||
-- 知识库表
|
||||
CREATE TABLE IF NOT EXISTS t_rag_knowledge_base
|
||||
(
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
type VARCHAR(50) NOT NULL,
|
||||
description VARCHAR(512),
|
||||
embedding_model VARCHAR(255) NOT NULL,
|
||||
chat_model VARCHAR(255),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_rag_knowledge_base IS '知识库表';
|
||||
COMMENT ON COLUMN t_rag_knowledge_base.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_rag_knowledge_base.name IS '知识库名称';
|
||||
COMMENT ON COLUMN t_rag_knowledge_base.type IS '知识库类型';
|
||||
COMMENT ON COLUMN t_rag_knowledge_base.description IS '知识库描述';
|
||||
COMMENT ON COLUMN t_rag_knowledge_base.embedding_model IS '嵌入模型';
|
||||
COMMENT ON COLUMN t_rag_knowledge_base.chat_model IS '聊天模型';
|
||||
COMMENT ON COLUMN t_rag_knowledge_base.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_rag_knowledge_base.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_rag_knowledge_base.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_rag_knowledge_base.updated_by IS '更新者';
|
||||
|
||||
-- 知识库文件表（修正表名注释为"知识库文件表"）
|
||||
CREATE TABLE IF NOT EXISTS t_rag_file
|
||||
(
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
knowledge_base_id VARCHAR(36) NOT NULL,
|
||||
file_name VARCHAR(255) NOT NULL,
|
||||
file_id VARCHAR(255) NOT NULL,
|
||||
chunk_count INTEGER,
|
||||
metadata JSONB,
|
||||
status VARCHAR(50),
|
||||
err_msg TEXT,
|
||||
progress INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_rag_file IS '知识库文件表';
|
||||
COMMENT ON COLUMN t_rag_file.id IS 'UUID';
|
||||
COMMENT ON COLUMN t_rag_file.knowledge_base_id IS '知识库ID';
|
||||
COMMENT ON COLUMN t_rag_file.file_name IS '文件名';
|
||||
COMMENT ON COLUMN t_rag_file.file_id IS '文件ID';
|
||||
COMMENT ON COLUMN t_rag_file.chunk_count IS '切片数';
|
||||
COMMENT ON COLUMN t_rag_file.metadata IS '元数据';
|
||||
COMMENT ON COLUMN t_rag_file.status IS '文件状态';
|
||||
COMMENT ON COLUMN t_rag_file.err_msg IS '错误信息';
|
||||
COMMENT ON COLUMN t_rag_file.progress IS '处理进度(0-100)';
|
||||
COMMENT ON COLUMN t_rag_file.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_rag_file.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_rag_file.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_rag_file.updated_by IS '更新者';
|
||||
|
||||
-- 创建外键约束
|
||||
ALTER TABLE t_rag_file
|
||||
ADD CONSTRAINT fk_rag_file_knowledge_base
|
||||
FOREIGN KEY (knowledge_base_id)
|
||||
REFERENCES t_rag_knowledge_base(id)
|
||||
ON DELETE CASCADE;
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_rag_file_knowledge_base ON t_rag_file(knowledge_base_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_rag_file_status ON t_rag_file(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_rag_file_created_at ON t_rag_file(created_at);
|
||||
|
||||
-- 创建触发器用于自动更新 updated_at
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
-- 为 t_rag_knowledge_base 表创建触发器
|
||||
DROP TRIGGER IF EXISTS update_rag_knowledge_base_updated_at ON t_rag_knowledge_base;
|
||||
CREATE TRIGGER update_rag_knowledge_base_updated_at
|
||||
BEFORE UPDATE ON t_rag_knowledge_base
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- 为 t_rag_file 表创建触发器
|
||||
DROP TRIGGER IF EXISTS update_rag_file_updated_at ON t_rag_file;
|
||||
CREATE TRIGGER update_rag_file_updated_at
|
||||
BEFORE UPDATE ON t_rag_file
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
@@ -0,0 +1,111 @@
|
||||
-- 使用现有的deepdata数据库
|
||||
\c deepdata;
|
||||
|
||||
-- 模型配置表
|
||||
CREATE TABLE IF NOT EXISTS t_models
|
||||
(
|
||||
id VARCHAR(36) PRIMARY KEY,
|
||||
model_name VARCHAR(100) NOT NULL,
|
||||
provider VARCHAR(50) NOT NULL,
|
||||
base_url VARCHAR(255) NOT NULL,
|
||||
api_key VARCHAR(512) DEFAULT '',
|
||||
type VARCHAR(50) NOT NULL,
|
||||
is_enabled BOOLEAN DEFAULT TRUE,
|
||||
is_default BOOLEAN DEFAULT FALSE,
|
||||
is_deleted BOOLEAN DEFAULT FALSE,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_models IS '模型配置表';
|
||||
COMMENT ON COLUMN t_models.id IS '主键ID';
|
||||
COMMENT ON COLUMN t_models.model_name IS '模型名称（如 qwen2）';
|
||||
COMMENT ON COLUMN t_models.provider IS '模型提供商（如 Ollama、OpenAI、DeepSeek）';
|
||||
COMMENT ON COLUMN t_models.base_url IS 'API 基础地址';
|
||||
COMMENT ON COLUMN t_models.api_key IS 'API 密钥（无密钥则为空）';
|
||||
COMMENT ON COLUMN t_models.type IS '模型类型（如 chat、embedding）';
|
||||
COMMENT ON COLUMN t_models.is_enabled IS '是否启用：1-启用，0-禁用';
|
||||
COMMENT ON COLUMN t_models.is_default IS '是否默认：1-默认，0-非默认';
|
||||
COMMENT ON COLUMN t_models.is_deleted IS '是否删除：1-已删除，0-未删除';
|
||||
COMMENT ON COLUMN t_models.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_models.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_models.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_models.updated_by IS '更新者';
|
||||
|
||||
-- 添加唯一约束
|
||||
ALTER TABLE t_models
|
||||
ADD CONSTRAINT uk_model_provider
|
||||
UNIQUE (model_name, provider, created_by);
|
||||
|
||||
COMMENT ON CONSTRAINT uk_model_provider ON t_models
|
||||
IS '避免同一提供商下模型名称重复';
|
||||
|
||||
-- 创建触发器用于自动更新 updated_at
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
CREATE TRIGGER update_t_models_updated_at
|
||||
BEFORE UPDATE ON t_models
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- 设置管理表
|
||||
CREATE TABLE IF NOT EXISTS t_sys_param
|
||||
(
|
||||
id VARCHAR(100) PRIMARY KEY,
|
||||
param_value TEXT NOT NULL,
|
||||
param_type VARCHAR(50) DEFAULT 'string',
|
||||
option_list TEXT,
|
||||
description VARCHAR(255) DEFAULT '',
|
||||
is_built_in BOOLEAN DEFAULT FALSE,
|
||||
can_modify BOOLEAN DEFAULT TRUE,
|
||||
is_enabled BOOLEAN DEFAULT TRUE,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by VARCHAR(255),
|
||||
updated_by VARCHAR(255)
|
||||
);
|
||||
|
||||
-- 添加注释
|
||||
COMMENT ON TABLE t_sys_param IS '设置管理表';
|
||||
COMMENT ON COLUMN t_sys_param.id IS '主键ID,设置项键';
|
||||
COMMENT ON COLUMN t_sys_param.param_value IS '设置项值';
|
||||
COMMENT ON COLUMN t_sys_param.param_type IS '设置项类型（仅 string、number、boolean 三种类型）';
|
||||
COMMENT ON COLUMN t_sys_param.option_list IS '选项列表（逗号分隔，仅对 enum 类型有效）';
|
||||
COMMENT ON COLUMN t_sys_param.description IS '设置项描述';
|
||||
COMMENT ON COLUMN t_sys_param.is_built_in IS '是否内置：1-是，0-否';
|
||||
COMMENT ON COLUMN t_sys_param.can_modify IS '是否可修改：1-可修改，0-不可修改';
|
||||
COMMENT ON COLUMN t_sys_param.is_enabled IS '是否启用：1-启用，0-禁用';
|
||||
COMMENT ON COLUMN t_sys_param.created_at IS '创建时间';
|
||||
COMMENT ON COLUMN t_sys_param.updated_at IS '更新时间';
|
||||
COMMENT ON COLUMN t_sys_param.created_by IS '创建者';
|
||||
COMMENT ON COLUMN t_sys_param.updated_by IS '更新者';
|
||||
|
||||
-- 创建触发器
|
||||
DROP TRIGGER IF EXISTS update_t_sys_param_updated_at ON t_sys_param;
|
||||
CREATE TRIGGER update_t_sys_param_updated_at
|
||||
BEFORE UPDATE ON t_sys_param
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- 插入初始数据
|
||||
INSERT INTO t_sys_param (id, param_value, param_type, option_list, description, is_built_in,
|
||||
can_modify, is_enabled, created_by, updated_by)
|
||||
VALUES
|
||||
('sys.knowledge.base.count', '200', 'number', '10,200,500', '知识库最大数量', true, true, true, 'system', 'system'),
|
||||
('SEARCH_API', 'tavily', 'string', 'tavily,infoquest,duckduckgo,brave_search,arxiv', 'deepflow使用的搜索引擎', true, true, true, 'system', 'system'),
|
||||
('TAVILY_API_KEY', 'tvly-dev-xxx', 'string', '', 'deepflow使用的搜索引擎所需的apiKey', true, true, true, 'system', 'system'),
|
||||
('BRAVE_SEARCH_API_KEY', 'api-xxx', 'string', '', 'deepflow使用的搜索引擎所需的apiKey', true, true, true, 'system', 'system'),
|
||||
('JINA_API_KEY', '', 'string', '', 'deepflow使用的JINA搜索引擎所需的apiKey', true, true, true, 'system', 'system'),
|
||||
('sys.management.dataset.pvc.name', 'deepdata-dataset-pvc', 'string', '', '数据集所在pvc名称', true, false,true, 'system', 'system'),
|
||||
('DATA_JUICER_EXECUTOR', 'default', 'string', 'default,ray', 'data-juicer使用的执行器', true, true, true, 'system', 'system'),
|
||||
('sys.home.page.url', '', 'string', '', '首页URL，访问根路径时自动重定向到此URL（为空则不重定向）', true, true, true, 'system', 'system')
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/bin/bash
|
||||
# 遇到错误即退出
|
||||
set -e
|
||||
|
||||
# 使用 -n 判断环境变量 HOME_PAGE_URL 是否非空（即已设置且有值）
|
||||
if [ -n "$HOME_PAGE_URL" ]; then
|
||||
echo "检测到 HOME_PAGE_URL 环境变量已配置，值为: $HOME_PAGE_URL。准备更新数据库..."
|
||||
|
||||
# 只有变量非空时，才会进入这里执行 SQL
|
||||
psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "deepdata" -v my_url="$HOME_PAGE_URL" <<-EOSQL
|
||||
|
||||
UPDATE t_sys_param
|
||||
SET param_value = :'my_url'
|
||||
WHERE id = 'sys.home.page.url';
|
||||
|
||||
EOSQL
|
||||
|
||||
echo "sys.home.page.url 更新完成！"
|
||||
else
|
||||
# 如果变量为空或未设置，打印提示并直接跳过
|
||||
echo "未配置 HOME_PAGE_URL 环境变量或值为空，跳过 sys.home.page.url 的更新操作。"
|
||||
fi
|
||||
Reference in New Issue
Block a user