figmar/deepdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(deepdata): 引入修改密码、文件预览组件与清洗算子增强，deepflow 改名为 deep-flow

Browse Source 
本次同步 09-12 至 09-28 的代码变更，主要内容：

新增功能
- 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService
  对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口
- 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、
  ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器
- 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator
- 新增 tools/minio-file-server MinIO 文件服务

功能增强
- 数据清洗：cleaning_task_service/scheduler/routes 重构，
  data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整
- 数据标注：annotation sync.py、project.py 接口与schema 更新
- 数据采集、算子市场、首页与菜单路由相关页面调整
- DataJuicer 算子接入（useOperatorOperations、dj_ops_schema）

结构调整
- deepflow/ 更名为 deep-flow/（419 个文件重命名）
- 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本

未纳入本次提交
- scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录）
- *.backup*、__pycache__ 产物
- zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物）
- DataX/（内嵌独立仓库，远程亦未收录其源码）
This commit is contained in:
沃巴蒂
2026-10-03 22:44:56 +08:00
parent cbb1048e18
commit ff22ed3a8b
1156 changed files with 129912 additions and 21716 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/db/00-database-init.sql
+4 -4
View File
@@ -1,4 +1,4 @@
\set ON_ERROR_STOP 0
CREATE DATABASE deepdata;
CREATE DATABASE labelstudio;
\set ON_ERROR_STOP 1
\set ON_ERROR_STOP 0
CREATE DATABASE deepdata;
CREATE DATABASE labelstudio;
\set ON_ERROR_STOP 1
scripts/db/data-annotation-init.sql
+451 -451
View File
@@ -1,451 +1,451 @@
-- 使用现有的deepdata数据库
\c deepdata;
-- 标注配置模板表
CREATE TABLE IF NOT EXISTS t_dm_annotation_templates (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(100) NOT NULL,
description VARCHAR(500),
data_type VARCHAR(50) NOT NULL,
labeling_type VARCHAR(50) NOT NULL,
configuration JSONB NOT NULL,
style VARCHAR(32) NOT NULL,
category VARCHAR(50) DEFAULT 'custom',
built_in BOOLEAN DEFAULT FALSE,
version VARCHAR(20) DEFAULT '1.0',
created_by VARCHAR(256),
updated_by VARCHAR(256),
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP
);
-- 添加注释
COMMENT ON TABLE t_dm_annotation_templates IS '标注配置模板表';
COMMENT ON COLUMN t_dm_annotation_templates.id IS 'UUID';
COMMENT ON COLUMN t_dm_annotation_templates.name IS '模板名称';
COMMENT ON COLUMN t_dm_annotation_templates.description IS '模板描述';
COMMENT ON COLUMN t_dm_annotation_templates.data_type IS '数据类型: image/text/audio/video/timeseries';
COMMENT ON COLUMN t_dm_annotation_templates.labeling_type IS '标注类型: classification/detection/segmentation/ner/relation/etc';
COMMENT ON COLUMN t_dm_annotation_templates.configuration IS '标注配置（包含labels定义等）';
COMMENT ON COLUMN t_dm_annotation_templates.style IS '样式配置: horizontal/vertical';
COMMENT ON COLUMN t_dm_annotation_templates.category IS '模板分类: medical/general/custom/system';
COMMENT ON COLUMN t_dm_annotation_templates.built_in IS '是否系统内置模板';
COMMENT ON COLUMN t_dm_annotation_templates.version IS '模板版本';
COMMENT ON COLUMN t_dm_annotation_templates.created_by IS '创建者';
COMMENT ON COLUMN t_dm_annotation_templates.updated_by IS '更新者';
COMMENT ON COLUMN t_dm_annotation_templates.created_at IS '创建时间';
COMMENT ON COLUMN t_dm_annotation_templates.updated_at IS '更新时间';
COMMENT ON COLUMN t_dm_annotation_templates.deleted_at IS '删除时间（软删除）';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dm_ann_tpl_data_type ON t_dm_annotation_templates(data_type);
CREATE INDEX IF NOT EXISTS idx_dm_ann_tpl_labeling_type ON t_dm_annotation_templates(labeling_type);
CREATE INDEX IF NOT EXISTS idx_dm_ann_tpl_category ON t_dm_annotation_templates(category);
CREATE INDEX IF NOT EXISTS idx_dm_ann_tpl_built_in ON t_dm_annotation_templates(built_in);
CREATE INDEX IF NOT EXISTS idx_dm_ann_tpl_deleted_at ON t_dm_annotation_templates(deleted_at);
-- 标注项目表
CREATE TABLE IF NOT EXISTS t_dm_labeling_projects (
id VARCHAR(36) PRIMARY KEY,
dataset_id VARCHAR(36) NOT NULL,
name VARCHAR(100) NOT NULL,
labeling_project_id VARCHAR(8) NOT NULL,
template_id VARCHAR(36),
configuration JSONB,
progress JSONB,
created_by VARCHAR(256),
updated_by VARCHAR(256),
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP
);
-- 添加注释
COMMENT ON TABLE t_dm_labeling_projects IS '标注项目表';
COMMENT ON COLUMN t_dm_labeling_projects.id IS 'UUID';
COMMENT ON COLUMN t_dm_labeling_projects.dataset_id IS '数据集ID';
COMMENT ON COLUMN t_dm_labeling_projects.name IS '项目名称';
COMMENT ON COLUMN t_dm_labeling_projects.labeling_project_id IS 'Label Studio项目ID';
COMMENT ON COLUMN t_dm_labeling_projects.template_id IS '使用的模板ID';
COMMENT ON COLUMN t_dm_labeling_projects.configuration IS '项目配置（可能包含对模板的自定义修改）';
COMMENT ON COLUMN t_dm_labeling_projects.progress IS '项目进度信息';
COMMENT ON COLUMN t_dm_labeling_projects.created_by IS '创建者';
COMMENT ON COLUMN t_dm_labeling_projects.updated_by IS '更新者';
COMMENT ON COLUMN t_dm_labeling_projects.created_at IS '创建时间';
COMMENT ON COLUMN t_dm_labeling_projects.updated_at IS '更新时间';
COMMENT ON COLUMN t_dm_labeling_projects.deleted_at IS '删除时间（软删除）';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dm_lp_dataset_id ON t_dm_labeling_projects(dataset_id);
CREATE INDEX IF NOT EXISTS idx_dm_lp_template_id ON t_dm_labeling_projects(template_id);
CREATE INDEX IF NOT EXISTS idx_dm_lp_labeling_project_id ON t_dm_labeling_projects(labeling_project_id);
CREATE INDEX IF NOT EXISTS idx_dm_lp_deleted_at ON t_dm_labeling_projects(deleted_at);
-- 外键约束
ALTER TABLE t_dm_labeling_projects ADD CONSTRAINT fk_dm_labeling_projects_template
FOREIGN KEY (template_id) REFERENCES t_dm_annotation_templates(id) ON DELETE SET NULL;
-- 自动标注任务表
CREATE TABLE IF NOT EXISTS t_dm_auto_annotation_tasks (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
dataset_id VARCHAR(36) NOT NULL,
dataset_name VARCHAR(255),
config JSONB NOT NULL,
file_ids JSONB,
status VARCHAR(50) NOT NULL DEFAULT 'pending',
progress INTEGER DEFAULT 0,
total_images INTEGER DEFAULT 0,
processed_images INTEGER DEFAULT 0,
detected_objects INTEGER DEFAULT 0,
output_path VARCHAR(500),
error_message TEXT,
created_by VARCHAR(256),
updated_by VARCHAR(256),
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
completed_at TIMESTAMP,
deleted_at TIMESTAMP
);
-- 添加注释
COMMENT ON TABLE t_dm_auto_annotation_tasks IS '自动标注任务表';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.id IS 'UUID';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.name IS '任务名称';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.dataset_id IS '数据集ID';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.dataset_name IS '数据集名称（冗余字段，方便查询）';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.config IS '任务配置（模型规模、置信度等）';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.file_ids IS '要处理的文件ID列表，为空则处理数据集所有图像';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.status IS '任务状态: pending/running/completed/failed';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.progress IS '任务进度 0-100';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.total_images IS '总图片数';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.processed_images IS '已处理图片数';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.detected_objects IS '检测到的对象总数';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.output_path IS '输出路径';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.error_message IS '错误信息';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.created_by IS '创建者';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.updated_by IS '更新者';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.created_at IS '创建时间';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.updated_at IS '更新时间';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.completed_at IS '完成时间';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.deleted_at IS '删除时间（软删除）';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dm_aat_dataset_id ON t_dm_auto_annotation_tasks(dataset_id);
CREATE INDEX IF NOT EXISTS idx_dm_aat_status ON t_dm_auto_annotation_tasks(status);
CREATE INDEX IF NOT EXISTS idx_dm_aat_created_at ON t_dm_auto_annotation_tasks(created_at);
CREATE INDEX IF NOT EXISTS idx_dm_aat_deleted_at ON t_dm_auto_annotation_tasks(deleted_at);
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- 为各个表创建触发器
DROP TRIGGER IF EXISTS update_dm_annotation_templates_updated_at ON t_dm_annotation_templates;
CREATE TRIGGER update_dm_annotation_templates_updated_at
BEFORE UPDATE ON t_dm_annotation_templates
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_dm_labeling_projects_updated_at ON t_dm_labeling_projects;
CREATE TRIGGER update_dm_labeling_projects_updated_at
BEFORE UPDATE ON t_dm_labeling_projects
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_dm_auto_annotation_tasks_updated_at ON t_dm_auto_annotation_tasks;
CREATE TRIGGER update_dm_auto_annotation_tasks_updated_at
BEFORE UPDATE ON t_dm_auto_annotation_tasks
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 内置标注模板初始化数据
-- 1. 图像分类模板
INSERT INTO t_dm_annotation_templates (id, name, description, data_type, labeling_type, configuration, style, category, built_in, version)
VALUES ('tpl-image-classification-001', '图像分类', '简单的多标签图像分类模板', '图像', '分类', '{
"labels": [
{
"fromName": "choice",
"toName": "image",
"type": "Choices",
"options": ["Cat", "Dog", "Bird", "Other"],
"required": true,
"description": "选择最符合图像内容的标签"
}
],
"objects": [
{
"name": "image",
"type": "Image",
"value": "$image"
}
]
}',
'horizontal',
'计算机视觉',
TRUE,
'1.0.0'
),
(
'tpl-object-detection-001',
'目标检测（边界框）',
'使用矩形边界框进行目标检测',
'图像',
'目标检测',
'{
"labels": [
{
"fromName": "label",
"toName": "image",
"type": "RectangleLabels",
"labels": ["Person", "Vehicle", "Animal", "Object"],
"required": false,
"description": "在图像中框出目标并标注类别"
}
],
"objects": [
{
"name": "image",
"type": "Image",
"value": "$image"
}
]
}',
'horizontal',
'计算机视觉',
TRUE,
'1.0.0'
),
(
'tpl-image-segmentation-001',
'图像分割（多边形）',
'使用多边形标注进行语义分割',
'图像',
'分割',
'{
"labels": [
{
"fromName": "label",
"toName": "image",
"type": "PolygonLabels",
"labels": ["Background", "Foreground", "Person", "Car"],
"required": false,
"description": "使用多边形框选需要分割的区域"
}
],
"objects": [
{
"name": "image",
"type": "Image",
"value": "$image"
}
]
}',
'horizontal',
'计算机视觉',
TRUE,
'1.0.0'
),
(
'tpl-text-classification-001',
'文本情感分类',
'将文本中表达的情感划分到预定义的类别',
'文本',
'分类',
'{
"labels": [
{
"fromName": "choice",
"toName": "text",
"type": "Choices",
"options": ["Positive", "Negative", "Neutral"],
"required": true,
"description": "对文本的情感或类别进行选择"
}
],
"objects": [
{
"name": "text",
"type": "Text",
"value": "$text"
}
]
}',
'vertical',
'自然语言处理',
TRUE,
'1.0.0'
),
(
'tpl-ner-001',
'命名实体识别',
'从文本中抽取并标注命名实体',
'文本',
'实体识别',
'{
"labels": [
{
"fromName": "label",
"toName": "text",
"type": "Labels",
"labels": ["PERSON", "ORG", "LOC", "DATE", "MISC"],
"required": false,
"description": "在文本中标注人物、地点等实体"
}
],
"objects": [
{
"name": "text",
"type": "Text",
"value": "$text"
}
]
}',
'vertical',
'自然语言处理',
TRUE,
'1.0.0'
),
(
'tpl-audio-classification-001',
'音频分类',
'将音频片段划分到不同类别',
'音频',
'分类',
'{
"labels": [
{
"fromName": "choice",
"toName": "audio",
"type": "Choices",
"options": ["Speech", "Music", "Noise", "Silence"],
"required": true,
"description": "选择音频片段对应的类别"
}
],
"objects": [
{
"name": "audio",
"type": "Audio",
"value": "$audio"
}
]
}',
'horizontal',
'音频',
TRUE,
'1.0.0'
),
(
'tpl-text-multilabel-001',
'文本多标签分类',
'可为文本选择多个标签，适用于主题、内容类别等多标签任务',
'文本',
'分类',
'{
"labels": [
{
"fromName": "labels",
"toName": "text",
"type": "Choices",
"options": ["Sports", "Politics", "Tech", "Entertainment"],
"required": true,
"choice": "multiple",
"description": "可选择多个标签"
}
],
"objects": [
{
"name": "text",
"type": "Text",
"value": "$text"
}
]
}',
'vertical',
'自然语言处理',
TRUE,
'1.0.0'
),
(
'tpl-text-summarization-001',
'文本摘要',
'根据原文撰写简要摘要',
'文本',
'摘要',
'{
"labels": [
{
"fromName": "summary",
"toName": "text",
"type": "TextArea",
"required": true,
"description": "在此填写摘要内容"
}
],
"objects": [
{
"name": "text",
"type": "Text",
"value": "$text"
}
]
}',
'vertical',
'自然语言处理',
TRUE,
'1.0.0'
),
(
'tpl-keyword-extract-001',
'关键词抽取',
'从文本中选出关键词或关键短语',
'文本',
'实体识别',
'{
"labels": [
{
"fromName": "kw",
"toName": "text",
"type": "Labels",
"labels": ["Keyword"],
"required": false,
"description": "高亮文本并标注关键词"
}
],
"objects": [
{
"name": "text",
"type": "Text",
"value": "$text"
}
]
}',
'vertical',
'自然语言处理',
TRUE,
'1.0.0'
)
ON CONFLICT (id) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
data_type = EXCLUDED.data_type,
labeling_type = EXCLUDED.labeling_type,
configuration = EXCLUDED.configuration,
style = EXCLUDED.style,
category = EXCLUDED.category,
built_in = EXCLUDED.built_in,
version = EXCLUDED.version,
created_by = COALESCE(EXCLUDED.created_by, 'system'),
updated_by = COALESCE(EXCLUDED.updated_by, 'system'),
updated_at = CURRENT_TIMESTAMP;
-- Ensure all annotation templates have created_by/updated_by set for data scope filtering
UPDATE t_dm_annotation_templates SET created_by = 'system', updated_by = 'system' WHERE created_by IS NULL OR created_by = '';
-- 使用现有的DeepData数据库
\c deepdata;
-- 标注配置模板表
CREATE TABLE IF NOT EXISTS t_dm_annotation_templates (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(100) NOT NULL,
description VARCHAR(500),
data_type VARCHAR(50) NOT NULL,
labeling_type VARCHAR(50) NOT NULL,
configuration JSONB NOT NULL,
style VARCHAR(32) NOT NULL,
category VARCHAR(50) DEFAULT 'custom',
built_in BOOLEAN DEFAULT FALSE,
version VARCHAR(20) DEFAULT '1.0',
created_by VARCHAR(256),
updated_by VARCHAR(256),
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP
);
-- 添加注释
COMMENT ON TABLE t_dm_annotation_templates IS '标注配置模板表';
COMMENT ON COLUMN t_dm_annotation_templates.id IS 'UUID';
COMMENT ON COLUMN t_dm_annotation_templates.name IS '模板名称';
COMMENT ON COLUMN t_dm_annotation_templates.description IS '模板描述';
COMMENT ON COLUMN t_dm_annotation_templates.data_type IS '数据类型: image/text/audio/video/timeseries';
COMMENT ON COLUMN t_dm_annotation_templates.labeling_type IS '标注类型: classification/detection/segmentation/ner/relation/etc';
COMMENT ON COLUMN t_dm_annotation_templates.configuration IS '标注配置（包含labels定义等）';
COMMENT ON COLUMN t_dm_annotation_templates.style IS '样式配置: horizontal/vertical';
COMMENT ON COLUMN t_dm_annotation_templates.category IS '模板分类: medical/general/custom/system';
COMMENT ON COLUMN t_dm_annotation_templates.built_in IS '是否系统内置模板';
COMMENT ON COLUMN t_dm_annotation_templates.version IS '模板版本';
COMMENT ON COLUMN t_dm_annotation_templates.created_by IS '创建者';
COMMENT ON COLUMN t_dm_annotation_templates.updated_by IS '更新者';
COMMENT ON COLUMN t_dm_annotation_templates.created_at IS '创建时间';
COMMENT ON COLUMN t_dm_annotation_templates.updated_at IS '更新时间';
COMMENT ON COLUMN t_dm_annotation_templates.deleted_at IS '删除时间（软删除）';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dm_ann_tpl_data_type ON t_dm_annotation_templates(data_type);
CREATE INDEX IF NOT EXISTS idx_dm_ann_tpl_labeling_type ON t_dm_annotation_templates(labeling_type);
CREATE INDEX IF NOT EXISTS idx_dm_ann_tpl_category ON t_dm_annotation_templates(category);
CREATE INDEX IF NOT EXISTS idx_dm_ann_tpl_built_in ON t_dm_annotation_templates(built_in);
CREATE INDEX IF NOT EXISTS idx_dm_ann_tpl_deleted_at ON t_dm_annotation_templates(deleted_at);
-- 标注项目表
CREATE TABLE IF NOT EXISTS t_dm_labeling_projects (
id VARCHAR(36) PRIMARY KEY,
dataset_id VARCHAR(36) NOT NULL,
name VARCHAR(100) NOT NULL,
labeling_project_id VARCHAR(8) NOT NULL,
template_id VARCHAR(36),
configuration JSONB,
progress JSONB,
created_by VARCHAR(256),
updated_by VARCHAR(256),
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
deleted_at TIMESTAMP
);
-- 添加注释
COMMENT ON TABLE t_dm_labeling_projects IS '标注项目表';
COMMENT ON COLUMN t_dm_labeling_projects.id IS 'UUID';
COMMENT ON COLUMN t_dm_labeling_projects.dataset_id IS '数据集ID';
COMMENT ON COLUMN t_dm_labeling_projects.name IS '项目名称';
COMMENT ON COLUMN t_dm_labeling_projects.labeling_project_id IS 'Label Studio项目ID';
COMMENT ON COLUMN t_dm_labeling_projects.template_id IS '使用的模板ID';
COMMENT ON COLUMN t_dm_labeling_projects.configuration IS '项目配置（可能包含对模板的自定义修改）';
COMMENT ON COLUMN t_dm_labeling_projects.progress IS '项目进度信息';
COMMENT ON COLUMN t_dm_labeling_projects.created_by IS '创建者';
COMMENT ON COLUMN t_dm_labeling_projects.updated_by IS '更新者';
COMMENT ON COLUMN t_dm_labeling_projects.created_at IS '创建时间';
COMMENT ON COLUMN t_dm_labeling_projects.updated_at IS '更新时间';
COMMENT ON COLUMN t_dm_labeling_projects.deleted_at IS '删除时间（软删除）';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dm_lp_dataset_id ON t_dm_labeling_projects(dataset_id);
CREATE INDEX IF NOT EXISTS idx_dm_lp_template_id ON t_dm_labeling_projects(template_id);
CREATE INDEX IF NOT EXISTS idx_dm_lp_labeling_project_id ON t_dm_labeling_projects(labeling_project_id);
CREATE INDEX IF NOT EXISTS idx_dm_lp_deleted_at ON t_dm_labeling_projects(deleted_at);
-- 外键约束
ALTER TABLE t_dm_labeling_projects ADD CONSTRAINT fk_dm_labeling_projects_template
FOREIGN KEY (template_id) REFERENCES t_dm_annotation_templates(id) ON DELETE SET NULL;
-- 自动标注任务表
CREATE TABLE IF NOT EXISTS t_dm_auto_annotation_tasks (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
dataset_id VARCHAR(36) NOT NULL,
dataset_name VARCHAR(255),
config JSONB NOT NULL,
file_ids JSONB,
status VARCHAR(50) NOT NULL DEFAULT 'pending',
progress INTEGER DEFAULT 0,
total_images INTEGER DEFAULT 0,
processed_images INTEGER DEFAULT 0,
detected_objects INTEGER DEFAULT 0,
output_path VARCHAR(500),
error_message TEXT,
created_by VARCHAR(256),
updated_by VARCHAR(256),
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
completed_at TIMESTAMP,
deleted_at TIMESTAMP
);
-- 添加注释
COMMENT ON TABLE t_dm_auto_annotation_tasks IS '自动标注任务表';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.id IS 'UUID';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.name IS '任务名称';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.dataset_id IS '数据集ID';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.dataset_name IS '数据集名称（冗余字段，方便查询）';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.config IS '任务配置（模型规模、置信度等）';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.file_ids IS '要处理的文件ID列表，为空则处理数据集所有图像';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.status IS '任务状态: pending/running/completed/failed';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.progress IS '任务进度 0-100';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.total_images IS '总图片数';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.processed_images IS '已处理图片数';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.detected_objects IS '检测到的对象总数';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.output_path IS '输出路径';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.error_message IS '错误信息';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.created_by IS '创建者';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.updated_by IS '更新者';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.created_at IS '创建时间';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.updated_at IS '更新时间';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.completed_at IS '完成时间';
COMMENT ON COLUMN t_dm_auto_annotation_tasks.deleted_at IS '删除时间（软删除）';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dm_aat_dataset_id ON t_dm_auto_annotation_tasks(dataset_id);
CREATE INDEX IF NOT EXISTS idx_dm_aat_status ON t_dm_auto_annotation_tasks(status);
CREATE INDEX IF NOT EXISTS idx_dm_aat_created_at ON t_dm_auto_annotation_tasks(created_at);
CREATE INDEX IF NOT EXISTS idx_dm_aat_deleted_at ON t_dm_auto_annotation_tasks(deleted_at);
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- 为各个表创建触发器
DROP TRIGGER IF EXISTS update_dm_annotation_templates_updated_at ON t_dm_annotation_templates;
CREATE TRIGGER update_dm_annotation_templates_updated_at
BEFORE UPDATE ON t_dm_annotation_templates
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_dm_labeling_projects_updated_at ON t_dm_labeling_projects;
CREATE TRIGGER update_dm_labeling_projects_updated_at
BEFORE UPDATE ON t_dm_labeling_projects
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_dm_auto_annotation_tasks_updated_at ON t_dm_auto_annotation_tasks;
CREATE TRIGGER update_dm_auto_annotation_tasks_updated_at
BEFORE UPDATE ON t_dm_auto_annotation_tasks
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 内置标注模板初始化数据
-- 1. 图像分类模板
INSERT INTO t_dm_annotation_templates (id, name, description, data_type, labeling_type, configuration, style, category, built_in, version)
VALUES ('tpl-image-classification-001', '图像分类', '简单的多标签图像分类模板', '图像', '分类', '{
"labels": [
{
"fromName": "choice",
"toName": "image",
"type": "Choices",
"options": ["Cat", "Dog", "Bird", "Other"],
"required": true,
"description": "选择最符合图像内容的标签"
}
],
"objects": [
{
"name": "image",
"type": "Image",
"value": "$image"
}
]
}',
'horizontal',
'计算机视觉',
TRUE,
'1.0.0'
),
(
'tpl-object-detection-001',
'目标检测（边界框）',
'使用矩形边界框进行目标检测',
'图像',
'目标检测',
'{
"labels": [
{
"fromName": "label",
"toName": "image",
"type": "RectangleLabels",
"labels": ["Person", "Vehicle", "Animal", "Object"],
"required": false,
"description": "在图像中框出目标并标注类别"
}
],
"objects": [
{
"name": "image",
"type": "Image",
"value": "$image"
}
]
}',
'horizontal',
'计算机视觉',
TRUE,
'1.0.0'
),
(
'tpl-image-segmentation-001',
'图像分割（多边形）',
'使用多边形标注进行语义分割',
'图像',
'分割',
'{
"labels": [
{
"fromName": "label",
"toName": "image",
"type": "PolygonLabels",
"labels": ["Background", "Foreground", "Person", "Car"],
"required": false,
"description": "使用多边形框选需要分割的区域"
}
],
"objects": [
{
"name": "image",
"type": "Image",
"value": "$image"
}
]
}',
'horizontal',
'计算机视觉',
TRUE,
'1.0.0'
),
(
'tpl-text-classification-001',
'文本情感分类',
'将文本中表达的情感划分到预定义的类别',
'文本',
'分类',
'{
"labels": [
{
"fromName": "choice",
"toName": "text",
"type": "Choices",
"options": ["Positive", "Negative", "Neutral"],
"required": true,
"description": "对文本的情感或类别进行选择"
}
],
"objects": [
{
"name": "text",
"type": "Text",
"value": "$text"
}
]
}',
'vertical',
'自然语言处理',
TRUE,
'1.0.0'
),
(
'tpl-ner-001',
'命名实体识别',
'从文本中抽取并标注命名实体',
'文本',
'实体识别',
'{
"labels": [
{
"fromName": "label",
"toName": "text",
"type": "Labels",
"labels": ["PERSON", "ORG", "LOC", "DATE", "MISC"],
"required": false,
"description": "在文本中标注人物、地点等实体"
}
],
"objects": [
{
"name": "text",
"type": "Text",
"value": "$text"
}
]
}',
'vertical',
'自然语言处理',
TRUE,
'1.0.0'
),
(
'tpl-audio-classification-001',
'音频分类',
'将音频片段划分到不同类别',
'音频',
'分类',
'{
"labels": [
{
"fromName": "choice",
"toName": "audio",
"type": "Choices",
"options": ["Speech", "Music", "Noise", "Silence"],
"required": true,
"description": "选择音频片段对应的类别"
}
],
"objects": [
{
"name": "audio",
"type": "Audio",
"value": "$audio"
}
]
}',
'horizontal',
'音频',
TRUE,
'1.0.0'
),
(
'tpl-text-multilabel-001',
'文本多标签分类',
'可为文本选择多个标签，适用于主题、内容类别等多标签任务',
'文本',
'分类',
'{
"labels": [
{
"fromName": "labels",
"toName": "text",
"type": "Choices",
"options": ["Sports", "Politics", "Tech", "Entertainment"],
"required": true,
"choice": "multiple",
"description": "可选择多个标签"
}
],
"objects": [
{
"name": "text",
"type": "Text",
"value": "$text"
}
]
}',
'vertical',
'自然语言处理',
TRUE,
'1.0.0'
),
(
'tpl-text-summarization-001',
'文本摘要',
'根据原文撰写简要摘要',
'文本',
'摘要',
'{
"labels": [
{
"fromName": "summary",
"toName": "text",
"type": "TextArea",
"required": true,
"description": "在此填写摘要内容"
}
],
"objects": [
{
"name": "text",
"type": "Text",
"value": "$text"
}
]
}',
'vertical',
'自然语言处理',
TRUE,
'1.0.0'
),
(
'tpl-keyword-extract-001',
'关键词抽取',
'从文本中选出关键词或关键短语',
'文本',
'实体识别',
'{
"labels": [
{
"fromName": "kw",
"toName": "text",
"type": "Labels",
"labels": ["Keyword"],
"required": false,
"description": "高亮文本并标注关键词"
}
],
"objects": [
{
"name": "text",
"type": "Text",
"value": "$text"
}
]
}',
'vertical',
'自然语言处理',
TRUE,
'1.0.0'
)
ON CONFLICT (id) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
data_type = EXCLUDED.data_type,
labeling_type = EXCLUDED.labeling_type,
configuration = EXCLUDED.configuration,
style = EXCLUDED.style,
category = EXCLUDED.category,
built_in = EXCLUDED.built_in,
version = EXCLUDED.version,
created_by = COALESCE(EXCLUDED.created_by, 'system'),
updated_by = COALESCE(EXCLUDED.updated_by, 'system'),
updated_at = CURRENT_TIMESTAMP;
-- Ensure all annotation templates have created_by/updated_by set for data scope filtering
UPDATE t_dm_annotation_templates SET created_by = 'system', updated_by = 'system' WHERE created_by IS NULL OR created_by = '';
scripts/db/data-cleaning-init.sql
+197 -197
View File
@@ -1,198 +1,198 @@
-- 使用现有的deepdata数据库
\c deepdata;
-- 清洗模板表
CREATE TABLE IF NOT EXISTS t_clean_template
(
id VARCHAR(64) PRIMARY KEY,
name VARCHAR(64) UNIQUE,
description VARCHAR(256),
created_by VARCHAR(256),
updated_by VARCHAR(256),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
COMMENT ON TABLE t_clean_template IS '清洗模板表';
COMMENT ON COLUMN t_clean_template.id IS '主键ID';
COMMENT ON COLUMN t_clean_template.name IS '模板名称';
COMMENT ON COLUMN t_clean_template.description IS '模板描述';
COMMENT ON COLUMN t_clean_template.created_at IS '创建时间';
COMMENT ON COLUMN t_clean_template.updated_at IS '更新时间';
COMMENT ON COLUMN t_clean_template.created_by IS '创建者';
COMMENT ON COLUMN t_clean_template.updated_by IS '更新者';
-- 清洗任务表
CREATE TABLE IF NOT EXISTS t_clean_task
(
id VARCHAR(64) PRIMARY KEY,
name VARCHAR(64) UNIQUE,
description VARCHAR(256),
status VARCHAR(256),
src_dataset_id VARCHAR(64),
src_dataset_name VARCHAR(64),
dest_dataset_id VARCHAR(64),
dest_dataset_name VARCHAR(64),
before_size BIGINT,
after_size BIGINT,
file_count INTEGER,
retry_count INTEGER,
started_at TIMESTAMP,
finished_at TIMESTAMP,
created_by VARCHAR(256),
updated_by VARCHAR(256),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
COMMENT ON TABLE t_clean_task IS '清洗任务表';
COMMENT ON COLUMN t_clean_task.id IS '主键ID';
COMMENT ON COLUMN t_clean_task.name IS '任务名称';
COMMENT ON COLUMN t_clean_task.description IS '任务描述';
COMMENT ON COLUMN t_clean_task.status IS '任务状态';
COMMENT ON COLUMN t_clean_task.src_dataset_id IS '源数据集ID';
COMMENT ON COLUMN t_clean_task.src_dataset_name IS '源数据集名称';
COMMENT ON COLUMN t_clean_task.dest_dataset_id IS '目标数据集ID';
COMMENT ON COLUMN t_clean_task.dest_dataset_name IS '目标数据集名称';
COMMENT ON COLUMN t_clean_task.before_size IS '清洗前大小';
COMMENT ON COLUMN t_clean_task.after_size IS '清洗后大小';
COMMENT ON COLUMN t_clean_task.file_count IS '文件数量';
COMMENT ON COLUMN t_clean_task.retry_count IS '重试次数';
COMMENT ON COLUMN t_clean_task.started_at IS '开始时间';
COMMENT ON COLUMN t_clean_task.finished_at IS '完成时间';
COMMENT ON COLUMN t_clean_task.created_at IS '创建时间';
COMMENT ON COLUMN t_clean_task.updated_at IS '更新时间';
COMMENT ON COLUMN t_clean_task.created_by IS '创建者';
COMMENT ON COLUMN t_clean_task.updated_by IS '更新者';
-- 操作员实例表
CREATE TABLE IF NOT EXISTS t_operator_instance
(
instance_id VARCHAR(256),
operator_id VARCHAR(256),
op_index INTEGER,
settings_override TEXT,
PRIMARY KEY (instance_id, operator_id, op_index)
);
COMMENT ON TABLE t_operator_instance IS '操作员实例表';
COMMENT ON COLUMN t_operator_instance.instance_id IS '实例ID';
COMMENT ON COLUMN t_operator_instance.operator_id IS '操作员ID';
COMMENT ON COLUMN t_operator_instance.op_index IS '操作序号';
COMMENT ON COLUMN t_operator_instance.settings_override IS '设置覆盖';
-- 清洗结果表
CREATE TABLE IF NOT EXISTS t_clean_result
(
instance_id VARCHAR(64),
src_file_id VARCHAR(64),
dest_file_id VARCHAR(64),
src_name VARCHAR(256),
dest_name VARCHAR(256),
src_type VARCHAR(256),
dest_type VARCHAR(256),
src_size BIGINT,
dest_size BIGINT,
status VARCHAR(256),
result TEXT,
PRIMARY KEY (instance_id, dest_file_id)
);
COMMENT ON TABLE t_clean_result IS '清洗结果表';
COMMENT ON COLUMN t_clean_result.instance_id IS '实例ID';
COMMENT ON COLUMN t_clean_result.src_file_id IS '源文件ID';
COMMENT ON COLUMN t_clean_result.dest_file_id IS '目标文件ID';
COMMENT ON COLUMN t_clean_result.src_name IS '源文件名';
COMMENT ON COLUMN t_clean_result.dest_name IS '目标文件名';
COMMENT ON COLUMN t_clean_result.src_type IS '源文件类型';
COMMENT ON COLUMN t_clean_result.dest_type IS '目标文件类型';
COMMENT ON COLUMN t_clean_result.src_size IS '源文件大小';
COMMENT ON COLUMN t_clean_result.dest_size IS '目标文件大小';
COMMENT ON COLUMN t_clean_result.status IS '状态';
COMMENT ON COLUMN t_clean_result.result IS '结果';
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- 为 t_clean_template 表创建触发器
DROP TRIGGER IF EXISTS update_clean_template_updated_at ON t_clean_template;
CREATE TRIGGER update_clean_template_updated_at
BEFORE UPDATE ON t_clean_template
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 插入初始数据 - 清洗模板
INSERT INTO t_clean_template (id, name, description, created_by, updated_by)
VALUES
('550e8400-e29b-41d4-a716-446655440001', '安全与隐私合规处理模板', '针对敏感数据进行严格清洗，移除PII（个人身份信息）、政治敏感、暴力色情内容，适用于模型对外发布前的安全合规检查。', 'system', 'system'),
('661f9500-f3ac-52e5-b827-557766550002', 'LLM SFT高质量文本清洗模板', '旨在生成高质量、低噪声的训练数据。包含去除乱码、重复内容、繁简转换、全角转半角以及格式标准化处理。', 'system', 'system'),
('772a0611-a4bd-63f6-c938-668877660003', 'RAG知识库构建预处理模板', '专为RAG场景设计。重点去除目录、图注、XML/HTML标签等对向量检索无意义的噪声，并进行段落级去重以优化切片质量。', 'system', 'system'),
('883b1722-b5ce-7407-d049-779988770004', '原始Web爬虫数据清洗模板', '针对互联网爬取的脏数据进行清洗。重点去除Emoji表情、URL链接、HTML标签以及不可见字符。', 'system', 'system'),
('994c2833-c6df-8518-e150-880099880005', '多模态/CV模型训练预处理模板', '针对图像数据集处理。包含去除模糊/重复/相似图片，图片方向校正，目标检测预标注，以及尺寸和格式的统一化。', 'system', 'system')
ON CONFLICT (id) DO NOTHING;
-- Also update existing templates that may have empty created_by
UPDATE t_clean_template SET created_by = 'system', updated_by = 'system' WHERE created_by IS NULL OR created_by = '';
INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override)
VALUES
('550e8400-e29b-41d4-a716-446655440001', 'PoliticalWordCleaner', 1, NULL),
('550e8400-e29b-41d4-a716-446655440001', 'SexualAndViolentWordCleaner', 2, NULL),
('550e8400-e29b-41d4-a716-446655440001', 'PiiDetector', 3, NULL),
('550e8400-e29b-41d4-a716-446655440001', 'AnonymizedIdNumber', 4, NULL),
('550e8400-e29b-41d4-a716-446655440001', 'AnonymizedCreditCardNumber', 5, NULL),
('550e8400-e29b-41d4-a716-446655440001', 'AnonymizedPhoneNumber', 6, NULL),
('550e8400-e29b-41d4-a716-446655440001', 'EmailNumberCleaner', 7, NULL),
('550e8400-e29b-41d4-a716-446655440001', 'AnonymizedIpAddress', 8, NULL)
ON CONFLICT (instance_id, operator_id, op_index) DO NOTHING;
INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override)
VALUES
('661f9500-f3ac-52e5-b827-557766550002', 'GrableCharactersCleaner', 1, NULL),
('661f9500-f3ac-52e5-b827-557766550002', 'InvisibleCharactersCleaner', 2, NULL),
('661f9500-f3ac-52e5-b827-557766550002', 'FullWidthCharacterCleaner', 3, NULL),
('661f9500-f3ac-52e5-b827-557766550002', 'TraditionalChineseCleaner', 4, NULL),
('661f9500-f3ac-52e5-b827-557766550002', 'FileWithShortOrLongLengthFilter', 5, '{"fileLength": [50, 8192]}'),
('661f9500-f3ac-52e5-b827-557766550002', 'FileWithHighRepeatPhraseRateFilter', 6, NULL),
('661f9500-f3ac-52e5-b827-557766550002', 'FileWithHighSpecialCharRateFilter', 7, NULL),
('661f9500-f3ac-52e5-b827-557766550002', 'DuplicateFilesFilter', 8, NULL)
ON CONFLICT (instance_id, operator_id, op_index) DO NOTHING;
INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override)
VALUES
('772a0611-a4bd-63f6-c938-668877660003', 'HtmlTagCleaner', 1, '{"removeTableTags": "false"}'), -- 表格对RAG可能有价值，暂不去除表格
('772a0611-a4bd-63f6-c938-668877660003', 'ContentCleaner', 2, NULL),
('772a0611-a4bd-63f6-c938-668877660003', 'LegendCleaner', 3, NULL),
('772a0611-a4bd-63f6-c938-668877660003', 'XMLTagCleaner', 4, NULL),
('772a0611-a4bd-63f6-c938-668877660003', 'UnicodeSpaceCleaner', 5, NULL),
('772a0611-a4bd-63f6-c938-668877660003', 'ExtraSpaceCleaner', 6, NULL),
('772a0611-a4bd-63f6-c938-668877660003', 'DuplicateSentencesFilter', 7, NULL),
('772a0611-a4bd-63f6-c938-668877660003', 'FileWithShortOrLongLengthFilter', 8, '{"fileLength": [20, 100000]}')
ON CONFLICT (instance_id, operator_id, op_index) DO NOTHING;
INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override)
VALUES
('883b1722-b5ce-7407-d049-779988770004', 'HtmlTagCleaner', 1, '{"removeTableTags": "true"}'),
('883b1722-b5ce-7407-d049-779988770004', 'AnonymizedUrlCleaner', 2, NULL),
('883b1722-b5ce-7407-d049-779988770004', 'EmojiCleaner', 3, NULL),
('883b1722-b5ce-7407-d049-779988770004', 'InvisibleCharactersCleaner', 4, NULL),
('883b1722-b5ce-7407-d049-779988770004', 'ExtraSpaceCleaner', 5, NULL),
('883b1722-b5ce-7407-d049-779988770004', 'DuplicateFilesFilter', 6, '{"fileDuplicateThreshold": 0.6}')
ON CONFLICT (instance_id, operator_id, op_index) DO NOTHING;
INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override)
VALUES
('994c2833-c6df-8518-e150-880099880005', 'ImgBlurredImagesCleaner', 1, NULL),
('994c2833-c6df-8518-e150-880099880005', 'ImgDuplicatedImagesCleaner', 2, NULL),
('994c2833-c6df-8518-e150-880099880005', 'ImgSimilarImagesCleaner', 3, NULL),
('994c2833-c6df-8518-e150-880099880005', 'ImgDirectionCorrect', 4, NULL),
('994c2833-c6df-8518-e150-880099880005', 'ImgResize', 5, '{"widthSize": 512, "heightSize": 512}'),
('994c2833-c6df-8518-e150-880099880005', 'ImgTypeUnify', 6, '{"imgType": "jpg"}')
-- 使用现有的DeepData数据库
\c deepdata;
-- 清洗模板表
CREATE TABLE IF NOT EXISTS t_clean_template
(
id VARCHAR(64) PRIMARY KEY,
name VARCHAR(64) UNIQUE,
description VARCHAR(256),
created_by VARCHAR(256),
updated_by VARCHAR(256),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
COMMENT ON TABLE t_clean_template IS '清洗模板表';
COMMENT ON COLUMN t_clean_template.id IS '主键ID';
COMMENT ON COLUMN t_clean_template.name IS '模板名称';
COMMENT ON COLUMN t_clean_template.description IS '模板描述';
COMMENT ON COLUMN t_clean_template.created_at IS '创建时间';
COMMENT ON COLUMN t_clean_template.updated_at IS '更新时间';
COMMENT ON COLUMN t_clean_template.created_by IS '创建者';
COMMENT ON COLUMN t_clean_template.updated_by IS '更新者';
-- 清洗任务表
CREATE TABLE IF NOT EXISTS t_clean_task
(
id VARCHAR(64) PRIMARY KEY,
name VARCHAR(64) UNIQUE,
description VARCHAR(256),
status VARCHAR(256),
src_dataset_id VARCHAR(64),
src_dataset_name VARCHAR(64),
dest_dataset_id VARCHAR(64),
dest_dataset_name VARCHAR(64),
before_size BIGINT,
after_size BIGINT,
file_count INTEGER,
retry_count INTEGER,
started_at TIMESTAMP,
finished_at TIMESTAMP,
created_by VARCHAR(256),
updated_by VARCHAR(256),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
COMMENT ON TABLE t_clean_task IS '清洗任务表';
COMMENT ON COLUMN t_clean_task.id IS '主键ID';
COMMENT ON COLUMN t_clean_task.name IS '任务名称';
COMMENT ON COLUMN t_clean_task.description IS '任务描述';
COMMENT ON COLUMN t_clean_task.status IS '任务状态';
COMMENT ON COLUMN t_clean_task.src_dataset_id IS '源数据集ID';
COMMENT ON COLUMN t_clean_task.src_dataset_name IS '源数据集名称';
COMMENT ON COLUMN t_clean_task.dest_dataset_id IS '目标数据集ID';
COMMENT ON COLUMN t_clean_task.dest_dataset_name IS '目标数据集名称';
COMMENT ON COLUMN t_clean_task.before_size IS '清洗前大小';
COMMENT ON COLUMN t_clean_task.after_size IS '清洗后大小';
COMMENT ON COLUMN t_clean_task.file_count IS '文件数量';
COMMENT ON COLUMN t_clean_task.retry_count IS '重试次数';
COMMENT ON COLUMN t_clean_task.started_at IS '开始时间';
COMMENT ON COLUMN t_clean_task.finished_at IS '完成时间';
COMMENT ON COLUMN t_clean_task.created_at IS '创建时间';
COMMENT ON COLUMN t_clean_task.updated_at IS '更新时间';
COMMENT ON COLUMN t_clean_task.created_by IS '创建者';
COMMENT ON COLUMN t_clean_task.updated_by IS '更新者';
-- 操作员实例表
CREATE TABLE IF NOT EXISTS t_operator_instance
(
instance_id VARCHAR(256),
operator_id VARCHAR(256),
op_index INTEGER,
settings_override TEXT,
PRIMARY KEY (instance_id, operator_id, op_index)
);
COMMENT ON TABLE t_operator_instance IS '操作员实例表';
COMMENT ON COLUMN t_operator_instance.instance_id IS '实例ID';
COMMENT ON COLUMN t_operator_instance.operator_id IS '操作员ID';
COMMENT ON COLUMN t_operator_instance.op_index IS '操作序号';
COMMENT ON COLUMN t_operator_instance.settings_override IS '设置覆盖';
-- 清洗结果表
CREATE TABLE IF NOT EXISTS t_clean_result
(
instance_id VARCHAR(64),
src_file_id VARCHAR(64),
dest_file_id VARCHAR(64),
src_name VARCHAR(256),
dest_name VARCHAR(256),
src_type VARCHAR(256),
dest_type VARCHAR(256),
src_size BIGINT,
dest_size BIGINT,
status VARCHAR(256),
result TEXT,
PRIMARY KEY (instance_id, dest_file_id)
);
COMMENT ON TABLE t_clean_result IS '清洗结果表';
COMMENT ON COLUMN t_clean_result.instance_id IS '实例ID';
COMMENT ON COLUMN t_clean_result.src_file_id IS '源文件ID';
COMMENT ON COLUMN t_clean_result.dest_file_id IS '目标文件ID';
COMMENT ON COLUMN t_clean_result.src_name IS '源文件名';
COMMENT ON COLUMN t_clean_result.dest_name IS '目标文件名';
COMMENT ON COLUMN t_clean_result.src_type IS '源文件类型';
COMMENT ON COLUMN t_clean_result.dest_type IS '目标文件类型';
COMMENT ON COLUMN t_clean_result.src_size IS '源文件大小';
COMMENT ON COLUMN t_clean_result.dest_size IS '目标文件大小';
COMMENT ON COLUMN t_clean_result.status IS '状态';
COMMENT ON COLUMN t_clean_result.result IS '结果';
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- 为 t_clean_template 表创建触发器
DROP TRIGGER IF EXISTS update_clean_template_updated_at ON t_clean_template;
CREATE TRIGGER update_clean_template_updated_at
BEFORE UPDATE ON t_clean_template
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 插入初始数据 - 清洗模板
INSERT INTO t_clean_template (id, name, description, created_by, updated_by)
VALUES
('550e8400-e29b-41d4-a716-446655440001', '安全与隐私合规处理模板', '针对敏感数据进行严格清洗，移除PII（个人身份信息）、政治敏感、暴力色情内容，适用于模型对外发布前的安全合规检查。', 'system', 'system'),
('661f9500-f3ac-52e5-b827-557766550002', 'LLM SFT高质量文本清洗模板', '旨在生成高质量、低噪声的训练数据。包含去除乱码、重复内容、繁简转换、全角转半角以及格式标准化处理。', 'system', 'system'),
('772a0611-a4bd-63f6-c938-668877660003', 'RAG知识库构建预处理模板', '专为RAG场景设计。重点去除目录、图注、XML/HTML标签等对向量检索无意义的噪声，并进行段落级去重以优化切片质量。', 'system', 'system'),
('883b1722-b5ce-7407-d049-779988770004', '原始Web爬虫数据清洗模板', '针对互联网爬取的脏数据进行清洗。重点去除Emoji表情、URL链接、HTML标签以及不可见字符。', 'system', 'system'),
('994c2833-c6df-8518-e150-880099880005', '多模态/CV模型训练预处理模板', '针对图像数据集处理。包含去除模糊/重复/相似图片，图片方向校正，目标检测预标注，以及尺寸和格式的统一化。', 'system', 'system')
ON CONFLICT (id) DO NOTHING;
-- Also update existing templates that may have empty created_by
UPDATE t_clean_template SET created_by = 'system', updated_by = 'system' WHERE created_by IS NULL OR created_by = '';
INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override)
VALUES
('550e8400-e29b-41d4-a716-446655440001', 'PoliticalWordCleaner', 1, NULL),
('550e8400-e29b-41d4-a716-446655440001', 'SexualAndViolentWordCleaner', 2, NULL),
('550e8400-e29b-41d4-a716-446655440001', 'PiiDetector', 3, NULL),
('550e8400-e29b-41d4-a716-446655440001', 'AnonymizedIdNumber', 4, NULL),
('550e8400-e29b-41d4-a716-446655440001', 'AnonymizedCreditCardNumber', 5, NULL),
('550e8400-e29b-41d4-a716-446655440001', 'AnonymizedPhoneNumber', 6, NULL),
('550e8400-e29b-41d4-a716-446655440001', 'EmailNumberCleaner', 7, NULL),
('550e8400-e29b-41d4-a716-446655440001', 'AnonymizedIpAddress', 8, NULL)
ON CONFLICT (instance_id, operator_id, op_index) DO NOTHING;
INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override)
VALUES
('661f9500-f3ac-52e5-b827-557766550002', 'GrableCharactersCleaner', 1, NULL),
('661f9500-f3ac-52e5-b827-557766550002', 'InvisibleCharactersCleaner', 2, NULL),
('661f9500-f3ac-52e5-b827-557766550002', 'FullWidthCharacterCleaner', 3, NULL),
('661f9500-f3ac-52e5-b827-557766550002', 'TraditionalChineseCleaner', 4, NULL),
('661f9500-f3ac-52e5-b827-557766550002', 'FileWithShortOrLongLengthFilter', 5, '{"fileLength": [50, 8192]}'),
('661f9500-f3ac-52e5-b827-557766550002', 'FileWithHighRepeatPhraseRateFilter', 6, NULL),
('661f9500-f3ac-52e5-b827-557766550002', 'FileWithHighSpecialCharRateFilter', 7, NULL),
('661f9500-f3ac-52e5-b827-557766550002', 'DuplicateFilesFilter', 8, NULL)
ON CONFLICT (instance_id, operator_id, op_index) DO NOTHING;
INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override)
VALUES
('772a0611-a4bd-63f6-c938-668877660003', 'HtmlTagCleaner', 1, '{"removeTableTags": "false"}'), -- 表格对RAG可能有价值，暂不去除表格
('772a0611-a4bd-63f6-c938-668877660003', 'ContentCleaner', 2, NULL),
('772a0611-a4bd-63f6-c938-668877660003', 'LegendCleaner', 3, NULL),
('772a0611-a4bd-63f6-c938-668877660003', 'XMLTagCleaner', 4, NULL),
('772a0611-a4bd-63f6-c938-668877660003', 'UnicodeSpaceCleaner', 5, NULL),
('772a0611-a4bd-63f6-c938-668877660003', 'ExtraSpaceCleaner', 6, NULL),
('772a0611-a4bd-63f6-c938-668877660003', 'DuplicateSentencesFilter', 7, NULL),
('772a0611-a4bd-63f6-c938-668877660003', 'FileWithShortOrLongLengthFilter', 8, '{"fileLength": [20, 100000]}')
ON CONFLICT (instance_id, operator_id, op_index) DO NOTHING;
INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override)
VALUES
('883b1722-b5ce-7407-d049-779988770004', 'HtmlTagCleaner', 1, '{"removeTableTags": "true"}'),
('883b1722-b5ce-7407-d049-779988770004', 'AnonymizedUrlCleaner', 2, NULL),
('883b1722-b5ce-7407-d049-779988770004', 'EmojiCleaner', 3, NULL),
('883b1722-b5ce-7407-d049-779988770004', 'InvisibleCharactersCleaner', 4, NULL),
('883b1722-b5ce-7407-d049-779988770004', 'ExtraSpaceCleaner', 5, NULL),
('883b1722-b5ce-7407-d049-779988770004', 'DuplicateFilesFilter', 6, '{"fileDuplicateThreshold": 0.6}')
ON CONFLICT (instance_id, operator_id, op_index) DO NOTHING;
INSERT INTO t_operator_instance (instance_id, operator_id, op_index, settings_override)
VALUES
('994c2833-c6df-8518-e150-880099880005', 'ImgBlurredImagesCleaner', 1, NULL),
('994c2833-c6df-8518-e150-880099880005', 'ImgDuplicatedImagesCleaner', 2, NULL),
('994c2833-c6df-8518-e150-880099880005', 'ImgSimilarImagesCleaner', 3, NULL),
('994c2833-c6df-8518-e150-880099880005', 'ImgDirectionCorrect', 4, NULL),
('994c2833-c6df-8518-e150-880099880005', 'ImgResize', 5, '{"widthSize": 512, "heightSize": 512}'),
('994c2833-c6df-8518-e150-880099880005', 'ImgTypeUnify', 6, '{"imgType": "jpg"}')
ON CONFLICT (instance_id, operator_id, op_index) DO NOTHING;
scripts/db/data-collection-init.sql
+248 -248
View File
@@ -1,248 +1,248 @@
-- 使用现有的deepdata数据库
\c deepdata;
-- 数据归集任务表
CREATE TABLE IF NOT EXISTS t_dc_collection_tasks (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
description TEXT,
sync_mode VARCHAR(20) DEFAULT 'ONCE',
template_id VARCHAR(36) NOT NULL,
template_name VARCHAR(255) NOT NULL,
target_path VARCHAR(1000) DEFAULT '',
config JSONB NOT NULL,
schedule_expression VARCHAR(255),
status VARCHAR(20) DEFAULT 'DRAFT',
retry_count INTEGER DEFAULT 3,
timeout_seconds INTEGER DEFAULT 3600,
last_execution_id VARCHAR(36),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_dc_collection_tasks IS '数据归集任务表';
COMMENT ON COLUMN t_dc_collection_tasks.id IS '任务ID（UUID）';
COMMENT ON COLUMN t_dc_collection_tasks.name IS '任务名称';
COMMENT ON COLUMN t_dc_collection_tasks.description IS '任务描述';
COMMENT ON COLUMN t_dc_collection_tasks.sync_mode IS '同步模式：ONCE/SCHEDULED';
COMMENT ON COLUMN t_dc_collection_tasks.template_id IS '归集模板ID';
COMMENT ON COLUMN t_dc_collection_tasks.template_name IS '归集模板名称';
COMMENT ON COLUMN t_dc_collection_tasks.target_path IS '目标存储路径';
COMMENT ON COLUMN t_dc_collection_tasks.config IS '归集配置（DataX配置），包含源端和目标端配置信息';
COMMENT ON COLUMN t_dc_collection_tasks.schedule_expression IS 'Cron调度表达式';
COMMENT ON COLUMN t_dc_collection_tasks.status IS '任务状态：DRAFT/READY/RUNNING/SUCCESS/FAILED/STOPPED';
COMMENT ON COLUMN t_dc_collection_tasks.retry_count IS '重试次数';
COMMENT ON COLUMN t_dc_collection_tasks.timeout_seconds IS '超时时间（秒）';
COMMENT ON COLUMN t_dc_collection_tasks.last_execution_id IS '最后执行ID（UUID）';
COMMENT ON COLUMN t_dc_collection_tasks.created_at IS '创建时间';
COMMENT ON COLUMN t_dc_collection_tasks.updated_at IS '更新时间';
COMMENT ON COLUMN t_dc_collection_tasks.created_by IS '创建者';
COMMENT ON COLUMN t_dc_collection_tasks.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dc_tasks_status ON t_dc_collection_tasks(status);
CREATE INDEX IF NOT EXISTS idx_dc_tasks_created_at ON t_dc_collection_tasks(created_at);
CREATE INDEX IF NOT EXISTS idx_dc_tasks_template_id ON t_dc_collection_tasks(template_id);
-- 任务执行明细表
CREATE TABLE IF NOT EXISTS t_dc_task_executions (
id VARCHAR(36) PRIMARY KEY,
task_id VARCHAR(36) NOT NULL,
task_name VARCHAR(255) NOT NULL,
status VARCHAR(20) DEFAULT 'RUNNING',
log_path VARCHAR(1000) NOT NULL,
started_at TIMESTAMP,
completed_at TIMESTAMP,
duration_seconds INTEGER DEFAULT 0,
error_message TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_dc_task_executions IS '任务执行明细表';
COMMENT ON COLUMN t_dc_task_executions.id IS '执行记录ID（UUID）';
COMMENT ON COLUMN t_dc_task_executions.task_id IS '任务ID';
COMMENT ON COLUMN t_dc_task_executions.task_name IS '任务名称';
COMMENT ON COLUMN t_dc_task_executions.status IS '执行状态：RUNNING/SUCCESS/FAILED/STOPPED';
COMMENT ON COLUMN t_dc_task_executions.log_path IS '日志文件路径';
COMMENT ON COLUMN t_dc_task_executions.started_at IS '开始时间';
COMMENT ON COLUMN t_dc_task_executions.completed_at IS '完成时间';
COMMENT ON COLUMN t_dc_task_executions.duration_seconds IS '执行时长（秒）';
COMMENT ON COLUMN t_dc_task_executions.error_message IS '错误信息';
COMMENT ON COLUMN t_dc_task_executions.created_at IS '创建时间';
COMMENT ON COLUMN t_dc_task_executions.updated_at IS '更新时间';
COMMENT ON COLUMN t_dc_task_executions.created_by IS '创建者';
COMMENT ON COLUMN t_dc_task_executions.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dc_executions_task_id ON t_dc_task_executions(task_id);
CREATE INDEX IF NOT EXISTS idx_dc_executions_status ON t_dc_task_executions(status);
CREATE INDEX IF NOT EXISTS idx_dc_executions_started_at ON t_dc_task_executions(started_at);
-- 外键约束
ALTER TABLE t_dc_task_executions ADD CONSTRAINT fk_dc_executions_task FOREIGN KEY (task_id)
REFERENCES t_dc_collection_tasks(id) ON DELETE CASCADE;
-- 数据归集模板配置表
CREATE TABLE IF NOT EXISTS t_dc_collection_templates (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL UNIQUE,
description TEXT,
source_type VARCHAR(64) NOT NULL,
source_name VARCHAR(64) NOT NULL,
target_type VARCHAR(64) NOT NULL,
target_name VARCHAR(64) NOT NULL,
template_content JSONB NOT NULL,
built_in BOOLEAN DEFAULT FALSE,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_dc_collection_templates IS '数据归集模板配置表';
COMMENT ON COLUMN t_dc_collection_templates.id IS '模板ID（UUID）';
COMMENT ON COLUMN t_dc_collection_templates.name IS '模板名称';
COMMENT ON COLUMN t_dc_collection_templates.description IS '模板描述';
COMMENT ON COLUMN t_dc_collection_templates.source_type IS '源数据源类型';
COMMENT ON COLUMN t_dc_collection_templates.source_name IS '源数据源名称';
COMMENT ON COLUMN t_dc_collection_templates.target_type IS '目标数据源类型';
COMMENT ON COLUMN t_dc_collection_templates.target_name IS '目标数据源名称';
COMMENT ON COLUMN t_dc_collection_templates.template_content IS '模板内容';
COMMENT ON COLUMN t_dc_collection_templates.built_in IS '是否系统内置';
COMMENT ON COLUMN t_dc_collection_templates.created_at IS '创建时间';
COMMENT ON COLUMN t_dc_collection_templates.updated_at IS '更新时间';
COMMENT ON COLUMN t_dc_collection_templates.created_by IS '创建者';
COMMENT ON COLUMN t_dc_collection_templates.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dc_templates_source_target ON t_dc_collection_templates(source_type, target_type);
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- 为各个表创建触发器
DROP TRIGGER IF EXISTS update_dc_collection_tasks_updated_at ON t_dc_collection_tasks;
CREATE TRIGGER update_dc_collection_tasks_updated_at
BEFORE UPDATE ON t_dc_collection_tasks
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_dc_task_executions_updated_at ON t_dc_task_executions;
CREATE TRIGGER update_dc_task_executions_updated_at
BEFORE UPDATE ON t_dc_task_executions
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_dc_collection_templates_updated_at ON t_dc_collection_templates;
CREATE TRIGGER update_dc_collection_templates_updated_at
BEFORE UPDATE ON t_dc_collection_templates
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 插入初始数据 - 数据归集模板
INSERT INTO t_dc_collection_templates (
id, name, description, source_type, source_name, target_type, target_name,
template_content, built_in, created_by, updated_by
) VALUES
(
'1',
'NAS归集模板',
'将NAS存储上的文件归集到平台上。',
'nfsreader',
'nfsreader',
'nfswriter',
'nfswriter',
'{"parameter": {"ip": {"name": "NAS地址","description": "NAS服务的地址，可以为IP或者域名。","type": "input", "required": true, "index": 1}, "path": {"name": "共享路径","description": "NAS服务的共享路径。","type": "input", "required": true, "index": 2}, "files": {"name": "文件列表","description": "指定文件列表进行归集。","type": "selectTag", "required": false, "index": 3}}, "reader": {}, "writer": {}}',
TRUE,
'system',
'system'
),
(
'2',
'S3存储归集模板',
'将S3对象存储（如OBS、MinIO、Ceph等）上的文件归集到平台上。',
's3reader',
's3reader',
's3writer',
's3writer',
'{"parameter": {"endpoint": {"name": "服务地址","description": "S3兼容存储的服务地址（如http://minio.example.com:9000）。","type": "input", "required": true, "index": 1}, "bucket": {"name": "存储桶名称","description": "S3存储桶名称。","type": "input", "required": true, "index": 2}, "accessKey": {"name": "Access Key","description": "S3访问密钥（Access Key ID）。","type": "input", "required": true, "index": 3}, "secretKey": {"name": "Secret Key","description": "S3密钥（Secret Access Key）。","type": "password", "required": true, "index": 4}, "prefix": {"name": "匹配前缀","description": "按照匹配前缀选中S3中的文件进行归集。","type": "input", "required": false, "index": 5}, "region": {"name": "区域","description": "S3区域（默认us-east-1）。","type": "input", "required": false, "index": 6}}, "reader": {}, "writer": {}}',
TRUE,
'system',
'system'
),
(
'4',
'StarRocks归集模板',
'将StarRocks中的数据以csv文件的形式归集到DeepData平台上。',
'starrocksreader',
'starrocksreader',
'txtfilewriter',
'txtfilewriter',
'{"parameter": {}, "reader": {"username": {"name": "用户名","description": "数据库的用户名。","type": "input", "required": true, "index": 2}, "password": {"name": "密码","description": "数据库的密码。","type": "password", "required": true, "index": 3}, "connection": {"name": "数据库连接信息", "description": "数据库连接信息。", "type": "multipleList", "size": 1, "index": 1, "properties": {"jdbcUrl": {"type": "inputList", "name": "数据库连接", "description": "数据库连接url。", "required": true, "index": 1}, "querySql": {"type": "inputList", "name": "查询sql", "description": "输入符合语法的sql查询语句。", "required": true, "index": 2}}}}, "writer": {"header": {"name": "列名","description": "查询结果的列名，最终会体现为csv文件的表头。","type": "selectTag", "required": false}}}',
TRUE,
'system',
'system'
),
(
'5',
'GlusterFS归集模板',
'将GlusterFS分布式文件系统上的文件归集到平台上。',
'glusterfsreader',
'glusterfsreader',
'glusterfswriter',
'glusterfswriter',
'{"parameter": {"ip": {"name": "服务器地址","description": "GlusterFS服务器的IP地址或域名。","type": "input", "required": true, "index": 1}, "volume": {"name": "卷名称","description": "GlusterFS卷名称。","type": "input", "required": true, "index": 2}, "path": {"name": "子路径","description": "卷内的子目录路径（可选）。","type": "input", "required": false, "index": 3}, "files": {"name": "文件列表","description": "指定文件列表进行归集。","type": "selectTag", "required": false, "index": 4}}, "reader": {}, "writer": {}}',
TRUE,
'system',
'system'),
(
'6',
'API归集模板',
'将指定API返回的数据以csv文件的形式归集到平台上。',
'apireader',
'apireader',
'txtfilewriter',
'txtfilewriter',
'{"parameter": {}, "reader": {"api": {"name": "接口地址","description": "API的访问地址。","type": "input", "required": true, "index": 1}, "method": {"name": "请求方式","description": "API的请求方式（默认为GET）。","type": "option", "options": ["GET", "POST"], "required": false, "index": 2}, "body": {"name": "请求体参数","description": "请求体参数，主要针对POST请求。示例：\n {\"query\": \"value\"}","type": "jsonobject", "required": false, "index": 3}, "headers": {"name": "请求头参数","description": "需要设置的请求头参数。示例：\n {\"Authorization\": \"***\"}","type": "jsonobject", "required": false, "index": 4}, "schema": {"name": "数据解析schema","description": "会依据schema解析API返回结果。示例：\n {\"dataPath\": \"返回体内定位到具体数据的路径，形如$.data\", \"fields\": [{\"name\": \"属性名\", \"alias\": \"别名\", \"path\": \"属性的路径\"}]}","type": "jsonobject", "required": true, "index": 5}}, "writer": {}}',
TRUE,
'system',
'system'),
(
'7',
'通用关系型数据库归集模板',
'将关系型数据库中的数据以csv文件的形式归集到平台上。当前支持postgres、opengauss、sqlserver、mysql、达梦、db2格式、PPAS格式等关系型数据库。',
'rdbmsreader',
'rdbmsreader',
'txtfilewriter',
'txtfilewriter',
'{"parameter": {}, "reader": {"username": {"name": "用户名","description": "数据库的用户名。","type": "input", "required": true, "index": 2}, "password": {"name": "密码","description": "数据库的密码。","type": "password", "required": true, "index": 3}, "connection": {"name": "数据库连接信息", "description": "数据库连接信息。", "type": "multipleList", "size": 1, "index": 1, "properties": {"jdbcUrl": {"type": "inputList", "name": "数据库连接", "description": "数据库连接url。", "required": true, "index": 1}, "querySql": {"type": "inputList", "name": "查询sql", "description": "输入符合语法的sql查询语句。", "required": true, "index": 2}}}}, "writer": {"header": {"name": "列名","description": "查询结果的列名，最终会体现为csv文件的表头。","type": "selectTag", "required": false}}}',
TRUE,
'system',
'system'
)
ON CONFLICT (id) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
source_type = EXCLUDED.source_type,
source_name = EXCLUDED.source_name,
target_type = EXCLUDED.target_type,
target_name = EXCLUDED.target_name,
template_content = EXCLUDED.template_content,
built_in = EXCLUDED.built_in,
created_by = EXCLUDED.created_by,
updated_by = EXCLUDED.updated_by,
updated_at = CURRENT_TIMESTAMP;
-- 使用现有的DeepData数据库
\c deepdata;
-- 数据归集任务表
CREATE TABLE IF NOT EXISTS t_dc_collection_tasks (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
description TEXT,
sync_mode VARCHAR(20) DEFAULT 'ONCE',
template_id VARCHAR(36) NOT NULL,
template_name VARCHAR(255) NOT NULL,
target_path VARCHAR(1000) DEFAULT '',
config JSONB NOT NULL,
schedule_expression VARCHAR(255),
status VARCHAR(20) DEFAULT 'DRAFT',
retry_count INTEGER DEFAULT 3,
timeout_seconds INTEGER DEFAULT 3600,
last_execution_id VARCHAR(36),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_dc_collection_tasks IS '数据归集任务表';
COMMENT ON COLUMN t_dc_collection_tasks.id IS '任务ID（UUID）';
COMMENT ON COLUMN t_dc_collection_tasks.name IS '任务名称';
COMMENT ON COLUMN t_dc_collection_tasks.description IS '任务描述';
COMMENT ON COLUMN t_dc_collection_tasks.sync_mode IS '同步模式：ONCE/SCHEDULED';
COMMENT ON COLUMN t_dc_collection_tasks.template_id IS '归集模板ID';
COMMENT ON COLUMN t_dc_collection_tasks.template_name IS '归集模板名称';
COMMENT ON COLUMN t_dc_collection_tasks.target_path IS '目标存储路径';
COMMENT ON COLUMN t_dc_collection_tasks.config IS '归集配置（DataX配置），包含源端和目标端配置信息';
COMMENT ON COLUMN t_dc_collection_tasks.schedule_expression IS 'Cron调度表达式';
COMMENT ON COLUMN t_dc_collection_tasks.status IS '任务状态：DRAFT/READY/RUNNING/SUCCESS/FAILED/STOPPED';
COMMENT ON COLUMN t_dc_collection_tasks.retry_count IS '重试次数';
COMMENT ON COLUMN t_dc_collection_tasks.timeout_seconds IS '超时时间（秒）';
COMMENT ON COLUMN t_dc_collection_tasks.last_execution_id IS '最后执行ID（UUID）';
COMMENT ON COLUMN t_dc_collection_tasks.created_at IS '创建时间';
COMMENT ON COLUMN t_dc_collection_tasks.updated_at IS '更新时间';
COMMENT ON COLUMN t_dc_collection_tasks.created_by IS '创建者';
COMMENT ON COLUMN t_dc_collection_tasks.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dc_tasks_status ON t_dc_collection_tasks(status);
CREATE INDEX IF NOT EXISTS idx_dc_tasks_created_at ON t_dc_collection_tasks(created_at);
CREATE INDEX IF NOT EXISTS idx_dc_tasks_template_id ON t_dc_collection_tasks(template_id);
-- 任务执行明细表
CREATE TABLE IF NOT EXISTS t_dc_task_executions (
id VARCHAR(36) PRIMARY KEY,
task_id VARCHAR(36) NOT NULL,
task_name VARCHAR(255) NOT NULL,
status VARCHAR(20) DEFAULT 'RUNNING',
log_path VARCHAR(1000) NOT NULL,
started_at TIMESTAMP,
completed_at TIMESTAMP,
duration_seconds INTEGER DEFAULT 0,
error_message TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_dc_task_executions IS '任务执行明细表';
COMMENT ON COLUMN t_dc_task_executions.id IS '执行记录ID（UUID）';
COMMENT ON COLUMN t_dc_task_executions.task_id IS '任务ID';
COMMENT ON COLUMN t_dc_task_executions.task_name IS '任务名称';
COMMENT ON COLUMN t_dc_task_executions.status IS '执行状态：RUNNING/SUCCESS/FAILED/STOPPED';
COMMENT ON COLUMN t_dc_task_executions.log_path IS '日志文件路径';
COMMENT ON COLUMN t_dc_task_executions.started_at IS '开始时间';
COMMENT ON COLUMN t_dc_task_executions.completed_at IS '完成时间';
COMMENT ON COLUMN t_dc_task_executions.duration_seconds IS '执行时长（秒）';
COMMENT ON COLUMN t_dc_task_executions.error_message IS '错误信息';
COMMENT ON COLUMN t_dc_task_executions.created_at IS '创建时间';
COMMENT ON COLUMN t_dc_task_executions.updated_at IS '更新时间';
COMMENT ON COLUMN t_dc_task_executions.created_by IS '创建者';
COMMENT ON COLUMN t_dc_task_executions.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dc_executions_task_id ON t_dc_task_executions(task_id);
CREATE INDEX IF NOT EXISTS idx_dc_executions_status ON t_dc_task_executions(status);
CREATE INDEX IF NOT EXISTS idx_dc_executions_started_at ON t_dc_task_executions(started_at);
-- 外键约束
ALTER TABLE t_dc_task_executions ADD CONSTRAINT fk_dc_executions_task FOREIGN KEY (task_id)
REFERENCES t_dc_collection_tasks(id) ON DELETE CASCADE;
-- 数据归集模板配置表
CREATE TABLE IF NOT EXISTS t_dc_collection_templates (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL UNIQUE,
description TEXT,
source_type VARCHAR(64) NOT NULL,
source_name VARCHAR(64) NOT NULL,
target_type VARCHAR(64) NOT NULL,
target_name VARCHAR(64) NOT NULL,
template_content JSONB NOT NULL,
built_in BOOLEAN DEFAULT FALSE,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_dc_collection_templates IS '数据归集模板配置表';
COMMENT ON COLUMN t_dc_collection_templates.id IS '模板ID（UUID）';
COMMENT ON COLUMN t_dc_collection_templates.name IS '模板名称';
COMMENT ON COLUMN t_dc_collection_templates.description IS '模板描述';
COMMENT ON COLUMN t_dc_collection_templates.source_type IS '源数据源类型';
COMMENT ON COLUMN t_dc_collection_templates.source_name IS '源数据源名称';
COMMENT ON COLUMN t_dc_collection_templates.target_type IS '目标数据源类型';
COMMENT ON COLUMN t_dc_collection_templates.target_name IS '目标数据源名称';
COMMENT ON COLUMN t_dc_collection_templates.template_content IS '模板内容';
COMMENT ON COLUMN t_dc_collection_templates.built_in IS '是否系统内置';
COMMENT ON COLUMN t_dc_collection_templates.created_at IS '创建时间';
COMMENT ON COLUMN t_dc_collection_templates.updated_at IS '更新时间';
COMMENT ON COLUMN t_dc_collection_templates.created_by IS '创建者';
COMMENT ON COLUMN t_dc_collection_templates.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dc_templates_source_target ON t_dc_collection_templates(source_type, target_type);
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- 为各个表创建触发器
DROP TRIGGER IF EXISTS update_dc_collection_tasks_updated_at ON t_dc_collection_tasks;
CREATE TRIGGER update_dc_collection_tasks_updated_at
BEFORE UPDATE ON t_dc_collection_tasks
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_dc_task_executions_updated_at ON t_dc_task_executions;
CREATE TRIGGER update_dc_task_executions_updated_at
BEFORE UPDATE ON t_dc_task_executions
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_dc_collection_templates_updated_at ON t_dc_collection_templates;
CREATE TRIGGER update_dc_collection_templates_updated_at
BEFORE UPDATE ON t_dc_collection_templates
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 插入初始数据 - 数据归集模板
INSERT INTO t_dc_collection_templates (
id, name, description, source_type, source_name, target_type, target_name,
template_content, built_in, created_by, updated_by
) VALUES
(
'1',
'NAS归集模板',
'将NAS存储上的文件归集到平台上。',
'nfsreader',
'nfsreader',
'nfswriter',
'nfswriter',
'{"parameter": {"ip": {"name": "NAS地址","description": "NAS服务的地址，可以为IP或者域名。","type": "input", "required": true, "index": 1}, "path": {"name": "共享路径","description": "NAS服务的共享路径。","type": "input", "required": true, "index": 2}, "files": {"name": "文件列表","description": "指定文件列表进行归集。","type": "selectTag", "required": false, "index": 3}}, "reader": {}, "writer": {}}',
TRUE,
'system',
'system'
),
(
'2',
'S3存储归集模板',
'将S3对象存储（如OBS、MinIO、Ceph等）上的文件归集到平台上。',
's3reader',
's3reader',
's3writer',
's3writer',
'{"parameter": {"endpoint": {"name": "服务地址","description": "S3兼容存储的服务地址（如http://minio.example.com:9000）。","type": "input", "required": true, "index": 1}, "bucket": {"name": "存储桶名称","description": "S3存储桶名称。","type": "input", "required": true, "index": 2}, "accessKey": {"name": "Access Key","description": "S3访问密钥（Access Key ID）。","type": "input", "required": true, "index": 3}, "secretKey": {"name": "Secret Key","description": "S3密钥（Secret Access Key）。","type": "password", "required": true, "index": 4}, "prefix": {"name": "匹配前缀","description": "按照匹配前缀选中S3中的文件进行归集。","type": "input", "required": false, "index": 5}, "region": {"name": "区域","description": "S3区域（默认us-east-1）。","type": "input", "required": false, "index": 6}}, "reader": {}, "writer": {}}',
TRUE,
'system',
'system'
),
(
'4',
'StarRocks归集模板',
'将StarRocks中的数据以csv文件的形式归集到DeepData平台上。',
'starrocksreader',
'starrocksreader',
'txtfilewriter',
'txtfilewriter',
'{"parameter": {}, "reader": {"username": {"name": "用户名","description": "数据库的用户名。","type": "input", "required": true, "index": 2}, "password": {"name": "密码","description": "数据库的密码。","type": "password", "required": true, "index": 3}, "connection": {"name": "数据库连接信息", "description": "数据库连接信息。", "type": "multipleList", "size": 1, "index": 1, "properties": {"jdbcUrl": {"type": "inputList", "name": "数据库连接", "description": "数据库连接url。", "required": true, "index": 1}, "querySql": {"type": "inputList", "name": "查询sql", "description": "输入符合语法的sql查询语句。", "required": true, "index": 2}}}}, "writer": {"header": {"name": "列名","description": "查询结果的列名，最终会体现为csv文件的表头。","type": "selectTag", "required": false}}}',
TRUE,
'system',
'system'
),
(
'5',
'GlusterFS归集模板',
'将GlusterFS分布式文件系统上的文件归集到平台上。',
'glusterfsreader',
'glusterfsreader',
'glusterfswriter',
'glusterfswriter',
'{"parameter": {"ip": {"name": "服务器地址","description": "GlusterFS服务器的IP地址或域名。","type": "input", "required": true, "index": 1}, "volume": {"name": "卷名称","description": "GlusterFS卷名称。","type": "input", "required": true, "index": 2}, "path": {"name": "子路径","description": "卷内的子目录路径（可选）。","type": "input", "required": false, "index": 3}, "files": {"name": "文件列表","description": "指定文件列表进行归集。","type": "selectTag", "required": false, "index": 4}}, "reader": {}, "writer": {}}',
TRUE,
'system',
'system'),
(
'6',
'API归集模板',
'将指定API返回的数据以csv文件的形式归集到平台上。',
'apireader',
'apireader',
'txtfilewriter',
'txtfilewriter',
'{"parameter": {}, "reader": {"api": {"name": "接口地址","description": "API的访问地址。","type": "input", "required": true, "index": 1}, "method": {"name": "请求方式","description": "API的请求方式（默认为GET）。","type": "option", "options": ["GET", "POST"], "required": false, "index": 2}, "body": {"name": "请求体参数","description": "请求体参数，主要针对POST请求。示例：\n {\"query\": \"value\"}","type": "jsonobject", "required": false, "index": 3}, "headers": {"name": "请求头参数","description": "需要设置的请求头参数。示例：\n {\"Authorization\": \"***\"}","type": "jsonobject", "required": false, "index": 4}, "schema": {"name": "数据解析schema","description": "会依据schema解析API返回结果。示例：\n {\"dataPath\": \"返回体内定位到具体数据的路径，形如$.data\", \"fields\": [{\"name\": \"属性名\", \"alias\": \"别名\", \"path\": \"属性的路径\"}]}","type": "jsonobject", "required": true, "index": 5}}, "writer": {}}',
TRUE,
'system',
'system'),
(
'7',
'通用关系型数据库归集模板',
'将关系型数据库中的数据以csv文件的形式归集到平台上。当前支持postgres、opengauss、sqlserver、mysql、达梦、db2格式、PPAS格式等关系型数据库。',
'rdbmsreader',
'rdbmsreader',
'txtfilewriter',
'txtfilewriter',
'{"parameter": {}, "reader": {"username": {"name": "用户名","description": "数据库的用户名。","type": "input", "required": true, "index": 2}, "password": {"name": "密码","description": "数据库的密码。","type": "password", "required": true, "index": 3}, "connection": {"name": "数据库连接信息", "description": "数据库连接信息。", "type": "multipleList", "size": 1, "index": 1, "properties": {"jdbcUrl": {"type": "inputList", "name": "数据库连接", "description": "数据库连接url。", "required": true, "index": 1}, "querySql": {"type": "inputList", "name": "查询sql", "description": "输入符合语法的sql查询语句。", "required": true, "index": 2}}}}, "writer": {"header": {"name": "列名","description": "查询结果的列名，最终会体现为csv文件的表头。","type": "selectTag", "required": false}}}',
TRUE,
'system',
'system'
)
ON CONFLICT (id) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
source_type = EXCLUDED.source_type,
source_name = EXCLUDED.source_name,
target_type = EXCLUDED.target_type,
target_name = EXCLUDED.target_name,
template_content = EXCLUDED.template_content,
built_in = EXCLUDED.built_in,
created_by = EXCLUDED.created_by,
updated_by = EXCLUDED.updated_by,
updated_at = CURRENT_TIMESTAMP;
scripts/db/data-common-init.sql
+84 -84
View File
@@ -1,85 +1,85 @@
-- 使用现有的deepdata数据库
\c deepdata;
-- 文件切片上传请求表
CREATE TABLE IF NOT EXISTS t_chunk_upload_request
(
id VARCHAR(36) PRIMARY KEY,
total_file_num INTEGER,
uploaded_file_num INTEGER,
upload_path VARCHAR(256),
timeout TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
service_id VARCHAR(64),
check_info TEXT
);
-- 添加注释
COMMENT ON TABLE t_chunk_upload_request IS '文件切片上传请求表';
COMMENT ON COLUMN t_chunk_upload_request.id IS 'UUID';
COMMENT ON COLUMN t_chunk_upload_request.total_file_num IS '总文件数';
COMMENT ON COLUMN t_chunk_upload_request.uploaded_file_num IS '已上传文件数';
COMMENT ON COLUMN t_chunk_upload_request.upload_path IS '文件路径';
COMMENT ON COLUMN t_chunk_upload_request.timeout IS '上传请求超时时间';
COMMENT ON COLUMN t_chunk_upload_request.service_id IS '上传请求所属服务：DATA-MANAGEMENT(数据管理);';
COMMENT ON COLUMN t_chunk_upload_request.check_info IS '业务信息';
-- 如果需要索引可以添加
CREATE INDEX IF NOT EXISTS idx_chunk_upload_service ON t_chunk_upload_request(service_id);
-- =========================
-- 数据血缘：节点表
-- 节点表示实体对象（归集来源、数据集、知识库、模型等）
-- =========================
CREATE TABLE IF NOT EXISTS t_lineage_node
(
id VARCHAR(36) PRIMARY KEY,
graph_id VARCHAR(36) NOT NULL,
node_type VARCHAR(64) NOT NULL,
name VARCHAR(256) NOT NULL,
description TEXT,
node_metadata TEXT
);
COMMENT ON TABLE t_lineage_node IS '数据血缘节点表（实体对象）';
COMMENT ON COLUMN t_lineage_node.id IS '主键ID';
COMMENT ON COLUMN t_lineage_node.graph_id IS '图ID';
COMMENT ON COLUMN t_lineage_node.node_type IS '节点类型：DATASOURCE/DATASET/KNOWLEDGE_BASE/MODEL等';
COMMENT ON COLUMN t_lineage_node.name IS '节点名称';
COMMENT ON COLUMN t_lineage_node.description IS '节点描述';
COMMENT ON COLUMN t_lineage_node.node_metadata IS '节点扩展信息（JSON）';
CREATE INDEX IF NOT EXISTS idx_lineage_node_id ON t_lineage_node(id);
CREATE INDEX IF NOT EXISTS idx_lineage_graph_id ON t_lineage_node(graph_id);
-- =========================
-- 数据血缘：边表
-- 边表示处理流程（归集任务、数据清洗、数据标注、数据合成、数据配比等）
-- =========================
CREATE TABLE IF NOT EXISTS t_lineage_edge
(
id VARCHAR(36) PRIMARY KEY,
process_id VARCHAR(36) NOT NULL,
graph_id VARCHAR(36) NOT NULL,
edge_type VARCHAR(64) NOT NULL,
name VARCHAR(256),
description TEXT,
edge_metadata TEXT,
from_node_id VARCHAR(36) NOT NULL,
to_node_id VARCHAR(36) NOT NULL
);
COMMENT ON TABLE t_lineage_edge IS '数据血缘边表（处理流程）';
COMMENT ON COLUMN t_lineage_edge.id IS '边ID';
COMMENT ON COLUMN t_lineage_edge.graph_id IS '图ID';
COMMENT ON COLUMN t_lineage_edge.process_id IS '处理流程ID';
COMMENT ON COLUMN t_lineage_edge.edge_type IS '边类型：DATA_COLLECTION/DATA_CLEANING/DATA_LABELING/DATA_SYNTHESIS/DATA_RATIO等';
COMMENT ON COLUMN t_lineage_edge.name IS '边名称';
COMMENT ON COLUMN t_lineage_edge.description IS '边描述';
COMMENT ON COLUMN t_lineage_edge.edge_metadata IS '边扩展信息（JSON）';
COMMENT ON COLUMN t_lineage_edge.from_node_id IS '源节点ID';
COMMENT ON COLUMN t_lineage_edge.to_node_id IS '目标节点ID';
CREATE INDEX IF NOT EXISTS idx_lineage_process_id ON t_lineage_edge(process_id);
CREATE INDEX IF NOT EXISTS idx_lineage_edge_graph_id ON t_lineage_edge(graph_id);
CREATE INDEX IF NOT EXISTS idx_lineage_edge_from ON t_lineage_edge(from_node_id);
-- 使用现有的DeepData数据库
\c deepdata;
-- 文件切片上传请求表
CREATE TABLE IF NOT EXISTS t_chunk_upload_request
(
id VARCHAR(36) PRIMARY KEY,
total_file_num INTEGER,
uploaded_file_num INTEGER,
upload_path VARCHAR(256),
timeout TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
service_id VARCHAR(64),
check_info TEXT
);
-- 添加注释
COMMENT ON TABLE t_chunk_upload_request IS '文件切片上传请求表';
COMMENT ON COLUMN t_chunk_upload_request.id IS 'UUID';
COMMENT ON COLUMN t_chunk_upload_request.total_file_num IS '总文件数';
COMMENT ON COLUMN t_chunk_upload_request.uploaded_file_num IS '已上传文件数';
COMMENT ON COLUMN t_chunk_upload_request.upload_path IS '文件路径';
COMMENT ON COLUMN t_chunk_upload_request.timeout IS '上传请求超时时间';
COMMENT ON COLUMN t_chunk_upload_request.service_id IS '上传请求所属服务：DATA-MANAGEMENT(数据管理);';
COMMENT ON COLUMN t_chunk_upload_request.check_info IS '业务信息';
-- 如果需要索引可以添加
CREATE INDEX IF NOT EXISTS idx_chunk_upload_service ON t_chunk_upload_request(service_id);
-- =========================
-- 数据血缘：节点表
-- 节点表示实体对象（归集来源、数据集、知识库、模型等）
-- =========================
CREATE TABLE IF NOT EXISTS t_lineage_node
(
id VARCHAR(36) PRIMARY KEY,
graph_id VARCHAR(36) NOT NULL,
node_type VARCHAR(64) NOT NULL,
name VARCHAR(256) NOT NULL,
description TEXT,
node_metadata TEXT
);
COMMENT ON TABLE t_lineage_node IS '数据血缘节点表（实体对象）';
COMMENT ON COLUMN t_lineage_node.id IS '主键ID';
COMMENT ON COLUMN t_lineage_node.graph_id IS '图ID';
COMMENT ON COLUMN t_lineage_node.node_type IS '节点类型：DATASOURCE/DATASET/KNOWLEDGE_BASE/MODEL等';
COMMENT ON COLUMN t_lineage_node.name IS '节点名称';
COMMENT ON COLUMN t_lineage_node.description IS '节点描述';
COMMENT ON COLUMN t_lineage_node.node_metadata IS '节点扩展信息（JSON）';
CREATE INDEX IF NOT EXISTS idx_lineage_node_id ON t_lineage_node(id);
CREATE INDEX IF NOT EXISTS idx_lineage_graph_id ON t_lineage_node(graph_id);
-- =========================
-- 数据血缘：边表
-- 边表示处理流程（归集任务、数据清洗、数据标注、数据合成、数据配比等）
-- =========================
CREATE TABLE IF NOT EXISTS t_lineage_edge
(
id VARCHAR(36) PRIMARY KEY,
process_id VARCHAR(36) NOT NULL,
graph_id VARCHAR(36) NOT NULL,
edge_type VARCHAR(64) NOT NULL,
name VARCHAR(256),
description TEXT,
edge_metadata TEXT,
from_node_id VARCHAR(36) NOT NULL,
to_node_id VARCHAR(36) NOT NULL
);
COMMENT ON TABLE t_lineage_edge IS '数据血缘边表（处理流程）';
COMMENT ON COLUMN t_lineage_edge.id IS '边ID';
COMMENT ON COLUMN t_lineage_edge.graph_id IS '图ID';
COMMENT ON COLUMN t_lineage_edge.process_id IS '处理流程ID';
COMMENT ON COLUMN t_lineage_edge.edge_type IS '边类型：DATA_COLLECTION/DATA_CLEANING/DATA_LABELING/DATA_SYNTHESIS/DATA_RATIO等';
COMMENT ON COLUMN t_lineage_edge.name IS '边名称';
COMMENT ON COLUMN t_lineage_edge.description IS '边描述';
COMMENT ON COLUMN t_lineage_edge.edge_metadata IS '边扩展信息（JSON）';
COMMENT ON COLUMN t_lineage_edge.from_node_id IS '源节点ID';
COMMENT ON COLUMN t_lineage_edge.to_node_id IS '目标节点ID';
CREATE INDEX IF NOT EXISTS idx_lineage_process_id ON t_lineage_edge(process_id);
CREATE INDEX IF NOT EXISTS idx_lineage_edge_graph_id ON t_lineage_edge(graph_id);
CREATE INDEX IF NOT EXISTS idx_lineage_edge_from ON t_lineage_edge(from_node_id);
CREATE INDEX IF NOT EXISTS idx_lineage_edge_to ON t_lineage_edge(to_node_id);
scripts/db/data-evaluation-init.sql
+151 -151
View File
@@ -1,151 +1,151 @@
-- 使用现有的deepdata数据库
\c deepdata;
-- 评估任务表
CREATE TABLE IF NOT EXISTS t_de_eval_task (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
description TEXT,
task_type VARCHAR(50) NOT NULL,
source_type VARCHAR(36),
source_id VARCHAR(36),
source_name VARCHAR(255),
status VARCHAR(50) DEFAULT 'PENDING',
eval_method VARCHAR(50) DEFAULT 'AUTO',
eval_process DOUBLE PRECISION NOT NULL DEFAULT 0,
eval_prompt TEXT,
eval_config TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_de_eval_task IS '评估任务表（UUID 主键）';
COMMENT ON COLUMN t_de_eval_task.id IS 'UUID';
COMMENT ON COLUMN t_de_eval_task.name IS '评估任务名称';
COMMENT ON COLUMN t_de_eval_task.description IS '评估任务描述';
COMMENT ON COLUMN t_de_eval_task.task_type IS '评估任务类型：QA';
COMMENT ON COLUMN t_de_eval_task.source_type IS '待评估对象类型：DATASET/SYNTHESIS';
COMMENT ON COLUMN t_de_eval_task.source_id IS '待评估对象ID';
COMMENT ON COLUMN t_de_eval_task.source_name IS '待评估对象名称';
COMMENT ON COLUMN t_de_eval_task.status IS '状态：PENDING/RUNNING/COMPLETED/STOPPED/FAILED';
COMMENT ON COLUMN t_de_eval_task.eval_method IS '评估方式：AUTO/MANUAL';
COMMENT ON COLUMN t_de_eval_task.eval_process IS '评估进度';
COMMENT ON COLUMN t_de_eval_task.eval_prompt IS '评估提示词';
COMMENT ON COLUMN t_de_eval_task.eval_config IS '评估配置';
COMMENT ON COLUMN t_de_eval_task.created_at IS '创建时间';
COMMENT ON COLUMN t_de_eval_task.updated_at IS '更新时间';
COMMENT ON COLUMN t_de_eval_task.created_by IS '创建者';
COMMENT ON COLUMN t_de_eval_task.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_de_eval_task_status ON t_de_eval_task(status);
CREATE INDEX IF NOT EXISTS idx_de_eval_task_created_at ON t_de_eval_task(created_at);
CREATE INDEX IF NOT EXISTS idx_de_eval_task_source_id ON t_de_eval_task(source_id);
-- 评估文件表
CREATE TABLE IF NOT EXISTS t_de_eval_file (
id VARCHAR(36) PRIMARY KEY,
task_id VARCHAR(36) NOT NULL,
file_id VARCHAR(36),
file_name VARCHAR(255),
total_count INTEGER DEFAULT 0,
evaluated_count INTEGER DEFAULT 0,
error_message TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_de_eval_file IS '评估文件表（UUID 主键）';
COMMENT ON COLUMN t_de_eval_file.id IS 'UUID';
COMMENT ON COLUMN t_de_eval_file.task_id IS '评估任务ID';
COMMENT ON COLUMN t_de_eval_file.file_id IS '文件ID';
COMMENT ON COLUMN t_de_eval_file.file_name IS '文件名';
COMMENT ON COLUMN t_de_eval_file.total_count IS '总数';
COMMENT ON COLUMN t_de_eval_file.evaluated_count IS '已评估数';
COMMENT ON COLUMN t_de_eval_file.error_message IS '错误信息';
COMMENT ON COLUMN t_de_eval_file.created_at IS '创建时间';
COMMENT ON COLUMN t_de_eval_file.updated_at IS '更新时间';
COMMENT ON COLUMN t_de_eval_file.created_by IS '创建者';
COMMENT ON COLUMN t_de_eval_file.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_de_eval_file_task_id ON t_de_eval_file(task_id);
CREATE INDEX IF NOT EXISTS idx_de_eval_file_file_id ON t_de_eval_file(file_id);
-- 评估条目表
CREATE TABLE IF NOT EXISTS t_de_eval_item (
id VARCHAR(36) PRIMARY KEY,
task_id VARCHAR(36) NOT NULL,
file_id VARCHAR(36),
item_id VARCHAR(36) NOT NULL,
eval_content TEXT,
eval_score DOUBLE PRECISION NOT NULL DEFAULT 0,
eval_result TEXT,
status VARCHAR(50) DEFAULT 'PENDING',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_de_eval_item IS '评估条目表（UUID 主键）';
COMMENT ON COLUMN t_de_eval_item.id IS 'UUID';
COMMENT ON COLUMN t_de_eval_item.task_id IS '评估任务ID';
COMMENT ON COLUMN t_de_eval_item.file_id IS '文件ID';
COMMENT ON COLUMN t_de_eval_item.item_id IS '评估条目ID';
COMMENT ON COLUMN t_de_eval_item.eval_content IS '评估内容';
COMMENT ON COLUMN t_de_eval_item.eval_score IS '评估分数';
COMMENT ON COLUMN t_de_eval_item.eval_result IS '评估结果';
COMMENT ON COLUMN t_de_eval_item.status IS '状态：PENDING/EVALUATED';
COMMENT ON COLUMN t_de_eval_item.created_at IS '创建时间';
COMMENT ON COLUMN t_de_eval_item.updated_at IS '更新时间';
COMMENT ON COLUMN t_de_eval_item.created_by IS '创建者';
COMMENT ON COLUMN t_de_eval_item.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_de_eval_item_task_id ON t_de_eval_item(task_id);
CREATE INDEX IF NOT EXISTS idx_de_eval_item_file_id ON t_de_eval_item(file_id);
CREATE INDEX IF NOT EXISTS idx_de_eval_item_status ON t_de_eval_item(status);
CREATE INDEX IF NOT EXISTS idx_de_eval_item_item_id ON t_de_eval_item(item_id);
-- 创建外键约束
ALTER TABLE t_de_eval_file ADD CONSTRAINT fk_de_eval_file_task FOREIGN KEY (task_id)
REFERENCES t_de_eval_task(id) ON DELETE CASCADE;
ALTER TABLE t_de_eval_item ADD CONSTRAINT fk_de_eval_item_task FOREIGN KEY (task_id)
REFERENCES t_de_eval_task(id) ON DELETE CASCADE;
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- 为各个表创建触发器
DROP TRIGGER IF EXISTS update_de_eval_task_updated_at ON t_de_eval_task;
CREATE TRIGGER update_de_eval_task_updated_at
BEFORE UPDATE ON t_de_eval_task
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_de_eval_file_updated_at ON t_de_eval_file;
CREATE TRIGGER update_de_eval_file_updated_at
BEFORE UPDATE ON t_de_eval_file
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_de_eval_item_updated_at ON t_de_eval_item;
CREATE TRIGGER update_de_eval_item_updated_at
BEFORE UPDATE ON t_de_eval_item
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 使用现有的DeepData数据库
\c deepdata;
-- 评估任务表
CREATE TABLE IF NOT EXISTS t_de_eval_task (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
description TEXT,
task_type VARCHAR(50) NOT NULL,
source_type VARCHAR(36),
source_id VARCHAR(36),
source_name VARCHAR(255),
status VARCHAR(50) DEFAULT 'PENDING',
eval_method VARCHAR(50) DEFAULT 'AUTO',
eval_process DOUBLE PRECISION NOT NULL DEFAULT 0,
eval_prompt TEXT,
eval_config TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_de_eval_task IS '评估任务表（UUID 主键）';
COMMENT ON COLUMN t_de_eval_task.id IS 'UUID';
COMMENT ON COLUMN t_de_eval_task.name IS '评估任务名称';
COMMENT ON COLUMN t_de_eval_task.description IS '评估任务描述';
COMMENT ON COLUMN t_de_eval_task.task_type IS '评估任务类型：QA';
COMMENT ON COLUMN t_de_eval_task.source_type IS '待评估对象类型：DATASET/SYNTHESIS';
COMMENT ON COLUMN t_de_eval_task.source_id IS '待评估对象ID';
COMMENT ON COLUMN t_de_eval_task.source_name IS '待评估对象名称';
COMMENT ON COLUMN t_de_eval_task.status IS '状态：PENDING/RUNNING/COMPLETED/STOPPED/FAILED';
COMMENT ON COLUMN t_de_eval_task.eval_method IS '评估方式：AUTO/MANUAL';
COMMENT ON COLUMN t_de_eval_task.eval_process IS '评估进度';
COMMENT ON COLUMN t_de_eval_task.eval_prompt IS '评估提示词';
COMMENT ON COLUMN t_de_eval_task.eval_config IS '评估配置';
COMMENT ON COLUMN t_de_eval_task.created_at IS '创建时间';
COMMENT ON COLUMN t_de_eval_task.updated_at IS '更新时间';
COMMENT ON COLUMN t_de_eval_task.created_by IS '创建者';
COMMENT ON COLUMN t_de_eval_task.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_de_eval_task_status ON t_de_eval_task(status);
CREATE INDEX IF NOT EXISTS idx_de_eval_task_created_at ON t_de_eval_task(created_at);
CREATE INDEX IF NOT EXISTS idx_de_eval_task_source_id ON t_de_eval_task(source_id);
-- 评估文件表
CREATE TABLE IF NOT EXISTS t_de_eval_file (
id VARCHAR(36) PRIMARY KEY,
task_id VARCHAR(36) NOT NULL,
file_id VARCHAR(36),
file_name VARCHAR(255),
total_count INTEGER DEFAULT 0,
evaluated_count INTEGER DEFAULT 0,
error_message TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_de_eval_file IS '评估文件表（UUID 主键）';
COMMENT ON COLUMN t_de_eval_file.id IS 'UUID';
COMMENT ON COLUMN t_de_eval_file.task_id IS '评估任务ID';
COMMENT ON COLUMN t_de_eval_file.file_id IS '文件ID';
COMMENT ON COLUMN t_de_eval_file.file_name IS '文件名';
COMMENT ON COLUMN t_de_eval_file.total_count IS '总数';
COMMENT ON COLUMN t_de_eval_file.evaluated_count IS '已评估数';
COMMENT ON COLUMN t_de_eval_file.error_message IS '错误信息';
COMMENT ON COLUMN t_de_eval_file.created_at IS '创建时间';
COMMENT ON COLUMN t_de_eval_file.updated_at IS '更新时间';
COMMENT ON COLUMN t_de_eval_file.created_by IS '创建者';
COMMENT ON COLUMN t_de_eval_file.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_de_eval_file_task_id ON t_de_eval_file(task_id);
CREATE INDEX IF NOT EXISTS idx_de_eval_file_file_id ON t_de_eval_file(file_id);
-- 评估条目表
CREATE TABLE IF NOT EXISTS t_de_eval_item (
id VARCHAR(36) PRIMARY KEY,
task_id VARCHAR(36) NOT NULL,
file_id VARCHAR(36),
item_id VARCHAR(36) NOT NULL,
eval_content TEXT,
eval_score DOUBLE PRECISION NOT NULL DEFAULT 0,
eval_result TEXT,
status VARCHAR(50) DEFAULT 'PENDING',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_de_eval_item IS '评估条目表（UUID 主键）';
COMMENT ON COLUMN t_de_eval_item.id IS 'UUID';
COMMENT ON COLUMN t_de_eval_item.task_id IS '评估任务ID';
COMMENT ON COLUMN t_de_eval_item.file_id IS '文件ID';
COMMENT ON COLUMN t_de_eval_item.item_id IS '评估条目ID';
COMMENT ON COLUMN t_de_eval_item.eval_content IS '评估内容';
COMMENT ON COLUMN t_de_eval_item.eval_score IS '评估分数';
COMMENT ON COLUMN t_de_eval_item.eval_result IS '评估结果';
COMMENT ON COLUMN t_de_eval_item.status IS '状态：PENDING/EVALUATED';
COMMENT ON COLUMN t_de_eval_item.created_at IS '创建时间';
COMMENT ON COLUMN t_de_eval_item.updated_at IS '更新时间';
COMMENT ON COLUMN t_de_eval_item.created_by IS '创建者';
COMMENT ON COLUMN t_de_eval_item.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_de_eval_item_task_id ON t_de_eval_item(task_id);
CREATE INDEX IF NOT EXISTS idx_de_eval_item_file_id ON t_de_eval_item(file_id);
CREATE INDEX IF NOT EXISTS idx_de_eval_item_status ON t_de_eval_item(status);
CREATE INDEX IF NOT EXISTS idx_de_eval_item_item_id ON t_de_eval_item(item_id);
-- 创建外键约束
ALTER TABLE t_de_eval_file ADD CONSTRAINT fk_de_eval_file_task FOREIGN KEY (task_id)
REFERENCES t_de_eval_task(id) ON DELETE CASCADE;
ALTER TABLE t_de_eval_item ADD CONSTRAINT fk_de_eval_item_task FOREIGN KEY (task_id)
REFERENCES t_de_eval_task(id) ON DELETE CASCADE;
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- 为各个表创建触发器
DROP TRIGGER IF EXISTS update_de_eval_task_updated_at ON t_de_eval_task;
CREATE TRIGGER update_de_eval_task_updated_at
BEFORE UPDATE ON t_de_eval_task
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_de_eval_file_updated_at ON t_de_eval_file;
CREATE TRIGGER update_de_eval_file_updated_at
BEFORE UPDATE ON t_de_eval_file
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_de_eval_item_updated_at ON t_de_eval_item;
CREATE TRIGGER update_de_eval_item_updated_at
BEFORE UPDATE ON t_de_eval_item
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
scripts/db/data-management-init.sql
+318 -315
View File
@@ -1,315 +1,318 @@
-- 使用现有的deepdata数据库
\c deepdata;
-- 数据集表
CREATE TABLE IF NOT EXISTS t_dm_datasets (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
description TEXT,
dataset_type VARCHAR(50) NOT NULL,
category VARCHAR(100),
path VARCHAR(500),
format VARCHAR(50),
schema_info JSONB,
size_bytes BIGINT DEFAULT 0,
file_count BIGINT DEFAULT 0,
record_count BIGINT DEFAULT 0,
retention_days INTEGER DEFAULT 0,
tags JSONB,
metadata JSONB,
status VARCHAR(50) DEFAULT 'DRAFT',
is_public BOOLEAN DEFAULT FALSE,
is_featured BOOLEAN DEFAULT FALSE,
version BIGINT NOT NULL DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
COMMENT ON TABLE t_dm_datasets IS '数据集表（UUID 主键）';
COMMENT ON COLUMN t_dm_datasets.id IS 'UUID';
COMMENT ON COLUMN t_dm_datasets.name IS '数据集名称';
COMMENT ON COLUMN t_dm_datasets.description IS '数据集描述';
COMMENT ON COLUMN t_dm_datasets.dataset_type IS '数据集类型：IMAGE/TEXT/QA/MULTIMODAL/OTHER';
COMMENT ON COLUMN t_dm_datasets.category IS '数据集分类：医学影像/问答/文献等';
COMMENT ON COLUMN t_dm_datasets.path IS '数据存储路径';
COMMENT ON COLUMN t_dm_datasets.format IS '数据格式：DCM/JPG/JSON/CSV等';
COMMENT ON COLUMN t_dm_datasets.schema_info IS '数据结构信息';
COMMENT ON COLUMN t_dm_datasets.size_bytes IS '数据大小(字节)';
COMMENT ON COLUMN t_dm_datasets.file_count IS '文件数量';
COMMENT ON COLUMN t_dm_datasets.record_count IS '记录数量';
COMMENT ON COLUMN t_dm_datasets.retention_days IS '数据保留天数（0表示长期保留）';
COMMENT ON COLUMN t_dm_datasets.tags IS '标签列表';
COMMENT ON COLUMN t_dm_datasets.metadata IS '元数据信息';
COMMENT ON COLUMN t_dm_datasets.status IS '状态：DRAFT/ACTIVE/ARCHIVED';
COMMENT ON COLUMN t_dm_datasets.is_public IS '是否公开';
COMMENT ON COLUMN t_dm_datasets.is_featured IS '是否推荐';
COMMENT ON COLUMN t_dm_datasets.version IS '版本号';
COMMENT ON COLUMN t_dm_datasets.created_at IS '创建时间';
COMMENT ON COLUMN t_dm_datasets.updated_at IS '更新时间';
COMMENT ON COLUMN t_dm_datasets.created_by IS '创建者';
COMMENT ON COLUMN t_dm_datasets.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dm_dataset_type ON t_dm_datasets(dataset_type);
CREATE INDEX IF NOT EXISTS idx_dm_category ON t_dm_datasets(category);
CREATE INDEX IF NOT EXISTS idx_dm_format ON t_dm_datasets(format);
CREATE INDEX IF NOT EXISTS idx_dm_status ON t_dm_datasets(status);
CREATE INDEX IF NOT EXISTS idx_dm_public ON t_dm_datasets(is_public);
CREATE INDEX IF NOT EXISTS idx_dm_featured ON t_dm_datasets(is_featured);
CREATE INDEX IF NOT EXISTS idx_dm_created_at ON t_dm_datasets(created_at);
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
DROP TRIGGER IF EXISTS update_t_dm_datasets_updated_at ON t_dm_datasets;
CREATE TRIGGER update_t_dm_datasets_updated_at
BEFORE UPDATE ON t_dm_datasets
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 数据集文件表
CREATE TABLE IF NOT EXISTS t_dm_dataset_files (
id VARCHAR(36) PRIMARY KEY,
dataset_id VARCHAR(36) NOT NULL,
file_name VARCHAR(255) NOT NULL,
file_path VARCHAR(1000) NOT NULL,
file_type VARCHAR(50),
file_size BIGINT DEFAULT 0,
check_sum VARCHAR(64),
tags JSONB,
tags_updated_at TIMESTAMP,
annotation JSONB,
metadata JSONB,
status VARCHAR(50) DEFAULT 'ACTIVE',
upload_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
last_access_time TIMESTAMP,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
COMMENT ON TABLE t_dm_dataset_files IS '数据集文件表（UUID 主键）';
COMMENT ON COLUMN t_dm_dataset_files.id IS 'UUID';
COMMENT ON COLUMN t_dm_dataset_files.dataset_id IS '所属数据集ID（UUID）';
COMMENT ON COLUMN t_dm_dataset_files.file_name IS '文件名';
COMMENT ON COLUMN t_dm_dataset_files.file_path IS '文件路径';
COMMENT ON COLUMN t_dm_dataset_files.file_type IS '文件格式：JPG/PNG/DCM/TXT等';
COMMENT ON COLUMN t_dm_dataset_files.file_size IS '文件大小(字节)';
COMMENT ON COLUMN t_dm_dataset_files.check_sum IS '文件校验和';
COMMENT ON COLUMN t_dm_dataset_files.tags IS '文件标签信息（结构化标签/标注概要）';
COMMENT ON COLUMN t_dm_dataset_files.tags_updated_at IS '标签最后更新时间';
COMMENT ON COLUMN t_dm_dataset_files.annotation IS '完整标注结果（原始JSON）';
COMMENT ON COLUMN t_dm_dataset_files.metadata IS '文件元数据';
COMMENT ON COLUMN t_dm_dataset_files.status IS '文件状态：ACTIVE/DELETED/PROCESSING';
COMMENT ON COLUMN t_dm_dataset_files.upload_time IS '上传时间';
COMMENT ON COLUMN t_dm_dataset_files.last_access_time IS '最后访问时间';
COMMENT ON COLUMN t_dm_dataset_files.created_at IS '创建时间';
COMMENT ON COLUMN t_dm_dataset_files.updated_at IS '更新时间';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dm_dataset ON t_dm_dataset_files(dataset_id);
CREATE INDEX IF NOT EXISTS idx_dm_file_type ON t_dm_dataset_files(file_type);
CREATE INDEX IF NOT EXISTS idx_dm_file_status ON t_dm_dataset_files(status);
CREATE INDEX IF NOT EXISTS idx_dm_upload_time ON t_dm_dataset_files(upload_time);
-- 外键约束
ALTER TABLE t_dm_dataset_files
ADD CONSTRAINT fk_dm_dataset_files_dataset
FOREIGN KEY (dataset_id)
REFERENCES t_dm_datasets(id)
ON DELETE CASCADE;
-- 创建触发器
DROP TRIGGER IF EXISTS update_t_dm_dataset_files_updated_at ON t_dm_dataset_files;
CREATE TRIGGER update_t_dm_dataset_files_updated_at
BEFORE UPDATE ON t_dm_dataset_files
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 数据集统计信息表
CREATE TABLE IF NOT EXISTS t_dm_dataset_statistics (
id VARCHAR(36) PRIMARY KEY,
dataset_id VARCHAR(36) NOT NULL,
stat_date DATE NOT NULL,
total_files BIGINT DEFAULT 0,
total_size BIGINT DEFAULT 0,
processed_files BIGINT DEFAULT 0,
error_files BIGINT DEFAULT 0,
download_count BIGINT DEFAULT 0,
view_count BIGINT DEFAULT 0,
quality_metrics JSONB,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
COMMENT ON TABLE t_dm_dataset_statistics IS '数据集统计信息表（UUID 主键）';
COMMENT ON COLUMN t_dm_dataset_statistics.id IS 'UUID';
COMMENT ON COLUMN t_dm_dataset_statistics.dataset_id IS '数据集ID（UUID）';
COMMENT ON COLUMN t_dm_dataset_statistics.stat_date IS '统计日期';
COMMENT ON COLUMN t_dm_dataset_statistics.total_files IS '总文件数';
COMMENT ON COLUMN t_dm_dataset_statistics.total_size IS '总大小(字节)';
COMMENT ON COLUMN t_dm_dataset_statistics.processed_files IS '已处理文件数';
COMMENT ON COLUMN t_dm_dataset_statistics.error_files IS '错误文件数';
COMMENT ON COLUMN t_dm_dataset_statistics.download_count IS '下载次数';
COMMENT ON COLUMN t_dm_dataset_statistics.view_count IS '查看次数';
COMMENT ON COLUMN t_dm_dataset_statistics.quality_metrics IS '质量指标';
COMMENT ON COLUMN t_dm_dataset_statistics.created_at IS '创建时间';
COMMENT ON COLUMN t_dm_dataset_statistics.updated_at IS '更新时间';
-- 创建唯一约束和索引
ALTER TABLE t_dm_dataset_statistics
ADD CONSTRAINT uk_dm_dataset_date
UNIQUE (dataset_id, stat_date);
CREATE INDEX IF NOT EXISTS idx_dm_stat_date ON t_dm_dataset_statistics(stat_date);
-- 外键约束
ALTER TABLE t_dm_dataset_statistics
ADD CONSTRAINT fk_dm_dataset_statistics_dataset
FOREIGN KEY (dataset_id)
REFERENCES t_dm_datasets(id)
ON DELETE CASCADE;
-- 创建触发器
DROP TRIGGER IF EXISTS update_t_dm_dataset_statistics_updated_at ON t_dm_dataset_statistics;
CREATE TRIGGER update_t_dm_dataset_statistics_updated_at
BEFORE UPDATE ON t_dm_dataset_statistics
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 标签表
CREATE TABLE IF NOT EXISTS t_dm_tags (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(100) NOT NULL UNIQUE,
description TEXT,
category VARCHAR(50),
color VARCHAR(7),
usage_count BIGINT DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
COMMENT ON TABLE t_dm_tags IS '标签表（UUID 主键）';
COMMENT ON COLUMN t_dm_tags.id IS 'UUID';
COMMENT ON COLUMN t_dm_tags.name IS '标签名称';
COMMENT ON COLUMN t_dm_tags.description IS '标签描述';
COMMENT ON COLUMN t_dm_tags.category IS '标签分类';
COMMENT ON COLUMN t_dm_tags.color IS '标签颜色(十六进制)';
COMMENT ON COLUMN t_dm_tags.usage_count IS '使用次数';
COMMENT ON COLUMN t_dm_tags.created_at IS '创建时间';
COMMENT ON COLUMN t_dm_tags.updated_at IS '更新时间';
COMMENT ON COLUMN t_dm_tags.created_by IS '创建者';
COMMENT ON COLUMN t_dm_tags.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dm_tag_category ON t_dm_tags(category);
CREATE INDEX IF NOT EXISTS idx_dm_tag_usage_count ON t_dm_tags(usage_count);
-- 创建触发器
DROP TRIGGER IF EXISTS update_t_dm_tags_updated_at ON t_dm_tags;
CREATE TRIGGER update_t_dm_tags_updated_at
BEFORE UPDATE ON t_dm_tags
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 数据集标签关联表
CREATE TABLE IF NOT EXISTS t_dm_dataset_tags (
dataset_id VARCHAR(36) NOT NULL,
tag_id VARCHAR(36) NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (dataset_id, tag_id)
);
COMMENT ON TABLE t_dm_dataset_tags IS '数据集标签关联表（UUID 外键）';
COMMENT ON COLUMN t_dm_dataset_tags.dataset_id IS '数据集ID（UUID）';
COMMENT ON COLUMN t_dm_dataset_tags.tag_id IS '标签ID（UUID）';
COMMENT ON COLUMN t_dm_dataset_tags.created_at IS '创建时间';
-- 外键约束
ALTER TABLE t_dm_dataset_tags
ADD CONSTRAINT fk_dm_dataset_tags_dataset
FOREIGN KEY (dataset_id)
REFERENCES t_dm_datasets(id)
ON DELETE CASCADE;
ALTER TABLE t_dm_dataset_tags
ADD CONSTRAINT fk_dm_dataset_tags_tag
FOREIGN KEY (tag_id)
REFERENCES t_dm_tags(id)
ON DELETE CASCADE;
-- ===========================================
-- 非数据管理表
-- ===========================================
-- 用户表
CREATE TABLE IF NOT EXISTS users (
id BIGSERIAL PRIMARY KEY,
username VARCHAR(255) NOT NULL UNIQUE,
email VARCHAR(255) NOT NULL UNIQUE,
password_hash VARCHAR(255) NOT NULL,
full_name VARCHAR(255),
avatar_url VARCHAR(500),
role VARCHAR(50) NOT NULL DEFAULT 'USER',
organization VARCHAR(255),
user_groups JSON,
enabled BOOLEAN NOT NULL DEFAULT TRUE,
last_login_at TIMESTAMP,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
COMMENT ON TABLE users IS '用户表';
COMMENT ON COLUMN users.id IS '主键ID';
COMMENT ON COLUMN users.username IS '用户名';
COMMENT ON COLUMN users.email IS '邮箱';
COMMENT ON COLUMN users.password_hash IS '密码哈希';
COMMENT ON COLUMN users.full_name IS '真实姓名';
COMMENT ON COLUMN users.avatar_url IS '头像URL';
COMMENT ON COLUMN users.role IS '角色：ADMIN/USER（M2 起组驱动用户存解析后的主角色键 admin/dev/annotator/viewer）';
COMMENT ON COLUMN users.organization IS '所属机构';
COMMENT ON COLUMN users.user_groups IS 'Casdoor 组列表(JSON)，如 ["deepdata-admin","deepdata-dev"]；组驱动角色解析源(M2)';
COMMENT ON COLUMN users.enabled IS '是否启用';
COMMENT ON COLUMN users.last_login_at IS '最后登录时间';
COMMENT ON COLUMN users.created_at IS '创建时间';
COMMENT ON COLUMN users.updated_at IS '更新时间';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_users_username ON users(username);
CREATE INDEX IF NOT EXISTS idx_users_email ON users(email);
CREATE INDEX IF NOT EXISTS idx_users_role ON users(role);
CREATE INDEX IF NOT EXISTS idx_users_enabled ON users(enabled);
-- 创建触发器
DROP TRIGGER IF EXISTS update_users_updated_at ON users;
CREATE TRIGGER update_users_updated_at
BEFORE UPDATE ON users
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 插入初始数据
INSERT INTO users (username, email, password_hash, full_name, role, organization)
VALUES
('admin', 'admin@deepdataengine.com', '$2a$10$/esawo436yxg2eodpl/JJ.3Xu6y9m91/ihXRHie9al.LUoNQR5fF.', '系统管理员', 'ADMIN', 'DeepData')
ON CONFLICT (username) DO NOTHING;
-- 创建视图：数据集统计摘要
CREATE OR REPLACE VIEW v_dm_dataset_summary AS
SELECT
COUNT(*) as total_datasets,
SUM(CASE WHEN status = 'ACTIVE' THEN 1 ELSE 0 END) as active_datasets,
SUM(CASE WHEN is_public = TRUE THEN 1 ELSE 0 END) as public_datasets,
SUM(CASE WHEN is_featured = TRUE THEN 1 ELSE 0 END) as featured_datasets,
COALESCE(SUM(file_count), 0) as total_files,
COALESCE(SUM(record_count), 0) as total_records,
COUNT(DISTINCT dataset_type) as dataset_types,
COUNT(DISTINCT category) as categories
FROM t_dm_datasets;
-- 使用现有的DeepData数据库
\c deepdata;
-- 数据集表
CREATE TABLE IF NOT EXISTS t_dm_datasets (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
description TEXT,
dataset_type VARCHAR(50) NOT NULL,
category VARCHAR(100),
path VARCHAR(500),
format VARCHAR(50),
schema_info JSONB,
size_bytes BIGINT DEFAULT 0,
file_count BIGINT DEFAULT 0,
record_count BIGINT DEFAULT 0,
retention_days INTEGER DEFAULT 0,
tags JSONB,
metadata JSONB,
status VARCHAR(50) DEFAULT 'DRAFT',
is_public BOOLEAN DEFAULT FALSE,
is_featured BOOLEAN DEFAULT FALSE,
version BIGINT NOT NULL DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
COMMENT ON TABLE t_dm_datasets IS '数据集表（UUID 主键）';
COMMENT ON COLUMN t_dm_datasets.id IS 'UUID';
COMMENT ON COLUMN t_dm_datasets.name IS '数据集名称';
COMMENT ON COLUMN t_dm_datasets.description IS '数据集描述';
COMMENT ON COLUMN t_dm_datasets.dataset_type IS '数据集类型：IMAGE/TEXT/QA/MULTIMODAL/OTHER';
COMMENT ON COLUMN t_dm_datasets.category IS '数据集分类：医学影像/问答/文献等';
COMMENT ON COLUMN t_dm_datasets.path IS '数据存储路径';
COMMENT ON COLUMN t_dm_datasets.format IS '数据格式：DCM/JPG/JSON/CSV等';
COMMENT ON COLUMN t_dm_datasets.schema_info IS '数据结构信息';
COMMENT ON COLUMN t_dm_datasets.size_bytes IS '数据大小(字节)';
COMMENT ON COLUMN t_dm_datasets.file_count IS '文件数量';
COMMENT ON COLUMN t_dm_datasets.record_count IS '记录数量';
COMMENT ON COLUMN t_dm_datasets.retention_days IS '数据保留天数（0表示长期保留）';
COMMENT ON COLUMN t_dm_datasets.tags IS '标签列表';
COMMENT ON COLUMN t_dm_datasets.metadata IS '元数据信息';
COMMENT ON COLUMN t_dm_datasets.status IS '状态：DRAFT/ACTIVE/ARCHIVED';
COMMENT ON COLUMN t_dm_datasets.is_public IS '是否公开';
COMMENT ON COLUMN t_dm_datasets.is_featured IS '是否推荐';
COMMENT ON COLUMN t_dm_datasets.version IS '版本号';
COMMENT ON COLUMN t_dm_datasets.created_at IS '创建时间';
COMMENT ON COLUMN t_dm_datasets.updated_at IS '更新时间';
COMMENT ON COLUMN t_dm_datasets.created_by IS '创建者';
COMMENT ON COLUMN t_dm_datasets.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dm_dataset_type ON t_dm_datasets(dataset_type);
CREATE INDEX IF NOT EXISTS idx_dm_category ON t_dm_datasets(category);
CREATE INDEX IF NOT EXISTS idx_dm_format ON t_dm_datasets(format);
CREATE INDEX IF NOT EXISTS idx_dm_status ON t_dm_datasets(status);
CREATE INDEX IF NOT EXISTS idx_dm_public ON t_dm_datasets(is_public);
CREATE INDEX IF NOT EXISTS idx_dm_featured ON t_dm_datasets(is_featured);
CREATE INDEX IF NOT EXISTS idx_dm_created_at ON t_dm_datasets(created_at);
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
DROP TRIGGER IF EXISTS update_t_dm_datasets_updated_at ON t_dm_datasets;
CREATE TRIGGER update_t_dm_datasets_updated_at
BEFORE UPDATE ON t_dm_datasets
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 数据集文件表
CREATE TABLE IF NOT EXISTS t_dm_dataset_files (
id VARCHAR(36) PRIMARY KEY,
dataset_id VARCHAR(36) NOT NULL,
file_name VARCHAR(255) NOT NULL,
file_path VARCHAR(1000) NOT NULL,
file_type VARCHAR(50),
file_size BIGINT DEFAULT 0,
check_sum VARCHAR(64),
tags JSONB,
tags_updated_at TIMESTAMP,
annotation JSONB,
metadata JSONB,
status VARCHAR(50) DEFAULT 'ACTIVE',
upload_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
last_access_time TIMESTAMP,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
COMMENT ON TABLE t_dm_dataset_files IS '数据集文件表（UUID 主键）';
COMMENT ON COLUMN t_dm_dataset_files.id IS 'UUID';
COMMENT ON COLUMN t_dm_dataset_files.dataset_id IS '所属数据集ID（UUID）';
COMMENT ON COLUMN t_dm_dataset_files.file_name IS '文件名';
COMMENT ON COLUMN t_dm_dataset_files.file_path IS '文件路径';
COMMENT ON COLUMN t_dm_dataset_files.file_type IS '文件格式：JPG/PNG/DCM/TXT等';
COMMENT ON COLUMN t_dm_dataset_files.file_size IS '文件大小(字节)';
COMMENT ON COLUMN t_dm_dataset_files.check_sum IS '文件校验和';
COMMENT ON COLUMN t_dm_dataset_files.tags IS '文件标签信息（结构化标签/标注概要）';
COMMENT ON COLUMN t_dm_dataset_files.tags_updated_at IS '标签最后更新时间';
COMMENT ON COLUMN t_dm_dataset_files.annotation IS '完整标注结果（原始JSON）';
COMMENT ON COLUMN t_dm_dataset_files.metadata IS '文件元数据';
COMMENT ON COLUMN t_dm_dataset_files.status IS '文件状态：ACTIVE/DELETED/PROCESSING';
COMMENT ON COLUMN t_dm_dataset_files.upload_time IS '上传时间';
COMMENT ON COLUMN t_dm_dataset_files.last_access_time IS '最后访问时间';
COMMENT ON COLUMN t_dm_dataset_files.created_at IS '创建时间';
COMMENT ON COLUMN t_dm_dataset_files.updated_at IS '更新时间';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dm_dataset ON t_dm_dataset_files(dataset_id);
CREATE INDEX IF NOT EXISTS idx_dm_file_type ON t_dm_dataset_files(file_type);
CREATE INDEX IF NOT EXISTS idx_dm_file_status ON t_dm_dataset_files(status);
CREATE INDEX IF NOT EXISTS idx_dm_upload_time ON t_dm_dataset_files(upload_time);
-- 外键约束
ALTER TABLE t_dm_dataset_files
ADD CONSTRAINT fk_dm_dataset_files_dataset
FOREIGN KEY (dataset_id)
REFERENCES t_dm_datasets(id)
ON DELETE CASCADE;
-- 创建触发器
DROP TRIGGER IF EXISTS update_t_dm_dataset_files_updated_at ON t_dm_dataset_files;
CREATE TRIGGER update_t_dm_dataset_files_updated_at
BEFORE UPDATE ON t_dm_dataset_files
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 数据集统计信息表
CREATE TABLE IF NOT EXISTS t_dm_dataset_statistics (
id VARCHAR(36) PRIMARY KEY,
dataset_id VARCHAR(36) NOT NULL,
stat_date DATE NOT NULL,
total_files BIGINT DEFAULT 0,
total_size BIGINT DEFAULT 0,
processed_files BIGINT DEFAULT 0,
error_files BIGINT DEFAULT 0,
download_count BIGINT DEFAULT 0,
view_count BIGINT DEFAULT 0,
quality_metrics JSONB,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
COMMENT ON TABLE t_dm_dataset_statistics IS '数据集统计信息表（UUID 主键）';
COMMENT ON COLUMN t_dm_dataset_statistics.id IS 'UUID';
COMMENT ON COLUMN t_dm_dataset_statistics.dataset_id IS '数据集ID（UUID）';
COMMENT ON COLUMN t_dm_dataset_statistics.stat_date IS '统计日期';
COMMENT ON COLUMN t_dm_dataset_statistics.total_files IS '总文件数';
COMMENT ON COLUMN t_dm_dataset_statistics.total_size IS '总大小(字节)';
COMMENT ON COLUMN t_dm_dataset_statistics.processed_files IS '已处理文件数';
COMMENT ON COLUMN t_dm_dataset_statistics.error_files IS '错误文件数';
COMMENT ON COLUMN t_dm_dataset_statistics.download_count IS '下载次数';
COMMENT ON COLUMN t_dm_dataset_statistics.view_count IS '查看次数';
COMMENT ON COLUMN t_dm_dataset_statistics.quality_metrics IS '质量指标';
COMMENT ON COLUMN t_dm_dataset_statistics.created_at IS '创建时间';
COMMENT ON COLUMN t_dm_dataset_statistics.updated_at IS '更新时间';
-- 创建唯一约束和索引
ALTER TABLE t_dm_dataset_statistics
ADD CONSTRAINT uk_dm_dataset_date
UNIQUE (dataset_id, stat_date);
CREATE INDEX IF NOT EXISTS idx_dm_stat_date ON t_dm_dataset_statistics(stat_date);
-- 外键约束
ALTER TABLE t_dm_dataset_statistics
ADD CONSTRAINT fk_dm_dataset_statistics_dataset
FOREIGN KEY (dataset_id)
REFERENCES t_dm_datasets(id)
ON DELETE CASCADE;
-- 创建触发器
DROP TRIGGER IF EXISTS update_t_dm_dataset_statistics_updated_at ON t_dm_dataset_statistics;
CREATE TRIGGER update_t_dm_dataset_statistics_updated_at
BEFORE UPDATE ON t_dm_dataset_statistics
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 标签表
CREATE TABLE IF NOT EXISTS t_dm_tags (
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(100) NOT NULL UNIQUE,
description TEXT,
category VARCHAR(50),
color VARCHAR(7),
usage_count BIGINT DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
COMMENT ON TABLE t_dm_tags IS '标签表（UUID 主键）';
COMMENT ON COLUMN t_dm_tags.id IS 'UUID';
COMMENT ON COLUMN t_dm_tags.name IS '标签名称';
COMMENT ON COLUMN t_dm_tags.description IS '标签描述';
COMMENT ON COLUMN t_dm_tags.category IS '标签分类';
COMMENT ON COLUMN t_dm_tags.color IS '标签颜色(十六进制)';
COMMENT ON COLUMN t_dm_tags.usage_count IS '使用次数';
COMMENT ON COLUMN t_dm_tags.created_at IS '创建时间';
COMMENT ON COLUMN t_dm_tags.updated_at IS '更新时间';
COMMENT ON COLUMN t_dm_tags.created_by IS '创建者';
COMMENT ON COLUMN t_dm_tags.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dm_tag_category ON t_dm_tags(category);
CREATE INDEX IF NOT EXISTS idx_dm_tag_usage_count ON t_dm_tags(usage_count);
-- 创建触发器
DROP TRIGGER IF EXISTS update_t_dm_tags_updated_at ON t_dm_tags;
CREATE TRIGGER update_t_dm_tags_updated_at
BEFORE UPDATE ON t_dm_tags
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 数据集标签关联表
CREATE TABLE IF NOT EXISTS t_dm_dataset_tags (
dataset_id VARCHAR(36) NOT NULL,
tag_id VARCHAR(36) NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (dataset_id, tag_id)
);
COMMENT ON TABLE t_dm_dataset_tags IS '数据集标签关联表（UUID 外键）';
COMMENT ON COLUMN t_dm_dataset_tags.dataset_id IS '数据集ID（UUID）';
COMMENT ON COLUMN t_dm_dataset_tags.tag_id IS '标签ID（UUID）';
COMMENT ON COLUMN t_dm_dataset_tags.created_at IS '创建时间';
-- 外键约束
ALTER TABLE t_dm_dataset_tags
ADD CONSTRAINT fk_dm_dataset_tags_dataset
FOREIGN KEY (dataset_id)
REFERENCES t_dm_datasets(id)
ON DELETE CASCADE;
ALTER TABLE t_dm_dataset_tags
ADD CONSTRAINT fk_dm_dataset_tags_tag
FOREIGN KEY (tag_id)
REFERENCES t_dm_tags(id)
ON DELETE CASCADE;
-- ===========================================
-- 非数据管理表
-- ===========================================
-- 用户表
CREATE TABLE IF NOT EXISTS users (
id BIGSERIAL PRIMARY KEY,
username VARCHAR(255) NOT NULL UNIQUE,
email VARCHAR(255) NOT NULL UNIQUE,
password_hash VARCHAR(255) NOT NULL,
full_name VARCHAR(255),
avatar_url VARCHAR(500),
role VARCHAR(50) NOT NULL DEFAULT 'USER',
organization VARCHAR(255),
enabled BOOLEAN NOT NULL DEFAULT TRUE,
last_login_at TIMESTAMP,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
COMMENT ON TABLE users IS '用户表';
COMMENT ON COLUMN users.id IS '主键ID';
COMMENT ON COLUMN users.username IS '用户名';
COMMENT ON COLUMN users.email IS '邮箱';
COMMENT ON COLUMN users.password_hash IS '密码哈希';
COMMENT ON COLUMN users.full_name IS '真实姓名';
COMMENT ON COLUMN users.avatar_url IS '头像URL';
COMMENT ON COLUMN users.role IS '角色：ADMIN/USER';
COMMENT ON COLUMN users.organization IS '所属机构';
COMMENT ON COLUMN users.enabled IS '是否启用';
COMMENT ON COLUMN users.last_login_at IS '最后登录时间';
COMMENT ON COLUMN users.created_at IS '创建时间';
COMMENT ON COLUMN users.updated_at IS '更新时间';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_users_username ON users(username);
CREATE INDEX IF NOT EXISTS idx_users_email ON users(email);
CREATE INDEX IF NOT EXISTS idx_users_role ON users(role);
CREATE INDEX IF NOT EXISTS idx_users_enabled ON users(enabled);
-- 创建触发器
DROP TRIGGER IF EXISTS update_users_updated_at ON users;
CREATE TRIGGER update_users_updated_at
BEFORE UPDATE ON users
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 插入初始数据
INSERT INTO users (username, email, password_hash, full_name, role, organization)
VALUES
('admin', 'admin@deepdataengine.com', '$2b$10$C3CtLQBPzJGkXWikSndUbexnK2same3qMOdqzdPTgJeIC.7j7uNSW', '系统管理员', 'ADMIN', 'DeepData')
ON CONFLICT (username) DO NOTHING;
INSERT INTO users (username, email, password_hash, full_name, role, organization)
VALUES
('test', 'test@deepdataengine.com', '$2b$10$cWwGt80ZcTM7y7nH4rQfQeEc9NxlfRjfj2s.C5xB7WAMmTySvyac.', '系统管理员', 'ADMIN', 'DeepData')
ON CONFLICT (username) DO NOTHING;
-- 创建视图：数据集统计摘要
CREATE OR REPLACE VIEW v_dm_dataset_summary AS
SELECT
COUNT(*) as total_datasets,
SUM(CASE WHEN status = 'ACTIVE' THEN 1 ELSE 0 END) as active_datasets,
SUM(CASE WHEN is_public = TRUE THEN 1 ELSE 0 END) as public_datasets,
SUM(CASE WHEN is_featured = TRUE THEN 1 ELSE 0 END) as featured_datasets,
COALESCE(SUM(file_count), 0) as total_files,
COALESCE(SUM(record_count), 0) as total_records,
COUNT(DISTINCT dataset_type) as dataset_types,
COUNT(DISTINCT category) as categories
FROM t_dm_datasets;
scripts/db/data-operator-data-juicer-init.sql.txt
+1174
View File
File diff suppressed because it is too large Load Diff
scripts/db/data-operator-init.sql
+530 -526
View File
File diff suppressed because one or more lines are too long
scripts/db/data-ratio-init.sql
+97 -97
View File
@@ -1,97 +1,97 @@
-- 使用现有的deepdata数据库
\c deepdata;
-- 配比实例表
CREATE TABLE IF NOT EXISTS t_st_ratio_instances
(
id VARCHAR(64) PRIMARY KEY,
name VARCHAR(64),
description TEXT,
target_dataset_id VARCHAR(64),
ratio_parameters JSONB,
merge_method VARCHAR(50),
status VARCHAR(20),
totals BIGINT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_st_ratio_instances IS '配比实例表（UUID 主键）';
COMMENT ON COLUMN t_st_ratio_instances.id IS 'UUID';
COMMENT ON COLUMN t_st_ratio_instances.name IS '名称';
COMMENT ON COLUMN t_st_ratio_instances.description IS '描述';
COMMENT ON COLUMN t_st_ratio_instances.target_dataset_id IS '模板数据集ID';
COMMENT ON COLUMN t_st_ratio_instances.ratio_parameters IS '配比参数';
COMMENT ON COLUMN t_st_ratio_instances.merge_method IS '合并方式';
COMMENT ON COLUMN t_st_ratio_instances.status IS '状态';
COMMENT ON COLUMN t_st_ratio_instances.totals IS '总数';
COMMENT ON COLUMN t_st_ratio_instances.created_at IS '创建时间';
COMMENT ON COLUMN t_st_ratio_instances.updated_at IS '更新时间';
COMMENT ON COLUMN t_st_ratio_instances.created_by IS '创建者';
COMMENT ON COLUMN t_st_ratio_instances.updated_by IS '更新者';
-- 配比关系表
CREATE TABLE IF NOT EXISTS t_st_ratio_relations
(
id VARCHAR(64) PRIMARY KEY,
ratio_instance_id VARCHAR(64),
source_dataset_id VARCHAR(64),
ratio_value VARCHAR(256),
counts BIGINT,
filter_conditions TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_st_ratio_relations IS '配比关系表（UUID 主键）';
COMMENT ON COLUMN t_st_ratio_relations.id IS 'UUID';
COMMENT ON COLUMN t_st_ratio_relations.ratio_instance_id IS '配比实例ID';
COMMENT ON COLUMN t_st_ratio_relations.source_dataset_id IS '源数据集ID';
COMMENT ON COLUMN t_st_ratio_relations.ratio_value IS '配比值';
COMMENT ON COLUMN t_st_ratio_relations.counts IS '条数';
COMMENT ON COLUMN t_st_ratio_relations.filter_conditions IS '过滤条件';
COMMENT ON COLUMN t_st_ratio_relations.created_at IS '创建时间';
COMMENT ON COLUMN t_st_ratio_relations.updated_at IS '更新时间';
COMMENT ON COLUMN t_st_ratio_relations.created_by IS '创建者';
COMMENT ON COLUMN t_st_ratio_relations.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_st_ratio_instances_target_dataset ON t_st_ratio_instances(target_dataset_id);
CREATE INDEX IF NOT EXISTS idx_st_ratio_instances_status ON t_st_ratio_instances(status);
CREATE INDEX IF NOT EXISTS idx_st_ratio_instances_created_at ON t_st_ratio_instances(created_at);
CREATE INDEX IF NOT EXISTS idx_st_ratio_relations_instance ON t_st_ratio_relations(ratio_instance_id);
CREATE INDEX IF NOT EXISTS idx_st_ratio_relations_source_dataset ON t_st_ratio_relations(source_dataset_id);
CREATE INDEX IF NOT EXISTS idx_st_ratio_relations_created_at ON t_st_ratio_relations(created_at);
-- 创建外键约束
ALTER TABLE t_st_ratio_relations ADD CONSTRAINT fk_st_ratio_relations_instance FOREIGN KEY (ratio_instance_id)
REFERENCES t_st_ratio_instances(id) ON DELETE CASCADE;
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- 为各个表创建触发器
DROP TRIGGER IF EXISTS update_st_ratio_instances_updated_at ON t_st_ratio_instances;
CREATE TRIGGER update_st_ratio_instances_updated_at
BEFORE UPDATE ON t_st_ratio_instances
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_st_ratio_relations_updated_at ON t_st_ratio_relations;
CREATE TRIGGER update_st_ratio_relations_updated_at
BEFORE UPDATE ON t_st_ratio_relations
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 使用现有的DeepData数据库
\c deepdata;
-- 配比实例表
CREATE TABLE IF NOT EXISTS t_st_ratio_instances
(
id VARCHAR(64) PRIMARY KEY,
name VARCHAR(64),
description TEXT,
target_dataset_id VARCHAR(64),
ratio_parameters JSONB,
merge_method VARCHAR(50),
status VARCHAR(20),
totals BIGINT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_st_ratio_instances IS '配比实例表（UUID 主键）';
COMMENT ON COLUMN t_st_ratio_instances.id IS 'UUID';
COMMENT ON COLUMN t_st_ratio_instances.name IS '名称';
COMMENT ON COLUMN t_st_ratio_instances.description IS '描述';
COMMENT ON COLUMN t_st_ratio_instances.target_dataset_id IS '模板数据集ID';
COMMENT ON COLUMN t_st_ratio_instances.ratio_parameters IS '配比参数';
COMMENT ON COLUMN t_st_ratio_instances.merge_method IS '合并方式';
COMMENT ON COLUMN t_st_ratio_instances.status IS '状态';
COMMENT ON COLUMN t_st_ratio_instances.totals IS '总数';
COMMENT ON COLUMN t_st_ratio_instances.created_at IS '创建时间';
COMMENT ON COLUMN t_st_ratio_instances.updated_at IS '更新时间';
COMMENT ON COLUMN t_st_ratio_instances.created_by IS '创建者';
COMMENT ON COLUMN t_st_ratio_instances.updated_by IS '更新者';
-- 配比关系表
CREATE TABLE IF NOT EXISTS t_st_ratio_relations
(
id VARCHAR(64) PRIMARY KEY,
ratio_instance_id VARCHAR(64),
source_dataset_id VARCHAR(64),
ratio_value VARCHAR(256),
counts BIGINT,
filter_conditions TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_st_ratio_relations IS '配比关系表（UUID 主键）';
COMMENT ON COLUMN t_st_ratio_relations.id IS 'UUID';
COMMENT ON COLUMN t_st_ratio_relations.ratio_instance_id IS '配比实例ID';
COMMENT ON COLUMN t_st_ratio_relations.source_dataset_id IS '源数据集ID';
COMMENT ON COLUMN t_st_ratio_relations.ratio_value IS '配比值';
COMMENT ON COLUMN t_st_ratio_relations.counts IS '条数';
COMMENT ON COLUMN t_st_ratio_relations.filter_conditions IS '过滤条件';
COMMENT ON COLUMN t_st_ratio_relations.created_at IS '创建时间';
COMMENT ON COLUMN t_st_ratio_relations.updated_at IS '更新时间';
COMMENT ON COLUMN t_st_ratio_relations.created_by IS '创建者';
COMMENT ON COLUMN t_st_ratio_relations.updated_by IS '更新者';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_st_ratio_instances_target_dataset ON t_st_ratio_instances(target_dataset_id);
CREATE INDEX IF NOT EXISTS idx_st_ratio_instances_status ON t_st_ratio_instances(status);
CREATE INDEX IF NOT EXISTS idx_st_ratio_instances_created_at ON t_st_ratio_instances(created_at);
CREATE INDEX IF NOT EXISTS idx_st_ratio_relations_instance ON t_st_ratio_relations(ratio_instance_id);
CREATE INDEX IF NOT EXISTS idx_st_ratio_relations_source_dataset ON t_st_ratio_relations(source_dataset_id);
CREATE INDEX IF NOT EXISTS idx_st_ratio_relations_created_at ON t_st_ratio_relations(created_at);
-- 创建外键约束
ALTER TABLE t_st_ratio_relations ADD CONSTRAINT fk_st_ratio_relations_instance FOREIGN KEY (ratio_instance_id)
REFERENCES t_st_ratio_instances(id) ON DELETE CASCADE;
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- 为各个表创建触发器
DROP TRIGGER IF EXISTS update_st_ratio_instances_updated_at ON t_st_ratio_instances;
CREATE TRIGGER update_st_ratio_instances_updated_at
BEFORE UPDATE ON t_st_ratio_instances
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_st_ratio_relations_updated_at ON t_st_ratio_relations;
CREATE TRIGGER update_st_ratio_relations_updated_at
BEFORE UPDATE ON t_st_ratio_relations
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
scripts/db/data-synthesis-init.sql
+158 -158
View File
@@ -1,158 +1,158 @@
-- 使用现有的deepdata数据库
\c deepdata;
-- 数据合成任务表
CREATE TABLE IF NOT EXISTS t_data_synth_instances
(
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
description TEXT,
status VARCHAR(20),
synth_type VARCHAR(20) NOT NULL,
progress INTEGER DEFAULT 0,
synth_config JSONB NOT NULL,
total_files INTEGER DEFAULT 0,
processed_files INTEGER DEFAULT 0,
total_chunks INTEGER DEFAULT 0,
processed_chunks INTEGER DEFAULT 0,
total_synth_data INTEGER DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_data_synth_instances IS '数据合成任务表（UUID 主键）';
COMMENT ON COLUMN t_data_synth_instances.id IS 'UUID';
COMMENT ON COLUMN t_data_synth_instances.name IS '任务名称';
COMMENT ON COLUMN t_data_synth_instances.description IS '任务描述';
COMMENT ON COLUMN t_data_synth_instances.status IS '任务状态';
COMMENT ON COLUMN t_data_synth_instances.synth_type IS '合成类型';
COMMENT ON COLUMN t_data_synth_instances.progress IS '任务进度(百分比)';
COMMENT ON COLUMN t_data_synth_instances.synth_config IS '合成配置';
COMMENT ON COLUMN t_data_synth_instances.total_files IS '总文件数';
COMMENT ON COLUMN t_data_synth_instances.processed_files IS '已处理文件数';
COMMENT ON COLUMN t_data_synth_instances.total_chunks IS '总文本块数';
COMMENT ON COLUMN t_data_synth_instances.processed_chunks IS '已处理文本块数';
COMMENT ON COLUMN t_data_synth_instances.total_synth_data IS '总合成数据量';
COMMENT ON COLUMN t_data_synth_instances.created_at IS '创建时间';
COMMENT ON COLUMN t_data_synth_instances.updated_at IS '更新时间';
COMMENT ON COLUMN t_data_synth_instances.created_by IS '创建者';
COMMENT ON COLUMN t_data_synth_instances.updated_by IS '更新者';
-- 数据合成文件任务表
CREATE TABLE IF NOT EXISTS t_data_synthesis_file_instances
(
id VARCHAR(36) PRIMARY KEY,
synthesis_instance_id VARCHAR(36),
file_name VARCHAR(255) NOT NULL,
source_file_id VARCHAR(255) NOT NULL,
target_file_location VARCHAR(1000),
status VARCHAR(20),
total_chunks INTEGER DEFAULT 0,
processed_chunks INTEGER DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_data_synthesis_file_instances IS '数据合成文件任务表（UUID 主键）';
COMMENT ON COLUMN t_data_synthesis_file_instances.id IS 'UUID';
COMMENT ON COLUMN t_data_synthesis_file_instances.synthesis_instance_id IS '数据合成任务ID';
COMMENT ON COLUMN t_data_synthesis_file_instances.file_name IS '文件名';
COMMENT ON COLUMN t_data_synthesis_file_instances.source_file_id IS '原始文件ID';
COMMENT ON COLUMN t_data_synthesis_file_instances.target_file_location IS '目标文件存储位置';
COMMENT ON COLUMN t_data_synthesis_file_instances.status IS '任务状态';
COMMENT ON COLUMN t_data_synthesis_file_instances.total_chunks IS '总文本块数';
COMMENT ON COLUMN t_data_synthesis_file_instances.processed_chunks IS '已处理文本块数';
COMMENT ON COLUMN t_data_synthesis_file_instances.created_at IS '创建时间';
COMMENT ON COLUMN t_data_synthesis_file_instances.updated_at IS '更新时间';
COMMENT ON COLUMN t_data_synthesis_file_instances.created_by IS '创建者';
COMMENT ON COLUMN t_data_synthesis_file_instances.updated_by IS '更新者';
-- 数据合成分块任务表
CREATE TABLE IF NOT EXISTS t_data_synthesis_chunk_instances
(
id VARCHAR(36) PRIMARY KEY,
synthesis_file_instance_id VARCHAR(36),
chunk_index INTEGER,
chunk_content TEXT,
metadata JSONB
);
-- 添加注释
COMMENT ON TABLE t_data_synthesis_chunk_instances IS '数据合成分块任务表（UUID 主键）';
COMMENT ON COLUMN t_data_synthesis_chunk_instances.id IS 'UUID';
COMMENT ON COLUMN t_data_synthesis_chunk_instances.synthesis_file_instance_id IS '数据合成文件任务ID';
COMMENT ON COLUMN t_data_synthesis_chunk_instances.chunk_index IS '分块索引';
COMMENT ON COLUMN t_data_synthesis_chunk_instances.chunk_content IS '分块内容';
COMMENT ON COLUMN t_data_synthesis_chunk_instances.metadata IS '分块元数据';
-- 数据合成任务队列表
CREATE TABLE IF NOT EXISTS t_data_synthesis_data
(
id VARCHAR(36) PRIMARY KEY,
data JSONB,
synthesis_file_instance_id VARCHAR(36),
chunk_instance_id VARCHAR(36)
);
-- 添加注释
COMMENT ON TABLE t_data_synthesis_data IS '数据合成任务队列表（UUID 主键）';
COMMENT ON COLUMN t_data_synthesis_data.id IS 'UUID';
COMMENT ON COLUMN t_data_synthesis_data.data IS '合成的数据';
COMMENT ON COLUMN t_data_synthesis_data.synthesis_file_instance_id IS '数据合成文件任务ID';
COMMENT ON COLUMN t_data_synthesis_data.chunk_instance_id IS '分块任务ID';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_synth_instances_status ON t_data_synth_instances(status);
CREATE INDEX IF NOT EXISTS idx_synth_instances_created_at ON t_data_synth_instances(created_at);
CREATE INDEX IF NOT EXISTS idx_synth_instances_synth_type ON t_data_synth_instances(synth_type);
CREATE INDEX IF NOT EXISTS idx_synth_file_instances_instance ON t_data_synthesis_file_instances(synthesis_instance_id);
CREATE INDEX IF NOT EXISTS idx_synth_file_instances_status ON t_data_synthesis_file_instances(status);
CREATE INDEX IF NOT EXISTS idx_synth_file_instances_source_file ON t_data_synthesis_file_instances(source_file_id);
CREATE INDEX IF NOT EXISTS idx_synth_chunk_instances_file ON t_data_synthesis_chunk_instances(synthesis_file_instance_id);
CREATE INDEX IF NOT EXISTS idx_synth_chunk_instances_index ON t_data_synthesis_chunk_instances(chunk_index);
CREATE INDEX IF NOT EXISTS idx_synth_data_file ON t_data_synthesis_data(synthesis_file_instance_id);
CREATE INDEX IF NOT EXISTS idx_synth_data_chunk ON t_data_synthesis_data(chunk_instance_id);
-- 创建外键约束
ALTER TABLE t_data_synthesis_file_instances ADD CONSTRAINT fk_synth_file_instances_instance
FOREIGN KEY (synthesis_instance_id) REFERENCES t_data_synth_instances(id) ON DELETE CASCADE;
ALTER TABLE t_data_synthesis_chunk_instances ADD CONSTRAINT fk_synth_chunk_instances_file
FOREIGN KEY (synthesis_file_instance_id) REFERENCES t_data_synthesis_file_instances(id) ON DELETE CASCADE;
ALTER TABLE t_data_synthesis_data ADD CONSTRAINT fk_synth_data_file FOREIGN KEY (synthesis_file_instance_id)
REFERENCES t_data_synthesis_file_instances(id) ON DELETE CASCADE;
ALTER TABLE t_data_synthesis_data ADD CONSTRAINT fk_synth_data_chunk FOREIGN KEY (chunk_instance_id)
REFERENCES t_data_synthesis_chunk_instances(id) ON DELETE CASCADE;
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- 为有 updated_at 字段的表创建触发器
DROP TRIGGER IF EXISTS update_synth_instances_updated_at ON t_data_synth_instances;
CREATE TRIGGER update_synth_instances_updated_at
BEFORE UPDATE ON t_data_synth_instances
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_synth_file_instances_updated_at ON t_data_synthesis_file_instances;
CREATE TRIGGER update_synth_file_instances_updated_at
BEFORE UPDATE ON t_data_synthesis_file_instances
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 使用现有的DeepData数据库
\c deepdata;
-- 数据合成任务表
CREATE TABLE IF NOT EXISTS t_data_synth_instances
(
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
description TEXT,
status VARCHAR(20),
synth_type VARCHAR(20) NOT NULL,
progress INTEGER DEFAULT 0,
synth_config JSONB NOT NULL,
total_files INTEGER DEFAULT 0,
processed_files INTEGER DEFAULT 0,
total_chunks INTEGER DEFAULT 0,
processed_chunks INTEGER DEFAULT 0,
total_synth_data INTEGER DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_data_synth_instances IS '数据合成任务表（UUID 主键）';
COMMENT ON COLUMN t_data_synth_instances.id IS 'UUID';
COMMENT ON COLUMN t_data_synth_instances.name IS '任务名称';
COMMENT ON COLUMN t_data_synth_instances.description IS '任务描述';
COMMENT ON COLUMN t_data_synth_instances.status IS '任务状态';
COMMENT ON COLUMN t_data_synth_instances.synth_type IS '合成类型';
COMMENT ON COLUMN t_data_synth_instances.progress IS '任务进度(百分比)';
COMMENT ON COLUMN t_data_synth_instances.synth_config IS '合成配置';
COMMENT ON COLUMN t_data_synth_instances.total_files IS '总文件数';
COMMENT ON COLUMN t_data_synth_instances.processed_files IS '已处理文件数';
COMMENT ON COLUMN t_data_synth_instances.total_chunks IS '总文本块数';
COMMENT ON COLUMN t_data_synth_instances.processed_chunks IS '已处理文本块数';
COMMENT ON COLUMN t_data_synth_instances.total_synth_data IS '总合成数据量';
COMMENT ON COLUMN t_data_synth_instances.created_at IS '创建时间';
COMMENT ON COLUMN t_data_synth_instances.updated_at IS '更新时间';
COMMENT ON COLUMN t_data_synth_instances.created_by IS '创建者';
COMMENT ON COLUMN t_data_synth_instances.updated_by IS '更新者';
-- 数据合成文件任务表
CREATE TABLE IF NOT EXISTS t_data_synthesis_file_instances
(
id VARCHAR(36) PRIMARY KEY,
synthesis_instance_id VARCHAR(36),
file_name VARCHAR(255) NOT NULL,
source_file_id VARCHAR(255) NOT NULL,
target_file_location VARCHAR(1000),
status VARCHAR(20),
total_chunks INTEGER DEFAULT 0,
processed_chunks INTEGER DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_data_synthesis_file_instances IS '数据合成文件任务表（UUID 主键）';
COMMENT ON COLUMN t_data_synthesis_file_instances.id IS 'UUID';
COMMENT ON COLUMN t_data_synthesis_file_instances.synthesis_instance_id IS '数据合成任务ID';
COMMENT ON COLUMN t_data_synthesis_file_instances.file_name IS '文件名';
COMMENT ON COLUMN t_data_synthesis_file_instances.source_file_id IS '原始文件ID';
COMMENT ON COLUMN t_data_synthesis_file_instances.target_file_location IS '目标文件存储位置';
COMMENT ON COLUMN t_data_synthesis_file_instances.status IS '任务状态';
COMMENT ON COLUMN t_data_synthesis_file_instances.total_chunks IS '总文本块数';
COMMENT ON COLUMN t_data_synthesis_file_instances.processed_chunks IS '已处理文本块数';
COMMENT ON COLUMN t_data_synthesis_file_instances.created_at IS '创建时间';
COMMENT ON COLUMN t_data_synthesis_file_instances.updated_at IS '更新时间';
COMMENT ON COLUMN t_data_synthesis_file_instances.created_by IS '创建者';
COMMENT ON COLUMN t_data_synthesis_file_instances.updated_by IS '更新者';
-- 数据合成分块任务表
CREATE TABLE IF NOT EXISTS t_data_synthesis_chunk_instances
(
id VARCHAR(36) PRIMARY KEY,
synthesis_file_instance_id VARCHAR(36),
chunk_index INTEGER,
chunk_content TEXT,
metadata JSONB
);
-- 添加注释
COMMENT ON TABLE t_data_synthesis_chunk_instances IS '数据合成分块任务表（UUID 主键）';
COMMENT ON COLUMN t_data_synthesis_chunk_instances.id IS 'UUID';
COMMENT ON COLUMN t_data_synthesis_chunk_instances.synthesis_file_instance_id IS '数据合成文件任务ID';
COMMENT ON COLUMN t_data_synthesis_chunk_instances.chunk_index IS '分块索引';
COMMENT ON COLUMN t_data_synthesis_chunk_instances.chunk_content IS '分块内容';
COMMENT ON COLUMN t_data_synthesis_chunk_instances.metadata IS '分块元数据';
-- 数据合成任务队列表
CREATE TABLE IF NOT EXISTS t_data_synthesis_data
(
id VARCHAR(36) PRIMARY KEY,
data JSONB,
synthesis_file_instance_id VARCHAR(36),
chunk_instance_id VARCHAR(36)
);
-- 添加注释
COMMENT ON TABLE t_data_synthesis_data IS '数据合成任务队列表（UUID 主键）';
COMMENT ON COLUMN t_data_synthesis_data.id IS 'UUID';
COMMENT ON COLUMN t_data_synthesis_data.data IS '合成的数据';
COMMENT ON COLUMN t_data_synthesis_data.synthesis_file_instance_id IS '数据合成文件任务ID';
COMMENT ON COLUMN t_data_synthesis_data.chunk_instance_id IS '分块任务ID';
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_synth_instances_status ON t_data_synth_instances(status);
CREATE INDEX IF NOT EXISTS idx_synth_instances_created_at ON t_data_synth_instances(created_at);
CREATE INDEX IF NOT EXISTS idx_synth_instances_synth_type ON t_data_synth_instances(synth_type);
CREATE INDEX IF NOT EXISTS idx_synth_file_instances_instance ON t_data_synthesis_file_instances(synthesis_instance_id);
CREATE INDEX IF NOT EXISTS idx_synth_file_instances_status ON t_data_synthesis_file_instances(status);
CREATE INDEX IF NOT EXISTS idx_synth_file_instances_source_file ON t_data_synthesis_file_instances(source_file_id);
CREATE INDEX IF NOT EXISTS idx_synth_chunk_instances_file ON t_data_synthesis_chunk_instances(synthesis_file_instance_id);
CREATE INDEX IF NOT EXISTS idx_synth_chunk_instances_index ON t_data_synthesis_chunk_instances(chunk_index);
CREATE INDEX IF NOT EXISTS idx_synth_data_file ON t_data_synthesis_data(synthesis_file_instance_id);
CREATE INDEX IF NOT EXISTS idx_synth_data_chunk ON t_data_synthesis_data(chunk_instance_id);
-- 创建外键约束
ALTER TABLE t_data_synthesis_file_instances ADD CONSTRAINT fk_synth_file_instances_instance
FOREIGN KEY (synthesis_instance_id) REFERENCES t_data_synth_instances(id) ON DELETE CASCADE;
ALTER TABLE t_data_synthesis_chunk_instances ADD CONSTRAINT fk_synth_chunk_instances_file
FOREIGN KEY (synthesis_file_instance_id) REFERENCES t_data_synthesis_file_instances(id) ON DELETE CASCADE;
ALTER TABLE t_data_synthesis_data ADD CONSTRAINT fk_synth_data_file FOREIGN KEY (synthesis_file_instance_id)
REFERENCES t_data_synthesis_file_instances(id) ON DELETE CASCADE;
ALTER TABLE t_data_synthesis_data ADD CONSTRAINT fk_synth_data_chunk FOREIGN KEY (chunk_instance_id)
REFERENCES t_data_synthesis_chunk_instances(id) ON DELETE CASCADE;
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- 为有 updated_at 字段的表创建触发器
DROP TRIGGER IF EXISTS update_synth_instances_updated_at ON t_data_synth_instances;
CREATE TRIGGER update_synth_instances_updated_at
BEFORE UPDATE ON t_data_synth_instances
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_synth_file_instances_updated_at ON t_data_synthesis_file_instances;
CREATE TRIGGER update_synth_file_instances_updated_at
BEFORE UPDATE ON t_data_synthesis_file_instances
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
scripts/db/rag-management-init.sql
+99 -99
View File
@@ -1,99 +1,99 @@
-- 使用现有的deepdata数据库
\c deepdata;
-- 知识库表
CREATE TABLE IF NOT EXISTS t_rag_knowledge_base
(
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
type VARCHAR(50) NOT NULL,
description VARCHAR(512),
embedding_model VARCHAR(255) NOT NULL,
chat_model VARCHAR(255),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_rag_knowledge_base IS '知识库表';
COMMENT ON COLUMN t_rag_knowledge_base.id IS 'UUID';
COMMENT ON COLUMN t_rag_knowledge_base.name IS '知识库名称';
COMMENT ON COLUMN t_rag_knowledge_base.type IS '知识库类型';
COMMENT ON COLUMN t_rag_knowledge_base.description IS '知识库描述';
COMMENT ON COLUMN t_rag_knowledge_base.embedding_model IS '嵌入模型';
COMMENT ON COLUMN t_rag_knowledge_base.chat_model IS '聊天模型';
COMMENT ON COLUMN t_rag_knowledge_base.created_at IS '创建时间';
COMMENT ON COLUMN t_rag_knowledge_base.updated_at IS '更新时间';
COMMENT ON COLUMN t_rag_knowledge_base.created_by IS '创建者';
COMMENT ON COLUMN t_rag_knowledge_base.updated_by IS '更新者';
-- 知识库文件表（修正表名注释为"知识库文件表"）
CREATE TABLE IF NOT EXISTS t_rag_file
(
id VARCHAR(36) PRIMARY KEY,
knowledge_base_id VARCHAR(36) NOT NULL,
file_name VARCHAR(255) NOT NULL,
file_id VARCHAR(255) NOT NULL,
chunk_count INTEGER,
metadata JSONB,
status VARCHAR(50),
err_msg TEXT,
progress INTEGER DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_rag_file IS '知识库文件表';
COMMENT ON COLUMN t_rag_file.id IS 'UUID';
COMMENT ON COLUMN t_rag_file.knowledge_base_id IS '知识库ID';
COMMENT ON COLUMN t_rag_file.file_name IS '文件名';
COMMENT ON COLUMN t_rag_file.file_id IS '文件ID';
COMMENT ON COLUMN t_rag_file.chunk_count IS '切片数';
COMMENT ON COLUMN t_rag_file.metadata IS '元数据';
COMMENT ON COLUMN t_rag_file.status IS '文件状态';
COMMENT ON COLUMN t_rag_file.err_msg IS '错误信息';
COMMENT ON COLUMN t_rag_file.progress IS '处理进度(0-100)';
COMMENT ON COLUMN t_rag_file.created_at IS '创建时间';
COMMENT ON COLUMN t_rag_file.updated_at IS '更新时间';
COMMENT ON COLUMN t_rag_file.created_by IS '创建者';
COMMENT ON COLUMN t_rag_file.updated_by IS '更新者';
-- 创建外键约束
ALTER TABLE t_rag_file
ADD CONSTRAINT fk_rag_file_knowledge_base
FOREIGN KEY (knowledge_base_id)
REFERENCES t_rag_knowledge_base(id)
ON DELETE CASCADE;
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_rag_file_knowledge_base ON t_rag_file(knowledge_base_id);
CREATE INDEX IF NOT EXISTS idx_rag_file_status ON t_rag_file(status);
CREATE INDEX IF NOT EXISTS idx_rag_file_created_at ON t_rag_file(created_at);
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- 为 t_rag_knowledge_base 表创建触发器
DROP TRIGGER IF EXISTS update_rag_knowledge_base_updated_at ON t_rag_knowledge_base;
CREATE TRIGGER update_rag_knowledge_base_updated_at
BEFORE UPDATE ON t_rag_knowledge_base
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 为 t_rag_file 表创建触发器
DROP TRIGGER IF EXISTS update_rag_file_updated_at ON t_rag_file;
CREATE TRIGGER update_rag_file_updated_at
BEFORE UPDATE ON t_rag_file
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 使用现有的DeepData数据库
\c deepdata;
-- 知识库表
CREATE TABLE IF NOT EXISTS t_rag_knowledge_base
(
id VARCHAR(36) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
type VARCHAR(50) NOT NULL,
description VARCHAR(512),
embedding_model VARCHAR(255) NOT NULL,
chat_model VARCHAR(255),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_rag_knowledge_base IS '知识库表';
COMMENT ON COLUMN t_rag_knowledge_base.id IS 'UUID';
COMMENT ON COLUMN t_rag_knowledge_base.name IS '知识库名称';
COMMENT ON COLUMN t_rag_knowledge_base.type IS '知识库类型';
COMMENT ON COLUMN t_rag_knowledge_base.description IS '知识库描述';
COMMENT ON COLUMN t_rag_knowledge_base.embedding_model IS '嵌入模型';
COMMENT ON COLUMN t_rag_knowledge_base.chat_model IS '聊天模型';
COMMENT ON COLUMN t_rag_knowledge_base.created_at IS '创建时间';
COMMENT ON COLUMN t_rag_knowledge_base.updated_at IS '更新时间';
COMMENT ON COLUMN t_rag_knowledge_base.created_by IS '创建者';
COMMENT ON COLUMN t_rag_knowledge_base.updated_by IS '更新者';
-- 知识库文件表（修正表名注释为"知识库文件表"）
CREATE TABLE IF NOT EXISTS t_rag_file
(
id VARCHAR(36) PRIMARY KEY,
knowledge_base_id VARCHAR(36) NOT NULL,
file_name VARCHAR(255) NOT NULL,
file_id VARCHAR(255) NOT NULL,
chunk_count INTEGER,
metadata JSONB,
status VARCHAR(50),
err_msg TEXT,
progress INTEGER DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_rag_file IS '知识库文件表';
COMMENT ON COLUMN t_rag_file.id IS 'UUID';
COMMENT ON COLUMN t_rag_file.knowledge_base_id IS '知识库ID';
COMMENT ON COLUMN t_rag_file.file_name IS '文件名';
COMMENT ON COLUMN t_rag_file.file_id IS '文件ID';
COMMENT ON COLUMN t_rag_file.chunk_count IS '切片数';
COMMENT ON COLUMN t_rag_file.metadata IS '元数据';
COMMENT ON COLUMN t_rag_file.status IS '文件状态';
COMMENT ON COLUMN t_rag_file.err_msg IS '错误信息';
COMMENT ON COLUMN t_rag_file.progress IS '处理进度(0-100)';
COMMENT ON COLUMN t_rag_file.created_at IS '创建时间';
COMMENT ON COLUMN t_rag_file.updated_at IS '更新时间';
COMMENT ON COLUMN t_rag_file.created_by IS '创建者';
COMMENT ON COLUMN t_rag_file.updated_by IS '更新者';
-- 创建外键约束
ALTER TABLE t_rag_file
ADD CONSTRAINT fk_rag_file_knowledge_base
FOREIGN KEY (knowledge_base_id)
REFERENCES t_rag_knowledge_base(id)
ON DELETE CASCADE;
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_rag_file_knowledge_base ON t_rag_file(knowledge_base_id);
CREATE INDEX IF NOT EXISTS idx_rag_file_status ON t_rag_file(status);
CREATE INDEX IF NOT EXISTS idx_rag_file_created_at ON t_rag_file(created_at);
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- 为 t_rag_knowledge_base 表创建触发器
DROP TRIGGER IF EXISTS update_rag_knowledge_base_updated_at ON t_rag_knowledge_base;
CREATE TRIGGER update_rag_knowledge_base_updated_at
BEFORE UPDATE ON t_rag_knowledge_base
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 为 t_rag_file 表创建触发器
DROP TRIGGER IF EXISTS update_rag_file_updated_at ON t_rag_file;
CREATE TRIGGER update_rag_file_updated_at
BEFORE UPDATE ON t_rag_file
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
scripts/db/setting-management-init.sql
+111 -111
View File
@@ -1,111 +1,111 @@
-- 使用现有的deepdata数据库
\c deepdata;
-- 模型配置表
CREATE TABLE IF NOT EXISTS t_models
(
id VARCHAR(36) PRIMARY KEY,
model_name VARCHAR(100) NOT NULL,
provider VARCHAR(50) NOT NULL,
base_url VARCHAR(255) NOT NULL,
api_key VARCHAR(512) DEFAULT '',
type VARCHAR(50) NOT NULL,
is_enabled BOOLEAN DEFAULT TRUE,
is_default BOOLEAN DEFAULT FALSE,
is_deleted BOOLEAN DEFAULT FALSE,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_models IS '模型配置表';
COMMENT ON COLUMN t_models.id IS '主键ID';
COMMENT ON COLUMN t_models.model_name IS '模型名称（如 qwen2）';
COMMENT ON COLUMN t_models.provider IS '模型提供商（如 Ollama、OpenAI、DeepSeek）';
COMMENT ON COLUMN t_models.base_url IS 'API 基础地址';
COMMENT ON COLUMN t_models.api_key IS 'API 密钥（无密钥则为空）';
COMMENT ON COLUMN t_models.type IS '模型类型（如 chat、embedding）';
COMMENT ON COLUMN t_models.is_enabled IS '是否启用：1-启用，0-禁用';
COMMENT ON COLUMN t_models.is_default IS '是否默认：1-默认，0-非默认';
COMMENT ON COLUMN t_models.is_deleted IS '是否删除：1-已删除，0-未删除';
COMMENT ON COLUMN t_models.created_at IS '创建时间';
COMMENT ON COLUMN t_models.updated_at IS '更新时间';
COMMENT ON COLUMN t_models.created_by IS '创建者';
COMMENT ON COLUMN t_models.updated_by IS '更新者';
-- 添加唯一约束
ALTER TABLE t_models
ADD CONSTRAINT uk_model_provider
UNIQUE (model_name, provider, created_by);
COMMENT ON CONSTRAINT uk_model_provider ON t_models
IS '避免同一提供商下模型名称重复';
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
CREATE TRIGGER update_t_models_updated_at
BEFORE UPDATE ON t_models
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 设置管理表
CREATE TABLE IF NOT EXISTS t_sys_param
(
id VARCHAR(100) PRIMARY KEY,
param_value TEXT NOT NULL,
param_type VARCHAR(50) DEFAULT 'string',
option_list TEXT,
description VARCHAR(255) DEFAULT '',
is_built_in BOOLEAN DEFAULT FALSE,
can_modify BOOLEAN DEFAULT TRUE,
is_enabled BOOLEAN DEFAULT TRUE,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_sys_param IS '设置管理表';
COMMENT ON COLUMN t_sys_param.id IS '主键ID,设置项键';
COMMENT ON COLUMN t_sys_param.param_value IS '设置项值';
COMMENT ON COLUMN t_sys_param.param_type IS '设置项类型（仅 string、number、boolean 三种类型）';
COMMENT ON COLUMN t_sys_param.option_list IS '选项列表（逗号分隔，仅对 enum 类型有效）';
COMMENT ON COLUMN t_sys_param.description IS '设置项描述';
COMMENT ON COLUMN t_sys_param.is_built_in IS '是否内置：1-是，0-否';
COMMENT ON COLUMN t_sys_param.can_modify IS '是否可修改：1-可修改，0-不可修改';
COMMENT ON COLUMN t_sys_param.is_enabled IS '是否启用：1-启用，0-禁用';
COMMENT ON COLUMN t_sys_param.created_at IS '创建时间';
COMMENT ON COLUMN t_sys_param.updated_at IS '更新时间';
COMMENT ON COLUMN t_sys_param.created_by IS '创建者';
COMMENT ON COLUMN t_sys_param.updated_by IS '更新者';
-- 创建触发器
DROP TRIGGER IF EXISTS update_t_sys_param_updated_at ON t_sys_param;
CREATE TRIGGER update_t_sys_param_updated_at
BEFORE UPDATE ON t_sys_param
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 插入初始数据
INSERT INTO t_sys_param (id, param_value, param_type, option_list, description, is_built_in,
can_modify, is_enabled, created_by, updated_by)
VALUES
('sys.knowledge.base.count', '200', 'number', '10,200,500', '知识库最大数量', true, true, true, 'system', 'system'),
('SEARCH_API', 'tavily', 'string', 'tavily,infoquest,duckduckgo,brave_search,arxiv', 'deepflow使用的搜索引擎', true, true, true, 'system', 'system'),
('TAVILY_API_KEY', 'tvly-dev-xxx', 'string', '', 'deepflow使用的搜索引擎所需的apiKey', true, true, true, 'system', 'system'),
('BRAVE_SEARCH_API_KEY', 'api-xxx', 'string', '', 'deepflow使用的搜索引擎所需的apiKey', true, true, true, 'system', 'system'),
('JINA_API_KEY', '', 'string', '', 'deepflow使用的JINA搜索引擎所需的apiKey', true, true, true, 'system', 'system'),
('sys.management.dataset.pvc.name', 'deepdata-dataset-pvc', 'string', '', '数据集所在pvc名称', true, false,true, 'system', 'system'),
('DATA_JUICER_EXECUTOR', 'default', 'string', 'default,ray', 'data-juicer使用的执行器', true, true, true, 'system', 'system'),
('sys.home.page.url', '', 'string', '', '首页URL，访问根路径时自动重定向到此URL（为空则不重定向）', true, true, true, 'system', 'system')
ON CONFLICT (id) DO NOTHING;
-- 使用现有的DeepData数据库
\c deepdata;
-- 模型配置表
CREATE TABLE IF NOT EXISTS t_models
(
id VARCHAR(36) PRIMARY KEY,
model_name VARCHAR(100) NOT NULL,
provider VARCHAR(50) NOT NULL,
base_url VARCHAR(255) NOT NULL,
api_key VARCHAR(512) DEFAULT '',
type VARCHAR(50) NOT NULL,
is_enabled BOOLEAN DEFAULT TRUE,
is_default BOOLEAN DEFAULT FALSE,
is_deleted BOOLEAN DEFAULT FALSE,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_models IS '模型配置表';
COMMENT ON COLUMN t_models.id IS '主键ID';
COMMENT ON COLUMN t_models.model_name IS '模型名称（如 qwen2）';
COMMENT ON COLUMN t_models.provider IS '模型提供商（如 Ollama、OpenAI、DeepSeek）';
COMMENT ON COLUMN t_models.base_url IS 'API 基础地址';
COMMENT ON COLUMN t_models.api_key IS 'API 密钥（无密钥则为空）';
COMMENT ON COLUMN t_models.type IS '模型类型（如 chat、embedding）';
COMMENT ON COLUMN t_models.is_enabled IS '是否启用：1-启用，0-禁用';
COMMENT ON COLUMN t_models.is_default IS '是否默认：1-默认，0-非默认';
COMMENT ON COLUMN t_models.is_deleted IS '是否删除：1-已删除，0-未删除';
COMMENT ON COLUMN t_models.created_at IS '创建时间';
COMMENT ON COLUMN t_models.updated_at IS '更新时间';
COMMENT ON COLUMN t_models.created_by IS '创建者';
COMMENT ON COLUMN t_models.updated_by IS '更新者';
-- 添加唯一约束
ALTER TABLE t_models
ADD CONSTRAINT uk_model_provider
UNIQUE (model_name, provider, created_by);
COMMENT ON CONSTRAINT uk_model_provider ON t_models
IS '避免同一提供商下模型名称重复';
-- 创建触发器用于自动更新 updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
CREATE TRIGGER update_t_models_updated_at
BEFORE UPDATE ON t_models
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 设置管理表
CREATE TABLE IF NOT EXISTS t_sys_param
(
id VARCHAR(100) PRIMARY KEY,
param_value TEXT NOT NULL,
param_type VARCHAR(50) DEFAULT 'string',
option_list TEXT,
description VARCHAR(255) DEFAULT '',
is_built_in BOOLEAN DEFAULT FALSE,
can_modify BOOLEAN DEFAULT TRUE,
is_enabled BOOLEAN DEFAULT TRUE,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
created_by VARCHAR(255),
updated_by VARCHAR(255)
);
-- 添加注释
COMMENT ON TABLE t_sys_param IS '设置管理表';
COMMENT ON COLUMN t_sys_param.id IS '主键ID,设置项键';
COMMENT ON COLUMN t_sys_param.param_value IS '设置项值';
COMMENT ON COLUMN t_sys_param.param_type IS '设置项类型（仅 string、number、boolean 三种类型）';
COMMENT ON COLUMN t_sys_param.option_list IS '选项列表（逗号分隔，仅对 enum 类型有效）';
COMMENT ON COLUMN t_sys_param.description IS '设置项描述';
COMMENT ON COLUMN t_sys_param.is_built_in IS '是否内置：1-是，0-否';
COMMENT ON COLUMN t_sys_param.can_modify IS '是否可修改：1-可修改，0-不可修改';
COMMENT ON COLUMN t_sys_param.is_enabled IS '是否启用：1-启用，0-禁用';
COMMENT ON COLUMN t_sys_param.created_at IS '创建时间';
COMMENT ON COLUMN t_sys_param.updated_at IS '更新时间';
COMMENT ON COLUMN t_sys_param.created_by IS '创建者';
COMMENT ON COLUMN t_sys_param.updated_by IS '更新者';
-- 创建触发器
DROP TRIGGER IF EXISTS update_t_sys_param_updated_at ON t_sys_param;
CREATE TRIGGER update_t_sys_param_updated_at
BEFORE UPDATE ON t_sys_param
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- 插入初始数据
INSERT INTO t_sys_param (id, param_value, param_type, option_list, description, is_built_in,
can_modify, is_enabled, created_by, updated_by)
VALUES
('sys.knowledge.base.count', '200', 'number', '10,200,500', '知识库最大数量', true, true, true, 'system', 'system'),
('SEARCH_API', 'tavily', 'string', 'tavily,infoquest,duckduckgo,brave_search,arxiv', 'deep-flow使用的搜索引擎', true, true, true, 'system', 'system'),
('TAVILY_API_KEY', 'tvly-dev-xxx', 'string', '', 'deep-flow使用的搜索引擎所需的apiKey', true, true, true, 'system', 'system'),
('BRAVE_SEARCH_API_KEY', 'api-xxx', 'string', '', 'deep-flow使用的搜索引擎所需的apiKey', true, true, true, 'system', 'system'),
('JINA_API_KEY', '', 'string', '', 'deep-flow使用的JINA搜索引擎所需的apiKey', true, true, true, 'system', 'system'),
('sys.management.dataset.pvc.name', 'deepdata-dataset-pvc', 'string', '', '数据集所在pvc名称', true, false,true, 'system', 'system'),
('DATA_JUICER_EXECUTOR', 'default', 'string', 'default,ray', 'data-juicer使用的执行器', true, true, true, 'system', 'system'),
('sys.home.page.url', '', 'string', '', '首页URL，访问根路径时自动重定向到此URL（为空则不重定向）', true, true, true, 'system', 'system')
ON CONFLICT (id) DO NOTHING;