沃巴蒂
ff22ed3a8b
本次同步 09-12 至 09-28 的代码变更，主要内容： 新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务 功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema） 结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本 未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
531 lines
584 KiB
PL/PgSQL
531 lines
584 KiB
PL/PgSQL
-- 使用现有的DeepData数据库
|
|
\c deepdata;
|
|
|
|
-- 算子表
|
|
CREATE TABLE IF NOT EXISTS t_operator
|
|
(
|
|
id VARCHAR(64) PRIMARY KEY,
|
|
name VARCHAR(64) UNIQUE,
|
|
description VARCHAR(255),
|
|
version VARCHAR(255),
|
|
inputs VARCHAR(255),
|
|
outputs VARCHAR(255),
|
|
runtime TEXT,
|
|
settings TEXT,
|
|
file_name TEXT,
|
|
file_size BIGINT,
|
|
metrics TEXT,
|
|
is_star BOOLEAN,
|
|
usage_count INT DEFAULT 0,
|
|
created_by VARCHAR(255),
|
|
updated_by VARCHAR(255),
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
COMMENT ON TABLE t_operator IS '算子表';
|
|
COMMENT ON COLUMN t_operator.id IS '主键ID';
|
|
COMMENT ON COLUMN t_operator.name IS '算子名称';
|
|
COMMENT ON COLUMN t_operator.description IS '描述';
|
|
COMMENT ON COLUMN t_operator.version IS '版本';
|
|
COMMENT ON COLUMN t_operator.inputs IS '输入类型';
|
|
COMMENT ON COLUMN t_operator.outputs IS '输出类型';
|
|
COMMENT ON COLUMN t_operator.runtime IS '运行时信息';
|
|
COMMENT ON COLUMN t_operator.settings IS '设置信息';
|
|
COMMENT ON COLUMN t_operator.file_name IS '文件名';
|
|
COMMENT ON COLUMN t_operator.file_size IS '文件大小';
|
|
COMMENT ON COLUMN t_operator.metrics IS '性能指标';
|
|
COMMENT ON COLUMN t_operator.usage_count IS '使用次数';
|
|
COMMENT ON COLUMN t_operator.is_star IS '是否收藏';
|
|
COMMENT ON COLUMN t_operator.created_by IS '创建者';
|
|
COMMENT ON COLUMN t_operator.updated_by IS '更新者';
|
|
COMMENT ON COLUMN t_operator.created_at IS '创建时间';
|
|
COMMENT ON COLUMN t_operator.updated_at IS '更新时间';
|
|
|
|
-- 算子版本表
|
|
CREATE TABLE IF NOT EXISTS t_operator_release
|
|
(
|
|
id VARCHAR(64),
|
|
version VARCHAR(255),
|
|
release_date TIMESTAMP,
|
|
changelog JSON,
|
|
created_by VARCHAR(255),
|
|
updated_by VARCHAR(255),
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY (id, version)
|
|
);
|
|
|
|
-- 算子分类表
|
|
CREATE TABLE IF NOT EXISTS t_operator_category
|
|
(
|
|
id VARCHAR(64) PRIMARY KEY,
|
|
name VARCHAR(64) UNIQUE,
|
|
name_en VARCHAR(64),
|
|
value VARCHAR(64) UNIQUE,
|
|
type VARCHAR(64),
|
|
parent_id VARCHAR(64),
|
|
created_by VARCHAR(255),
|
|
updated_by VARCHAR(255),
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
COMMENT ON TABLE t_operator_category IS '算子分类表';
|
|
COMMENT ON COLUMN t_operator_category.id IS '主键ID';
|
|
COMMENT ON COLUMN t_operator_category.name IS '分类名称';
|
|
COMMENT ON COLUMN t_operator_category.value IS '分类值';
|
|
COMMENT ON COLUMN t_operator_category.type IS '分类类型';
|
|
COMMENT ON COLUMN t_operator_category.parent_id IS '父分类ID';
|
|
COMMENT ON COLUMN t_operator_category.created_at IS '创建时间';
|
|
|
|
-- 算子分类关联表
|
|
CREATE TABLE IF NOT EXISTS t_operator_category_relation
|
|
(
|
|
category_id VARCHAR(64),
|
|
operator_id VARCHAR(64),
|
|
created_by VARCHAR(255),
|
|
updated_by VARCHAR(255),
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
PRIMARY KEY (category_id, operator_id)
|
|
);
|
|
|
|
COMMENT ON TABLE t_operator_category_relation IS '算子分类关联表';
|
|
COMMENT ON COLUMN t_operator_category_relation.category_id IS '分类ID';
|
|
COMMENT ON COLUMN t_operator_category_relation.operator_id IS '算子ID';
|
|
|
|
-- 外键约束
|
|
ALTER TABLE t_operator_category_relation
|
|
ADD CONSTRAINT fk_operator_category_relation_category
|
|
FOREIGN KEY (category_id)
|
|
REFERENCES t_operator_category(id)
|
|
ON DELETE CASCADE;
|
|
|
|
ALTER TABLE t_operator_category_relation
|
|
ADD CONSTRAINT fk_operator_category_relation_operator
|
|
FOREIGN KEY (operator_id)
|
|
REFERENCES t_operator(id)
|
|
ON DELETE CASCADE;
|
|
|
|
-- 创建触发器用于自动更新 updated_at
|
|
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
|
RETURNS TRIGGER AS $$
|
|
BEGIN
|
|
NEW.updated_at = CURRENT_TIMESTAMP;
|
|
RETURN NEW;
|
|
END;
|
|
$$ language 'plpgsql';
|
|
|
|
DROP TRIGGER IF EXISTS update_t_operator_updated_at ON t_operator;
|
|
CREATE TRIGGER update_t_operator_updated_at
|
|
BEFORE UPDATE ON t_operator
|
|
FOR EACH ROW
|
|
EXECUTE FUNCTION update_updated_at_column();
|
|
|
|
-- 算子视图
|
|
CREATE OR REPLACE VIEW v_operator AS
|
|
SELECT
|
|
o.id AS operator_id,
|
|
o.name AS operator_name,
|
|
o.description,
|
|
o.version,
|
|
o.inputs,
|
|
o.outputs,
|
|
o.runtime,
|
|
o.settings,
|
|
o.is_star,
|
|
o.file_name,
|
|
o.file_size,
|
|
o.usage_count,
|
|
o.metrics,
|
|
o.created_at,
|
|
o.updated_at,
|
|
o.created_by,
|
|
o.updated_by,
|
|
toc.id AS category_id,
|
|
toc.name AS category_name,
|
|
toc.name_en AS category_name_en
|
|
FROM t_operator_category_relation tocr
|
|
LEFT JOIN t_operator o ON tocr.operator_id = o.id
|
|
LEFT JOIN t_operator_category toc ON tocr.category_id = toc.id;
|
|
|
|
COMMENT ON VIEW v_operator IS '算子视图';
|
|
|
|
INSERT INTO t_operator_category(id, name, name_en, value, type, parent_id)
|
|
VALUES ('64465bec-b46b-11f0-8291-00155d0e4808', '模态', 'Modal', 'modal', 'predefined', '0'),
|
|
('873000a2-65b3-474b-8ccc-4813c08c76fb', '语言', 'Language', 'language', 'predefined', '0'),
|
|
('4857cc9e-7b72-429e-b2a8-ddd1c48c4483', '功能', 'Function', 'function', 'predefined', '0'),
|
|
('d8a5df7a-52a9-42c2-83c4-01062e60f597', '文本', 'Text', 'text', 'predefined', '64465bec-b46b-11f0-8291-00155d0e4808'),
|
|
('de36b61c-9e8a-4422-8c31-d30585c7100f', '图片', 'Image', 'image', 'predefined', '64465bec-b46b-11f0-8291-00155d0e4808'),
|
|
('42dd9392-73e4-458c-81ff-41751ada47b5', '音频', 'Audio', 'audio', 'predefined', '64465bec-b46b-11f0-8291-00155d0e4808'),
|
|
('a233d584-73c8-4188-ad5d-8f7c8dda9c27', '视频', 'Video', 'video', 'predefined', '64465bec-b46b-11f0-8291-00155d0e4808'),
|
|
('4d7dbd77-0a92-44f3-9056-2cd62d4a71e4', '多模态', 'Multimodal', 'multimodal', 'predefined', '64465bec-b46b-11f0-8291-00155d0e4808'),
|
|
('9eda9d5d-072b-499b-916c-797a0a8750e1', 'Python', 'Python', 'python', 'predefined', '873000a2-65b3-474b-8ccc-4813c08c76fb'),
|
|
('8c09476a-a922-418f-a908-733f8a0de521', '清洗', 'Cleaning', 'cleaning', 'predefined', '4857cc9e-7b72-429e-b2a8-ddd1c48c4483'),
|
|
('cfa9d8e2-5b5f-4f1e-9f12-1234567890ab', '标注', 'Annotation', 'annotation', 'predefined', '4857cc9e-7b72-429e-b2a8-ddd1c48c4483'),
|
|
('16e2d99e-eafb-44fc-acd0-f35a2bad28f8', '来源', 'Origin', 'origin', 'predefined', '0'),
|
|
('96a3b07a-3439-4557-a835-525faad60ca3', '系统预置', 'System Preset', 'predefined', 'predefined', '16e2d99e-eafb-44fc-acd0-f35a2bad28f8'),
|
|
('ec2cdd17-8b93-4a81-88c4-ac9e98d10757', '用户上传', 'User Upload', 'customized', 'predefined', '16e2d99e-eafb-44fc-acd0-f35a2bad28f8'),
|
|
('0ed75eea-e20b-11f0-88e6-00155d5c9528', '归属', 'Vendor', 'vendor', 'predefined', '0'),
|
|
('431e7798-5426-4e1a-aae6-b9905a836b34', 'DeepData', 'DeepData', 'deepdata', 'predefined', '0ed75eea-e20b-11f0-88e6-00155d5c9528'),
|
|
('79b385b4-fde8-4617-bcba-02a176938996', 'DataJuicer', 'DataJuicer', 'data-juicer', 'predefined', '0ed75eea-e20b-11f0-88e6-00155d5c9528'),
|
|
('f00eaa3e-96c1-4de4-96cd-9848ef5429ec', '其他', 'Others', 'others', 'predefined', '0ed75eea-e20b-11f0-88e6-00155d5c9528')
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
INSERT INTO t_operator
|
|
(id, name, description, version, inputs, outputs, runtime, settings, file_name, file_size, is_star, created_by, updated_by)
|
|
VALUES
|
|
('MineruFormatter', 'MinerU PDF文本抽取', '基于MinerU API，抽取PDF中的文本。', '1.0.0', 'text', 'text', null, '{"mineruApi":{"name":"Mineru Api地址","description":"指定mineru服务的api地址，默认为本地同一集群内地址。","type":"input","defaultVal":"http://deepdata-mineru:8000","required":false},"exportType":{"name":"导出类型","description":"指定清洗结果文件类型。若指定为md且后续存在其他清洗算子，可能导致文件格式错乱。","type":"select","defaultVal":"markdown","required":false,"options":[{"label":"markdown","value":"md"},{"label":"txt","value":"txt"}]}}', '', 12288, false, 'system', 'system'),
|
|
('FileWithHighRepeatPhraseRateFilter', '文档词重复率检查', '去除重复词过多的文档。', '1.0.0', 'text', 'text', null, '{"repeatPhraseRatio": {"name": "文档词重复率", "description": "某个词的统计数/文档总词数 > 设定值，该文档被去除。", "type": "slider", "defaultVal": 0.5, "min": 0, "max": 1, "step": 0.1}, "hitStopwords": {"name": "去除停用词", "description": "统计重复词时，选择是否要去除停用词。", "type": "switch", "defaultVal": false, "required": true, "checkedLabel": "去除", "unCheckedLabel": "不去除"}}', '', 16384, false, 'system', 'system'),
|
|
('FileWithHighRepeatWordRateFilter', '文档字重复率检查', '去除重复字过多的文档。', '1.0.0', 'text', 'text', null, '{"repeatWordRatio": {"name": "文档字重复率", "description": "某个字的统计数/文档总字数 > 设定值，该文档被去除。", "type": "slider", "defaultVal": 0.5, "min": 0, "max": 1, "step": 0.1}}', '', 8192, false, 'system', 'system'),
|
|
('FileWithHighSpecialCharRateFilter', '文档特殊字符率检查', '去除特殊字符过多的文档。', '1.0.0', 'text', 'text', null, '{"specialCharRatio": {"name": "文档特殊字符率", "description": "特殊字符的统计数/文档总字数 > 设定值，该文档被去除。", "type": "slider", "defaultVal": 0.3, "min": 0, "max": 1, "step": 0.1}}', '', 5120, false, 'system', 'system'),
|
|
('DuplicateFilesFilter', '相似文档去除', '相似文档去除。', '1.0.0', 'text', 'text', null, '{"fileDuplicateThreshold": {"name": "文档相似度", "description": "基于MinHash算法和Jaccard相似度，计算当前文档与数据集中其它文档相似性，超过设定值，该文档被去除。", "type": "slider", "defaultVal": 0.5, "min": 0, "max": 1, "step": 0.1}}', '', 13312, false, 'system', 'system'),
|
|
('FileWithManySensitiveWordsFilter', '文档敏感词率检查', '去除敏感词过多的文档。', '1.0.0', 'text', 'text', null, '{"sensitiveWordsRate": {"name": "文档敏感词率", "description": "敏感词的字数/文档总字数 > 设定值，该文档被去除。", "type": "slider", "defaultVal": 0.01, "min": 0, "max": 1, "step": 0.01}}', '', 29696, false, 'system', 'system'),
|
|
('FileWithShortOrLongLengthFilter', '文档字数检查', '字数不在指定范围会被过滤掉。', '1.0.0', 'text', 'text', null, '{"fileLength": {"name": "文档字数", "description": "过滤字数不在指定范围内的文档，如[10,10000000]。若输入为空，则不对字数上/下限做限制。", "type": "range", "defaultVal": [10, 10000000], "min": 0, "max": 10000000000000000, "step": 1}}', '', 8192, false, 'system', 'system'),
|
|
('ContentCleaner', '文档目录去除', '去除文档中的目录。', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
|
('AnonymizedCreditCardNumber', '信用卡号匿名化', '信用卡号匿名化', '1.0.0', 'text', 'text', null, null, '', 8192, false, 'system', 'system'),
|
|
('EmailNumberCleaner', '邮件地址匿名化', '邮件地址匿名化', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
|
('EmojiCleaner', '文档表情去除', '去除文档中表情字符或者emoji符号。', '1.0.0', 'text', 'text', null, null, '', 5120, false, 'system', 'system'),
|
|
('ExtraSpaceCleaner', '多余空格去除', '移除文档首尾、句中或标点符号附近多余空格和 tab 等。', '1.0.0', 'text', 'text', null, null, '', 8192, false, 'system', 'system'),
|
|
('FullWidthCharacterCleaner', '全角转半角', '将文档中的所有全角字符转换成半角字符。', '1.0.0', 'text', 'text', null, null, '', 8192, false, 'system', 'system'),
|
|
('GrableCharactersCleaner', '文档乱码去除', '去除文档中的乱码和无意义的unicode。', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
|
('HtmlTagCleaner', 'HTML标签去除', '移除文档中HTML标签，如 <html>、<dev>、<p> 等。', '1.0.0', 'text', 'text', null, '{"removeTableTags":{"name":"是否去除表格标签","description":"若为是，则会去除表格标签<tr><td>等。","type":"switch","defaultVal":"false","required":false,"checkedLabel":"是","unCheckedLabel":"否"}}', '', 12288, false, 'system', 'system'),
|
|
('AnonymizedIdNumber', '身份证号匿名化', '身份证号匿名化。', '1.0.0', 'text', 'text', null, null, '', 36864, false, 'system', 'system'),
|
|
('InvisibleCharactersCleaner', '不可见字符去除', '去除文档中的不可见字符，例如 0-31 号字符中的部分字符。', '1.0.0', 'text', 'text', null, null, '', 5120, false, 'system', 'system'),
|
|
('AnonymizedIpAddress', 'IP地址匿名化', 'IP地址匿名化', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
|
('LegendCleaner', '图注表注去除', '去除文档中的图注、表注等内容。', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
|
('AnonymizedPhoneNumber', '电话号码匿名化', '电话号码匿名化', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
|
('PoliticalWordCleaner', '政治文本匿名化', '将政治文本进行匿名化。', '1.0.0', 'text', 'text', null, null, '', 8192, false, 'system', 'system'),
|
|
('DuplicateSentencesFilter', '文档局部内容去重', '文档局部内容去重。', '1.0.0', 'text', 'text', null, null, '', 5120, false, 'system', 'system'),
|
|
('SexualAndViolentWordCleaner', '暴力色情文本匿名化', '将暴力、色情文本进行匿名化。', '1.0.0', 'text', 'text', null, null, '', 20480, false, 'system', 'system'),
|
|
('TraditionalChineseCleaner', '繁体转简体', '将繁体转换为简体。', '1.0.0', 'text', 'text', null, null, '', 5120, false, 'system', 'system'),
|
|
('UnicodeSpaceCleaner', '空格标准化', '将文档中不同的 unicode 空格，如 u2008，转换为正常空格\\u0020。', '1.0.0', 'text', 'text', null, null, '', 8192, false, 'system', 'system'),
|
|
('AnonymizedUrlCleaner', 'URL网址匿名化', '将文档中的url网址匿名化。', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
|
('XMLTagCleaner', 'XML标签去除', '去除XML中的标签。', '1.0.0', 'text', 'text', null, null, '', 4096, false, 'system', 'system'),
|
|
('ImgBlurredImagesCleaner', '模糊图片过滤', '去除模糊的图片。', '1.0.0', 'image', 'image', null, '{"blurredThreshold": {"name": "梯度函数值", "description": "梯度函数值取值越小，图片模糊度越高。", "type": "slider", "defaultVal": 1000, "min": 1, "max": 10000, "step": 1}}', '', 5120, false, 'system', 'system'),
|
|
('ImgBrightness', '图片亮度增强', '自适应调节图片的亮度。', '1.0.0', 'image', 'image', null, null, '', 4096, false, 'system', 'system'),
|
|
('ImgContrast', '图片对比度增强', '自适应调节图片的对比度。', '1.0.0', 'image', 'image', null, null, '', 4096, false, 'system', 'system'),
|
|
('ImgDenoise', '图片噪点去除', '去除图片中的噪点，主要适用于自然场景。', '1.0.0', 'image', 'image', null, null, '', 4096, false, 'system', 'system'),
|
|
('ImgDuplicatedImagesCleaner', '重复图片去除', '去除重复的图片。', '1.0.0', 'image', 'image', null, null, '', 8192, false, 'system', 'system'),
|
|
('ImgPerspectiveTransformation', '图片透视变换', '自适应校正图片的视角，主要适用于文档校正场景。', '1.0.0', 'image', 'image', null, null, '', 8192, false, 'system', 'system'),
|
|
('ImgResize', '图片重采样', '将图片放大或缩小到指定像素。', '1.0.0', 'image', 'image', null, '{"widthSize":{"name":"宽度","type":"inputNumber","description":"像素","defaultVal":256,"min":1,"max":4096,"step":1},"heightSize":{"type":"inputNumber","name":"高度","description":"像素","defaultVal":256,"min":1,"max":4096,"step":1}}', '', 8192, false, 'system', 'system'),
|
|
('ImgSaturation', '图片饱和度增强', '自适应调节图片的饱和度，主要适用于自然场景图片。', '1.0.0', 'image', 'image', null, null, '', 4096, false, 'system', 'system'),
|
|
('ImgShadowRemove', '图片阴影去除', '去除图片中的阴影，主要适用于文档场景。', '1.0.0', 'image', 'image', null, null, '', 4096, false, 'system', 'system'),
|
|
('ImgSharpness', '图片锐度增强', '自适应调节图片的锐度，主要适用于自然场景图片。', '1.0.0', 'image', 'image', null, null, '', 4096, false, 'system', 'system'),
|
|
('ImgSimilarImagesCleaner', '相似图片去除', '去除相似的图片。', '1.0.0', 'image', 'image', null, '{"similarThreshold": {"name": "相似度", "description": "相似度取值越大，图片相似度越高。", "type": "slider", "defaultVal": 0.8, "min": 0, "max": 1, "step": 0.01}}', '', 14336, false, 'system', 'system'),
|
|
('ImgTypeUnify', '图片格式转换', '将图片编码格式统一为jpg、jpeg、png、bmp格式。', '1.0.0', 'image', 'image', null, '{"imgType": {"name": "图片编码格式", "type": "select", "defaultVal": "jpg", "options": [{"label": "jpg", "value": "jpg"}, {"label": "png", "value": "png"}, {"label": "jpeg", "value": "jpeg"}, {"label": "bmp", "value": "bmp"}]}}', '', 5120, false, 'system', 'system'),
|
|
('ImgDirectionCorrect', '图片方向校正', '将含有文字的图片校正到文字水平方向，主要适用于文档场景。', '1.0.0', 'image', 'image', null, null, '', 8192, false, 'system', 'system'),
|
|
('PiiDetector', '高级匿名化', '高级匿名化算子，检测命名实体并匿名化。', '1.0.0', 'text', 'text', null, null, '', 8192, false, 'system', 'system'),
|
|
('ObjectDetectionRectangle', '图像目标检测与预标注', '基于 YOLOv8 的图像目标检测算子。对输入图像进行目标检测，输出带矩形框与类别标签的标注图像，并生成结构化标注 JSON（包含类别、置信度与边界框坐标）。支持将检测结果导出为 Label Studio 兼容的 predictions 预标注格式（rectanglelabels），可在标注任务中直接加载并进行人工校正，从而显著降低人工标注成本并提升标注效率。', '1.0.0', 'image', 'image,json', null, null, '', 12288, false, 'system', 'system')
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
|
SELECT c.id, o.id
|
|
FROM t_operator_category c
|
|
CROSS JOIN t_operator o
|
|
WHERE c.id IN ('d8a5df7a-52a9-42c2-83c4-01062e60f597', '9eda9d5d-072b-499b-916c-797a0a8750e1', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34', '8c09476a-a922-418f-a908-733f8a0de521')
|
|
AND o.id IN ('FileWithShortOrLongLengthFilter', 'FileWithHighRepeatPhraseRateFilter',
|
|
'FileWithHighRepeatWordRateFilter', 'FileWithHighSpecialCharRateFilter', 'FileWithManySensitiveWordsFilter',
|
|
'DuplicateFilesFilter', 'DuplicateSentencesFilter', 'AnonymizedCreditCardNumber', 'AnonymizedIdNumber',
|
|
'AnonymizedIpAddress', 'AnonymizedPhoneNumber', 'AnonymizedUrlCleaner', 'HtmlTagCleaner', 'XMLTagCleaner',
|
|
'ContentCleaner', 'EmailNumberCleaner', 'EmojiCleaner', 'ExtraSpaceCleaner', 'FullWidthCharacterCleaner',
|
|
'GrableCharactersCleaner', 'InvisibleCharactersCleaner', 'LegendCleaner', 'PoliticalWordCleaner',
|
|
'SexualAndViolentWordCleaner', 'TraditionalChineseCleaner', 'UnicodeSpaceCleaner', 'MineruFormatter',
|
|
'PiiDetector')
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
|
SELECT c.id, o.id
|
|
FROM t_operator_category c
|
|
CROSS JOIN t_operator o
|
|
WHERE c.id IN ('de36b61c-9e8a-4422-8c31-d30585c7100f', '9eda9d5d-072b-499b-916c-797a0a8750e1', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34', '8c09476a-a922-418f-a908-733f8a0de521')
|
|
AND o.id IN ('ImgBlurredImagesCleaner', 'ImgBrightness', 'ImgContrast', 'ImgDenoise',
|
|
'ImgDuplicatedImagesCleaner', 'ImgPerspectiveTransformation', 'ImgResize', 'ImgSaturation',
|
|
'ImgShadowRemove', 'ImgSharpness', 'ImgSimilarImagesCleaner', 'ImgTypeUnify', 'ImgDirectionCorrect')
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
-- 图像目标检测与预标注算子：功能归类为「标注」而非「清洗」
|
|
-- INSERT INTO t_operator_category_relation(category_id, operator_id)
|
|
-- SELECT c.id, o.id
|
|
-- FROM t_operator_category c
|
|
-- CROSS JOIN t_operator o
|
|
-- WHERE c.id IN ('de36b61c-9e8a-4422-8c31-d30585c7100f', 'cfa9d8e2-5b5f-4f1e-9f12-1234567890ab', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34','9eda9d5d-072b-499b-916c-797a0a8750e1')
|
|
-- AND o.id IN ('ObjectDetectionRectangle')
|
|
-- ON CONFLICT DO NOTHING;
|
|
|
|
|
|
INSERT INTO t_operator
|
|
(id, name, description, version, inputs, outputs, runtime, settings, file_name, file_size, metrics, is_star, created_by, updated_by)
|
|
VALUES
|
|
('alphanumeric_filter', 'alphanumeric_filter', '过滤字母/数字比例在指定范围内的样本', '1.4.4', 'text', 'text', NULL, '{"tokenization": {"name": "tokenization", "description": "是否使用分词", "type": "switch", "defaultVal": false, "required": false}, "min_ratio": {"name": "min_ratio", "description": "最小比例下限", "type": "inputNumber", "defaultVal": 0.25, "required": false, "min": 0, "max": 10000}, "max_ratio": {"name": "max_ratio", "description": "最大比例上限", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('audio_add_gaussian_noise_mapper', 'audio_add_gaussian_noise_mapper', '向音频样本添加高斯噪声', '1.4.4', 'audio', 'audio', NULL, '{"min_amplitude": {"name": "min_amplitude", "description": "参数: min_amplitude", "type": "inputNumber", "defaultVal": 0.001, "required": false, "min": 0, "max": 10000}, "max_amplitude": {"name": "max_amplitude", "description": "参数: max_amplitude", "type": "inputNumber", "defaultVal": 0.015, "required": false, "min": 0, "max": 10000}, "p": {"name": "p", "description": "执行概率", "type": "inputNumber", "defaultVal": 0.5, "required": false, "min": 0, "max": 10000}, "save_dir": {"name": "save_dir", "description": "保存目录", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('audio_duration_filter', 'audio_duration_filter', '过滤音频持续时间在指定范围内的样本', '1.4.4', 'audio', 'audio', NULL, '{"min_duration": {"name": "min_duration", "description": "最小时长(秒)", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "max_duration": {"name": "max_duration", "description": "最大时长(秒)", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('audio_ffmpeg_wrapped_mapper', 'audio_ffmpeg_wrapped_mapper', '使用FFmpeg音频滤镜处理音频文件', '1.4.4', 'audio', 'audio', NULL, '{"filter_name": {"name": "filter_name", "description": "FFmpeg滤镜名称", "type": "input", "defaultVal": "", "required": false}, "filter_kwargs": {"name": "filter_kwargs", "description": "滤镜参数", "type": "input", "defaultVal": "None", "required": false}, "global_args": {"name": "global_args", "description": "全局参数", "type": "input", "defaultVal": "", "required": false}, "capture_stderr": {"name": "capture_stderr", "description": "是否捕获标准错误输出", "type": "switch", "defaultVal": true, "required": false}, "overwrite_output": {"name": "overwrite_output", "description": "是否覆盖输出文件", "type": "switch", "defaultVal": true, "required": false}, "save_dir": {"name": "save_dir", "description": "保存目录", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('audio_nmf_snr_filter', 'audio_nmf_snr_filter', '过滤音频信噪比(SNR)在指定范围内的样本', '1.4.4', 'audio', 'audio', NULL, '{"min_snr": {"name": "min_snr", "description": "最小信噪比(dB)", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "max_snr": {"name": "max_snr", "description": "最大信噪比(dB)", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "nmf_iter_num": {"name": "nmf_iter_num", "description": "NMF迭代次数", "type": "inputNumber", "defaultVal": 500, "required": false, "min": 0, "max": 10000}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('audio_size_filter', 'audio_size_filter', '根据音频文件大小过滤样本', '1.4.4', 'audio', 'audio', NULL, '{"min_size": {"name": "min_size", "description": "最小大小", "type": "input", "defaultVal": "0", "required": false}, "max_size": {"name": "max_size", "description": "最大大小", "type": "input", "defaultVal": "1TB", "required": false}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('average_line_length_filter', 'average_line_length_filter', '过滤平均行长在指定范围内的样本', '1.4.4', 'text', 'text', NULL, '{"min_len": {"name": "min_len", "description": "最小长度", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "max_len": {"name": "max_len", "description": "最大长度", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('calibrate_qa_mapper', 'calibrate_qa_mapper', '使用API模型根据参考文本校准QA对', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "reference_template": {"name": "reference_template", "description": "参考模板", "type": "input", "defaultVal": "", "required": false}, "qa_pair_template": {"name": "qa_pair_template", "description": "问答对模板", "type": "input", "defaultVal": "", "required": false}, "output_pattern": {"name": "output_pattern", "description": "输出匹配模式", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('calibrate_query_mapper', 'calibrate_query_mapper', '根据参考文本校准QA对中的问题', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "reference_template": {"name": "reference_template", "description": "参考模板", "type": "input", "defaultVal": "", "required": false}, "qa_pair_template": {"name": "qa_pair_template", "description": "问答对模板", "type": "input", "defaultVal": "", "required": false}, "output_pattern": {"name": "output_pattern", "description": "输出匹配模式", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('calibrate_response_mapper', 'calibrate_response_mapper', '根据参考文本校准QA对中的回答', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "reference_template": {"name": "reference_template", "description": "参考模板", "type": "input", "defaultVal": "", "required": false}, "qa_pair_template": {"name": "qa_pair_template", "description": "问答对模板", "type": "input", "defaultVal": "", "required": false}, "output_pattern": {"name": "output_pattern", "description": "输出匹配模式", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('character_repetition_filter', 'character_repetition_filter', '过滤字符级n-gram重复率在指定范围内的样本', '1.4.4', 'text', 'text', NULL, '{"rep_len": {"name": "rep_len", "description": "N-gram重复长度", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "min_ratio": {"name": "min_ratio", "description": "最小比例下限", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "max_ratio": {"name": "max_ratio", "description": "最大比例上限", "type": "inputNumber", "defaultVal": 0.5, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('chinese_convert_mapper', 'chinese_convert_mapper', '在繁体中文、简体中文和日文汉字之间转换文本', '1.4.4', 'text', 'text', NULL, '{"mode": {"name": "mode", "description": "转换模式", "type": "input", "defaultVal": "s2t", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('clean_copyright_mapper', 'clean_copyright_mapper', '清理文本样本开头的版权声明', '1.4.4', 'text', 'text', NULL, '{"video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('clean_email_mapper', 'clean_email_mapper', '使用正则表达式清理文本中的邮箱地址', '1.4.4', 'text', 'text', NULL, '{"pattern": {"name": "pattern", "description": "正则表达式模式", "type": "input", "defaultVal": "", "required": false}, "repl": {"name": "repl", "description": "替换内容", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('clean_html_mapper', 'clean_html_mapper', '清理HTML代码，将HTML转换为纯文本', '1.4.4', 'text', 'text', NULL, '{"video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('clean_ip_mapper', 'clean_ip_mapper', '清理文本中的IPv4和IPv6地址', '1.4.4', 'text', 'text', NULL, '{"pattern": {"name": "pattern", "description": "正则表达式模式", "type": "input", "defaultVal": "", "required": false}, "repl": {"name": "repl", "description": "替换内容", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('clean_links_mapper', 'clean_links_mapper', '清理文本中的http/https/ftp链接', '1.4.4', 'text', 'text', NULL, '{"pattern": {"name": "pattern", "description": "正则表达式模式", "type": "input", "defaultVal": "", "required": false}, "repl": {"name": "repl", "description": "替换内容", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('detect_character_attributes_mapper', 'detect_character_attributes_mapper', '输入图像、标题和角色名称，提取角色的属性信息', '1.4.4', 'text', 'text', NULL, '{"detect_character_locations_mapper_args": {"name": "detect_character_locations_mapper_args", "description": "角色位置检测参数(JSON)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('detect_character_locations_mapper', 'detect_character_locations_mapper', '根据图像和角色名称列表，提取每个出现角色的边界框', '1.4.4', 'text', 'text', NULL, '{"mllm_mapper_args": {"name": "mllm_mapper_args", "description": "多模态大模型参数(JSON)", "type": "input", "defaultVal": "{}", "required": false}, "image_text_matching_filter_args": {"name": "image_text_matching_filter_args", "description": "图文匹配过滤参数(JSON)", "type": "input", "defaultVal": "{}", "required": false}, "yoloe_path": {"name": "yoloe_path", "description": "YOLOE模型文件路径", "type": "input", "defaultVal": "yoloe-11l-seg.pt", "required": false}, "iou_threshold": {"name": "iou_threshold", "description": "IoU重叠度阈值", "type": "input", "defaultVal": "0.7", "required": false}, "matching_score_threshold": {"name": "matching_score_threshold", "description": "匹配分数阈值", "type": "input", "defaultVal": "0.4", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('detect_main_character_mapper', 'detect_main_character_mapper', '根据图像及其标题提取所有主要角色名称', '1.4.4', 'text', 'text', NULL, '{"mllm_mapper_args": {"name": "mllm_mapper_args", "description": "多模态大模型参数(JSON)", "type": "input", "defaultVal": "{}", "required": false}, "filter_min_character_num": {"name": "filter_min_character_num", "description": "最少字符过滤阈值", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('dialog_intent_detection_mapper', 'dialog_intent_detection_mapper', '通过分析对话历史、查询和回答生成用户意图标签', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "intent_candidates": {"name": "intent_candidates", "description": "意图候选列表", "type": "input", "defaultVal": "", "required": false}, "max_round": {"name": "max_round", "description": "最大对话轮数", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "labels_key": {"name": "labels_key", "description": "标签字段名", "type": "input", "defaultVal": "dialog_intent_labels", "required": false}, "analysis_key": {"name": "analysis_key", "description": "分析结果字段名", "type": "input", "defaultVal": "dialog_intent_labels_analysis", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "query_template": {"name": "query_template", "description": "查询模板", "type": "input", "defaultVal": "", "required": false}, "response_template": {"name": "response_template", "description": "响应模板", "type": "input", "defaultVal": "", "required": false}, "candidate_template": {"name": "candidate_template", "description": "候选模板", "type": "input", "defaultVal": "", "required": false}, "analysis_template": {"name": "analysis_template", "description": "分析模板", "type": "input", "defaultVal": "", "required": false}, "labels_template": {"name": "labels_template", "description": "标签模板", "type": "input", "defaultVal": "", "required": false}, "analysis_pattern": {"name": "analysis_pattern", "description": "分析匹配模式", "type": "input", "defaultVal": "", "required": false}, "labels_pattern": {"name": "labels_pattern", "description": "标签匹配模式", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('dialog_sentiment_detection_mapper', 'dialog_sentiment_detection_mapper', '生成对话中用户查询的情感标签和分析', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "sentiment_candidates": {"name": "sentiment_candidates", "description": "情感候选列表", "type": "input", "defaultVal": "", "required": false}, "max_round": {"name": "max_round", "description": "最大对话轮数", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "labels_key": {"name": "labels_key", "description": "标签字段名", "type": "input", "defaultVal": "dialog_sentiment_labels", "required": false}, "analysis_key": {"name": "analysis_key", "description": "分析结果字段名", "type": "input", "defaultVal": "dialog_sentiment_labels_analysis", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "query_template": {"name": "query_template", "description": "查询模板", "type": "input", "defaultVal": "", "required": false}, "response_template": {"name": "response_template", "description": "响应模板", "type": "input", "defaultVal": "", "required": false}, "candidate_template": {"name": "candidate_template", "description": "候选模板", "type": "input", "defaultVal": "", "required": false}, "analysis_template": {"name": "analysis_template", "description": "分析模板", "type": "input", "defaultVal": "", "required": false}, "labels_template": {"name": "labels_template", "description": "标签模板", "type": "input", "defaultVal": "", "required": false}, "analysis_pattern": {"name": "analysis_pattern", "description": "分析匹配模式", "type": "input", "defaultVal": "", "required": false}, "labels_pattern": {"name": "labels_pattern", "description": "标签匹配模式", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('dialog_sentiment_intensity_mapper', 'dialog_sentiment_intensity_mapper', '预测对话中用户情感强度，范围从-5到5', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "max_round": {"name": "max_round", "description": "最大对话轮数", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "intensities_key": {"name": "intensities_key", "description": "强度字段名", "type": "input", "defaultVal": "dialog_sentiment_intensity", "required": false}, "analysis_key": {"name": "analysis_key", "description": "分析结果字段名", "type": "input", "defaultVal": "dialog_sentiment_intensity_analysis", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "query_template": {"name": "query_template", "description": "查询模板", "type": "input", "defaultVal": "", "required": false}, "response_template": {"name": "response_template", "description": "响应模板", "type": "input", "defaultVal": "", "required": false}, "analysis_template": {"name": "analysis_template", "description": "分析模板", "type": "input", "defaultVal": "", "required": false}, "intensity_template": {"name": "intensity_template", "description": "强度模板", "type": "input", "defaultVal": "", "required": false}, "analysis_pattern": {"name": "analysis_pattern", "description": "分析匹配模式", "type": "input", "defaultVal": "", "required": false}, "intensity_pattern": {"name": "intensity_pattern", "description": "强度匹配模式", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('dialog_topic_detection_mapper', 'dialog_topic_detection_mapper', '生成对话中用户的主题标签和分析', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "topic_candidates": {"name": "topic_candidates", "description": "主题候选列表", "type": "input", "defaultVal": "", "required": false}, "max_round": {"name": "max_round", "description": "最大对话轮数", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "labels_key": {"name": "labels_key", "description": "标签字段名", "type": "input", "defaultVal": "dialog_topic_labels", "required": false}, "analysis_key": {"name": "analysis_key", "description": "分析结果字段名", "type": "input", "defaultVal": "dialog_topic_labels_analysis", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "query_template": {"name": "query_template", "description": "查询模板", "type": "input", "defaultVal": "", "required": false}, "response_template": {"name": "response_template", "description": "响应模板", "type": "input", "defaultVal": "", "required": false}, "candidate_template": {"name": "candidate_template", "description": "候选模板", "type": "input", "defaultVal": "", "required": false}, "analysis_template": {"name": "analysis_template", "description": "分析模板", "type": "input", "defaultVal": "", "required": false}, "labels_template": {"name": "labels_template", "description": "标签模板", "type": "input", "defaultVal": "", "required": false}, "analysis_pattern": {"name": "analysis_pattern", "description": "分析匹配模式", "type": "input", "defaultVal": "", "required": false}, "labels_pattern": {"name": "labels_pattern", "description": "标签匹配模式", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('document_deduplicator', 'document_deduplicator', '使用精确匹配在文档级别删除重复样本', '1.4.4', 'text', 'text', NULL, '{"lowercase": {"name": "lowercase", "description": "是否转换为小写", "type": "switch", "defaultVal": false, "required": false}, "ignore_non_character": {"name": "ignore_non_character", "description": "是否忽略非字符", "type": "switch", "defaultVal": false, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('document_minhash_deduplicator', 'document_minhash_deduplicator', '使用MinHash LSH算法在文档级别删除重复样本', '1.4.4', 'text', 'text', NULL, '{"tokenization": {"name": "tokenization", "description": "是否使用分词", "type": "input", "defaultVal": "space", "required": false}, "window_size": {"name": "window_size", "description": "滑动窗口大小", "type": "inputNumber", "defaultVal": 5, "required": false, "min": 0, "max": 10000}, "lowercase": {"name": "lowercase", "description": "是否转换为小写", "type": "switch", "defaultVal": true, "required": false}, "ignore_pattern": {"name": "ignore_pattern", "description": "忽略模式(正则)", "type": "input", "defaultVal": "", "required": false}, "num_permutations": {"name": "num_permutations", "description": "排列函数数量", "type": "inputNumber", "defaultVal": 256, "required": false, "min": 0, "max": 10000}, "jaccard_threshold": {"name": "jaccard_threshold", "description": "Jaccard相似度阈值", "type": "inputNumber", "defaultVal": 0.7, "required": false, "min": 0, "max": 10000}, "num_bands": {"name": "num_bands", "description": "LSH分桶数量", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "num_rows_per_band": {"name": "num_rows_per_band", "description": "每桶行数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "tokenizer_model": {"name": "tokenizer_model", "description": "分词器模型路径", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('document_simhash_deduplicator', 'document_simhash_deduplicator', '使用SimHash算法在文档级别删除重复样本', '1.4.4', 'text', 'text', NULL, '{"tokenization": {"name": "tokenization", "description": "是否使用分词", "type": "input", "defaultVal": "space", "required": false}, "window_size": {"name": "window_size", "description": "滑动窗口大小", "type": "inputNumber", "defaultVal": 6, "required": false, "min": 0, "max": 10000}, "lowercase": {"name": "lowercase", "description": "是否转换为小写", "type": "switch", "defaultVal": true, "required": false}, "ignore_pattern": {"name": "ignore_pattern", "description": "忽略模式(正则)", "type": "input", "defaultVal": "", "required": false}, "num_blocks": {"name": "num_blocks", "description": "分块数量", "type": "inputNumber", "defaultVal": 6, "required": false, "min": 0, "max": 10000}, "hamming_distance": {"name": "hamming_distance", "description": "汉明距离阈值", "type": "inputNumber", "defaultVal": 4, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('download_file_mapper', 'download_file_mapper', '下载URL文件到本地或加载到内存中', '1.4.4', 'text', 'text', NULL, '{"download_field": {"name": "download_field", "description": "下载链接字段名", "type": "input", "defaultVal": "", "required": false}, "save_dir": {"name": "save_dir", "description": "保存目录", "type": "input", "defaultVal": "", "required": false}, "save_field": {"name": "save_field", "description": "保存路径字段名", "type": "input", "defaultVal": "", "required": false}, "resume_download": {"name": "resume_download", "description": "是否支持断点续传", "type": "switch", "defaultVal": false, "required": false}, "timeout": {"name": "timeout", "description": "超时时间(秒)", "type": "inputNumber", "defaultVal": 30, "required": false, "min": 0, "max": 10000}, "max_concurrent": {"name": "max_concurrent", "description": "最大并发数", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('entity_attribute_aggregator', 'entity_attribute_aggregator', '汇总一组文档中实体的给定属性', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "entity": {"name": "entity", "description": "参数: entity", "type": "input", "defaultVal": "", "required": false}, "attribute": {"name": "attribute", "description": "参数: attribute", "type": "input", "defaultVal": "", "required": false}, "input_key": {"name": "input_key", "description": "参数: input_key", "type": "input", "defaultVal": "event_description", "required": false}, "output_key": {"name": "output_key", "description": "参数: output_key", "type": "input", "defaultVal": "entity_attribute", "required": false}, "word_limit": {"name": "word_limit", "description": "参数: word_limit", "type": "inputNumber", "defaultVal": 100, "required": false, "min": 0, "max": 10000}, "max_token_num": {"name": "max_token_num", "description": "参数: max_token_num", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt_template": {"name": "system_prompt_template", "description": "参数: system_prompt_template", "type": "input", "defaultVal": "", "required": false}, "example_prompt": {"name": "example_prompt", "description": "参数: example_prompt", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "output_pattern_template": {"name": "output_pattern_template", "description": "参数: output_pattern_template", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('expand_macro_mapper', 'expand_macro_mapper', '', '1.4.4', 'text', 'text', NULL, '{"video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('extract_entity_attribute_mapper', 'extract_entity_attribute_mapper', '', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "query_entities": {"name": "query_entities", "description": "参数: query_entities", "type": "input", "defaultVal": "[]", "required": false}, "query_attributes": {"name": "query_attributes", "description": "参数: query_attributes", "type": "input", "defaultVal": "[]", "required": false}, "entity_key": {"name": "entity_key", "description": "参数: entity_key", "type": "input", "defaultVal": "main_entities", "required": false}, "attribute_key": {"name": "attribute_key", "description": "参数: attribute_key", "type": "input", "defaultVal": "attributes", "required": false}, "attribute_desc_key": {"name": "attribute_desc_key", "description": "参数: attribute_desc_key", "type": "input", "defaultVal": "attribute_descriptions", "required": false}, "support_text_key": {"name": "support_text_key", "description": "支持文本字段名", "type": "input", "defaultVal": "attribute_support_texts", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt_template": {"name": "system_prompt_template", "description": "参数: system_prompt_template", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "attr_pattern_template": {"name": "attr_pattern_template", "description": "参数: attr_pattern_template", "type": "input", "defaultVal": "", "required": false}, "demo_pattern": {"name": "demo_pattern", "description": "参数: demo_pattern", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "drop_text": {"name": "drop_text", "description": "参数: drop_text", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('extract_entity_relation_mapper', 'extract_entity_relation_mapper', '', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "entity_types": {"name": "entity_types", "description": "参数: entity_types", "type": "input", "defaultVal": "", "required": false}, "entity_key": {"name": "entity_key", "description": "参数: entity_key", "type": "input", "defaultVal": "entity", "required": false}, "relation_key": {"name": "relation_key", "description": "参数: relation_key", "type": "input", "defaultVal": "relation", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "prompt_template": {"name": "prompt_template", "description": "参数: prompt_template", "type": "input", "defaultVal": "", "required": false}, "tuple_delimiter": {"name": "tuple_delimiter", "description": "参数: tuple_delimiter", "type": "input", "defaultVal": "", "required": false}, "record_delimiter": {"name": "record_delimiter", "description": "参数: record_delimiter", "type": "input", "defaultVal": "", "required": false}, "completion_delimiter": {"name": "completion_delimiter", "description": "参数: completion_delimiter", "type": "input", "defaultVal": "", "required": false}, "max_gleaning": {"name": "max_gleaning", "description": "参数: max_gleaning", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "continue_prompt": {"name": "continue_prompt", "description": "参数: continue_prompt", "type": "input", "defaultVal": "", "required": false}, "if_loop_prompt": {"name": "if_loop_prompt", "description": "参数: if_loop_prompt", "type": "input", "defaultVal": "", "required": false}, "entity_pattern": {"name": "entity_pattern", "description": "参数: entity_pattern", "type": "input", "defaultVal": "", "required": false}, "relation_pattern": {"name": "relation_pattern", "description": "参数: relation_pattern", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "drop_text": {"name": "drop_text", "description": "参数: drop_text", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('extract_event_mapper', 'extract_event_mapper', '使用API模型从文本中提取事件信息', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "event_desc_key": {"name": "event_desc_key", "description": "参数: event_desc_key", "type": "input", "defaultVal": "event_description", "required": false}, "relevant_char_key": {"name": "relevant_char_key", "description": "参数: relevant_char_key", "type": "input", "defaultVal": "relevant_characters", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "output_pattern": {"name": "output_pattern", "description": "输出匹配模式", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "drop_text": {"name": "drop_text", "description": "参数: drop_text", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('extract_keyword_mapper', 'extract_keyword_mapper', '使用API模型从文本中提取关键词', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "keyword_key": {"name": "keyword_key", "description": "参数: keyword_key", "type": "input", "defaultVal": "keyword", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "prompt_template": {"name": "prompt_template", "description": "参数: prompt_template", "type": "input", "defaultVal": "", "required": false}, "completion_delimiter": {"name": "completion_delimiter", "description": "参数: completion_delimiter", "type": "input", "defaultVal": "", "required": false}, "output_pattern": {"name": "output_pattern", "description": "输出匹配模式", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "drop_text": {"name": "drop_text", "description": "参数: drop_text", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('extract_nickname_mapper', 'extract_nickname_mapper', '从对话历史中提取昵称', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "nickname_key": {"name": "nickname_key", "description": "参数: nickname_key", "type": "input", "defaultVal": "nickname", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "output_pattern": {"name": "output_pattern", "description": "输出匹配模式", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "drop_text": {"name": "drop_text", "description": "参数: drop_text", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('extract_support_text_mapper', 'extract_support_text_mapper', '从参考文本中提取支持给定QA对的文本片段', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "summary_key": {"name": "summary_key", "description": "参数: summary_key", "type": "input", "defaultVal": "event_description", "required": false}, "support_text_key": {"name": "support_text_key", "description": "支持文本字段名", "type": "input", "defaultVal": "support_text", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "drop_text": {"name": "drop_text", "description": "参数: drop_text", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('extract_tables_from_html_mapper', 'extract_tables_from_html_mapper', '从HTML中提取表格数据并转换为JSON格式', '1.4.4', 'text', 'text', NULL, '{"tables_field_name": {"name": "tables_field_name", "description": "表格字段名", "type": "input", "defaultVal": "html_tables", "required": false}, "retain_html_tags": {"name": "retain_html_tags", "description": "参数: retain_html_tags", "type": "switch", "defaultVal": false, "required": false}, "include_header": {"name": "include_header", "description": "是否包含表头", "type": "switch", "defaultVal": true, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('fix_unicode_mapper', 'fix_unicode_mapper', '修复文本中的Unicode错误', '1.4.4', 'text', 'text', NULL, '{"normalization": {"name": "normalization", "description": "参数: normalization", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('flagged_words_filter', 'flagged_words_filter', '过滤标记词比例在指定范围内的样本', '1.4.4', 'text', 'text', NULL, '{"lang": {"name": "lang", "description": "目标语言", "type": "input", "defaultVal": "en", "required": false}, "tokenization": {"name": "tokenization", "description": "是否使用分词", "type": "switch", "defaultVal": false, "required": false}, "min_ratio": {"name": "min_ratio", "description": "最小比例下限", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "max_ratio": {"name": "max_ratio", "description": "最大比例上限", "type": "inputNumber", "defaultVal": 0.045, "required": false, "min": 0, "max": 10000}, "flagged_words_dir": {"name": "flagged_words_dir", "description": "参数: flagged_words_dir", "type": "input", "defaultVal": "C:\\Users\\Administrator/.cache\\data_juicer\\assets", "required": false}, "use_words_aug": {"name": "use_words_aug", "description": "参数: use_words_aug", "type": "switch", "defaultVal": false, "required": false}, "words_aug_group_sizes": {"name": "words_aug_group_sizes", "description": "参数: words_aug_group_sizes", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "words_aug_join_char": {"name": "words_aug_join_char", "description": "参数: words_aug_join_char", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('frequency_specified_field_selector', 'frequency_specified_field_selector', '根据指定字段的频率选择样本', '1.4.4', 'text', 'text', NULL, '{"field_key": {"name": "field_key", "description": "参数: field_key", "type": "input", "defaultVal": "", "required": false}, "top_ratio": {"name": "top_ratio", "description": "参数: top_ratio", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "topk": {"name": "topk", "description": "参数: topk", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "reverse": {"name": "reverse", "description": "参数: reverse", "type": "switch", "defaultVal": true, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('general_field_filter', 'general_field_filter', '根据常规字段筛选条件保留样本', '1.4.4', 'text', 'text', NULL, '{"filter_condition": {"name": "filter_condition", "description": "参数: filter_condition", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('general_fused_op', 'general_fused_op', '通用融合算子', '1.4.4', 'text', 'text', NULL, '{"batch_size": {"name": "batch_size", "description": "参数: batch_size", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "fused_op_list": {"name": "fused_op_list", "description": "参数: fused_op_list", "type": "input", "defaultVal": "None", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('generate_qa_from_examples_mapper', 'generate_qa_from_examples_mapper', '基于示例使用HF模型生成QA对', '1.4.4', 'text', 'text', NULL, '{"hf_model": {"name": "hf_model", "description": "HuggingFace模型路径", "type": "input", "defaultVal": "Qwen/Qwen2.5-7B-Instruct", "required": false}, "seed_file": {"name": "seed_file", "description": "参数: seed_file", "type": "input", "defaultVal": "", "required": false}, "example_num": {"name": "example_num", "description": "示例数量", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "similarity_threshold": {"name": "similarity_threshold", "description": "参数: similarity_threshold", "type": "inputNumber", "defaultVal": 0.7, "required": false, "min": 0, "max": 10000}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "example_template": {"name": "example_template", "description": "参数: example_template", "type": "input", "defaultVal": "", "required": false}, "qa_pair_template": {"name": "qa_pair_template", "description": "问答对模板", "type": "input", "defaultVal": "", "required": false}, "output_pattern": {"name": "output_pattern", "description": "输出匹配模式", "type": "input", "defaultVal": "", "required": false}, "enable_vllm": {"name": "enable_vllm", "description": "参数: enable_vllm", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "None", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "None", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('generate_qa_from_text_mapper', 'generate_qa_from_text_mapper', '基于文本内容使用HF模型生成QA对', '1.4.4', 'text', 'text', NULL, '{"hf_model": {"name": "hf_model", "description": "HuggingFace模型路径", "type": "input", "defaultVal": "alibaba-pai/pai-qwen1_5-7b-doc2qa", "required": false}, "max_num": {"name": "max_num", "description": "参数: max_num", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "output_pattern": {"name": "output_pattern", "description": "输出匹配模式", "type": "input", "defaultVal": "", "required": false}, "enable_vllm": {"name": "enable_vllm", "description": "参数: enable_vllm", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "None", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "None", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('human_preference_annotation_mapper', 'human_preference_annotation_mapper', '使用LLM进行人类偏好标注', '1.4.4', 'text', 'text', NULL, '{"label_config_file": {"name": "label_config_file", "description": "标注配置文件路径", "type": "input", "defaultVal": "", "required": false}, "answer1_key": {"name": "answer1_key", "description": "参数: answer1_key", "type": "input", "defaultVal": "answer1", "required": false}, "answer2_key": {"name": "answer2_key", "description": "参数: answer2_key", "type": "input", "defaultVal": "answer2", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "chosen_key": {"name": "chosen_key", "description": "参数: chosen_key", "type": "input", "defaultVal": "chosen", "required": false}, "rejected_key": {"name": "rejected_key", "description": "参数: rejected_key", "type": "input", "defaultVal": "rejected", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_aesthetics_filter', 'image_aesthetics_filter', '过滤美学评分在指定范围内的图像样本', '1.4.4', 'image', 'image', NULL, '{"hf_scorer_model": {"name": "hf_scorer_model", "description": "评分模型路径", "type": "input", "defaultVal": "", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0.5, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 1.0, "required": false, "min": 0, "max": 10000}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_aspect_ratio_filter', 'image_aspect_ratio_filter', '过滤长宽比在指定范围内的图像样本', '1.4.4', 'image', 'image', NULL, '{"min_ratio": {"name": "min_ratio", "description": "最小比例下限", "type": "inputNumber", "defaultVal": 0.333, "required": false, "min": 0, "max": 10000}, "max_ratio": {"name": "max_ratio", "description": "最大比例上限", "type": "inputNumber", "defaultVal": 3.0, "required": false, "min": 0, "max": 10000}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_blur_mapper', 'image_blur_mapper', '对图像进行模糊处理', '1.4.4', 'image', 'image', NULL, '{"p": {"name": "p", "description": "执行概率", "type": "inputNumber", "defaultVal": 0.2, "required": false, "min": 0, "max": 10000}, "blur_type": {"name": "blur_type", "description": "参数: blur_type", "type": "input", "defaultVal": "gaussian", "required": false}, "radius": {"name": "radius", "description": "参数: radius", "type": "inputNumber", "defaultVal": 2.0, "required": false, "min": 0, "max": 10000}, "save_dir": {"name": "save_dir", "description": "保存目录", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_captioning_from_gpt4v_mapper', 'image_captioning_from_gpt4v_mapper', '使用GPT-4 Vision模型为图像生成文本标题', '1.4.4', 'image', 'image', NULL, '{"mode": {"name": "mode", "description": "转换模式", "type": "input", "defaultVal": "description", "required": false}, "api_key": {"name": "api_key", "description": "API密钥", "type": "input", "defaultVal": "", "required": false}, "max_token": {"name": "max_token", "description": "参数: max_token", "type": "inputNumber", "defaultVal": 500, "required": false, "min": 0, "max": 10000}, "temperature": {"name": "temperature", "description": "温度参数(控制创造性)", "type": "inputNumber", "defaultVal": 1.0, "required": false, "min": 0, "max": 10000}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "user_prompt": {"name": "user_prompt", "description": "参数: user_prompt", "type": "input", "defaultVal": "", "required": false}, "user_prompt_key": {"name": "user_prompt_key", "description": "参数: user_prompt_key", "type": "input", "defaultVal": "", "required": false}, "keep_original_sample": {"name": "keep_original_sample", "description": "是否保留原始样本", "type": "switch", "defaultVal": true, "required": false}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_captioning_mapper', 'image_captioning_mapper', '使用HF模型为图像生成文本标题', '1.4.4', 'image', 'image', NULL, '{"hf_img2seq": {"name": "hf_img2seq", "description": "Image-to-Seq模型路径", "type": "input", "defaultVal": "Salesforce/blip2-opt-2.7b", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "caption_num": {"name": "caption_num", "description": "参数: caption_num", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "keep_candidate_mode": {"name": "keep_candidate_mode", "description": "参数: keep_candidate_mode", "type": "input", "defaultVal": "random_any", "required": false}, "keep_original_sample": {"name": "keep_original_sample", "description": "是否保留原始样本", "type": "switch", "defaultVal": true, "required": false}, "prompt": {"name": "prompt", "description": "提示词", "type": "input", "defaultVal": "", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_deduplicator', 'image_deduplicator', '通过图像精确匹配删除重复样本', '1.4.4', 'image', 'image', NULL, '{"method": {"name": "method", "description": "参数: method", "type": "input", "defaultVal": "phash", "required": false}, "consider_text": {"name": "consider_text", "description": "是否考虑文本内容", "type": "switch", "defaultVal": false, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_detection_yolo_mapper', 'image_detection_yolo_mapper', '使用YOLO模型进行图像目标检测', '1.4.4', 'image', 'image', NULL, '{"imgsz": {"name": "imgsz", "description": "参数: imgsz", "type": "input", "defaultVal": "640", "required": false}, "conf": {"name": "conf", "description": "参数: conf", "type": "input", "defaultVal": "0.05", "required": false}, "iou": {"name": "iou", "description": "参数: iou", "type": "input", "defaultVal": "0.5", "required": false}, "model_path": {"name": "model_path", "description": "模型文件路径", "type": "input", "defaultVal": "yolo11n.pt", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_diffusion_mapper', 'image_diffusion_mapper', '使用Diffusion模型从文本生成图像', '1.4.4', 'image', 'image', NULL, '{"hf_diffusion": {"name": "hf_diffusion", "description": "Diffusion模型路径", "type": "input", "defaultVal": "CompVis/stable-diffusion-v1-4", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "torch_dtype": {"name": "torch_dtype", "description": "PyTorch数据类型", "type": "input", "defaultVal": "fp32", "required": false}, "revision": {"name": "revision", "description": "参数: revision", "type": "input", "defaultVal": "main", "required": false}, "strength": {"name": "strength", "description": "参数: strength", "type": "inputNumber", "defaultVal": 0.8, "required": false, "min": 0, "max": 10000}, "guidance_scale": {"name": "guidance_scale", "description": "引导缩放系数", "type": "inputNumber", "defaultVal": 7.5, "required": false, "min": 0, "max": 10000}, "aug_num": {"name": "aug_num", "description": "参数: aug_num", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "keep_original_sample": {"name": "keep_original_sample", "description": "是否保留原始样本", "type": "switch", "defaultVal": true, "required": false}, "caption_key": {"name": "caption_key", "description": "参数: caption_key", "type": "input", "defaultVal": "", "required": false}, "hf_img2seq": {"name": "hf_img2seq", "description": "Image-to-Seq模型路径", "type": "input", "defaultVal": "Salesforce/blip2-opt-2.7b", "required": false}, "save_dir": {"name": "save_dir", "description": "保存目录", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_face_blur_mapper', 'image_face_blur_mapper', '对图像中的人脸进行模糊处理', '1.4.4', 'image', 'image', NULL, '{"cv_classifier": {"name": "cv_classifier", "description": "OpenCV分类器路径", "type": "input", "defaultVal": "", "required": false}, "blur_type": {"name": "blur_type", "description": "参数: blur_type", "type": "input", "defaultVal": "gaussian", "required": false}, "radius": {"name": "radius", "description": "参数: radius", "type": "inputNumber", "defaultVal": 2.0, "required": false, "min": 0, "max": 10000}, "save_dir": {"name": "save_dir", "description": "保存目录", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_face_count_filter', 'image_face_count_filter', '过滤人脸数量在指定范围内的图像样本', '1.4.4', 'image', 'image', NULL, '{"cv_classifier": {"name": "cv_classifier", "description": "OpenCV分类器路径", "type": "input", "defaultVal": "", "required": false}, "min_face_count": {"name": "min_face_count", "description": "参数: min_face_count", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "max_face_count": {"name": "max_face_count", "description": "参数: max_face_count", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_face_ratio_filter', 'image_face_ratio_filter', '过滤人脸面积占比在指定范围内的图像样本', '1.4.4', 'image', 'image', NULL, '{"cv_classifier": {"name": "cv_classifier", "description": "OpenCV分类器路径", "type": "input", "defaultVal": "", "required": false}, "min_ratio": {"name": "min_ratio", "description": "最小比例下限", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "max_ratio": {"name": "max_ratio", "description": "最大比例上限", "type": "inputNumber", "defaultVal": 0.4, "required": false, "min": 0, "max": 10000}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_nsfw_filter', 'image_nsfw_filter', '过滤NSFW评分在指定范围内的图像样本', '1.4.4', 'image', 'image', NULL, '{"hf_nsfw_model": {"name": "hf_nsfw_model", "description": "NSFW检测模型路径", "type": "input", "defaultVal": "Falconsai/nsfw_image_detection", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 0.5, "required": false, "min": 0, "max": 10000}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_pair_similarity_filter', 'image_pair_similarity_filter', '根据图像对之间的相似度过滤', '1.4.4', 'image', 'image', NULL, '{"hf_clip": {"name": "hf_clip", "description": "CLIP模型路径", "type": "input", "defaultVal": "openai/clip-vit-base-patch32", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "input", "defaultVal": "False", "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_remove_background_mapper', 'image_remove_background_mapper', '移除图像背景', '1.4.4', 'image', 'image', NULL, '{"alpha_matting": {"name": "alpha_matting", "description": "是否使用Alpha Matting抠图", "type": "switch", "defaultVal": false, "required": false}, "alpha_matting_foreground_threshold": {"name": "alpha_matting_foreground_threshold", "description": "参数: alpha_matting_foreground_threshold", "type": "inputNumber", "defaultVal": 240, "required": false, "min": 0, "max": 10000}, "alpha_matting_background_threshold": {"name": "alpha_matting_background_threshold", "description": "参数: alpha_matting_background_threshold", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "alpha_matting_erode_size": {"name": "alpha_matting_erode_size", "description": "参数: alpha_matting_erode_size", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "bgcolor": {"name": "bgcolor", "description": "参数: bgcolor", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "save_dir": {"name": "save_dir", "description": "保存目录", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_segment_mapper', 'image_segment_mapper', '使用Segment Anything模型进行图像分割', '1.4.4', 'image', 'image', NULL, '{"imgsz": {"name": "imgsz", "description": "参数: imgsz", "type": "input", "defaultVal": "1024", "required": false}, "conf": {"name": "conf", "description": "参数: conf", "type": "input", "defaultVal": "0.05", "required": false}, "iou": {"name": "iou", "description": "参数: iou", "type": "input", "defaultVal": "0.5", "required": false}, "model_path": {"name": "model_path", "description": "模型文件路径", "type": "input", "defaultVal": "FastSAM-x.pt", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_shape_filter', 'image_shape_filter', '过滤图像尺寸在指定范围内的样本', '1.4.4', 'image', 'image', NULL, '{"min_width": {"name": "min_width", "description": "最小宽度", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "max_width": {"name": "max_width", "description": "最大宽度", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "min_height": {"name": "min_height", "description": "最小高度", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "max_height": {"name": "max_height", "description": "最大高度", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_size_filter', 'image_size_filter', '过滤图像文件大小在指定范围内的样本', '1.4.4', 'image', 'image', NULL, '{"min_size": {"name": "min_size", "description": "最小大小", "type": "input", "defaultVal": "0", "required": false}, "max_size": {"name": "max_size", "description": "最大大小", "type": "input", "defaultVal": "1TB", "required": false}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_tagging_mapper', 'image_tagging_mapper', '为图像生成标签', '1.4.4', 'image', 'image', NULL, '{"tag_field_name": {"name": "tag_field_name", "description": "标签字段名", "type": "input", "defaultVal": "image_tags", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_text_matching_filter', 'image_text_matching_filter', '根据图文匹配分数过滤图像文本对', '1.4.4', 'image', 'image', NULL, '{"hf_blip": {"name": "hf_blip", "description": "BLIP模型路径", "type": "input", "defaultVal": "Salesforce/blip-itm-base-coco", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0.003, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 1.0, "required": false, "min": 0, "max": 10000}, "horizontal_flip": {"name": "horizontal_flip", "description": "参数: horizontal_flip", "type": "switch", "defaultVal": false, "required": false}, "vertical_flip": {"name": "vertical_flip", "description": "参数: vertical_flip", "type": "switch", "defaultVal": false, "required": false}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "reduce_mode": {"name": "reduce_mode", "description": "参数: reduce_mode", "type": "input", "defaultVal": "avg", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_text_similarity_filter', 'image_text_similarity_filter', '根据图文相似度分数过滤图像文本对', '1.4.4', 'image', 'image', NULL, '{"hf_clip": {"name": "hf_clip", "description": "CLIP模型路径", "type": "input", "defaultVal": "openai/clip-vit-base-patch32", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0.1, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 1.0, "required": false, "min": 0, "max": 10000}, "horizontal_flip": {"name": "horizontal_flip", "description": "参数: horizontal_flip", "type": "switch", "defaultVal": false, "required": false}, "vertical_flip": {"name": "vertical_flip", "description": "参数: vertical_flip", "type": "switch", "defaultVal": false, "required": false}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "reduce_mode": {"name": "reduce_mode", "description": "参数: reduce_mode", "type": "input", "defaultVal": "avg", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('image_watermark_filter', 'image_watermark_filter', '过滤含有水印的图像样本', '1.4.4', 'image', 'image', NULL, '{"hf_watermark_model": {"name": "hf_watermark_model", "description": "水印检测模型路径", "type": "input", "defaultVal": "amrul-hzz/watermark_detector", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "prob_threshold": {"name": "prob_threshold", "description": "参数: prob_threshold", "type": "inputNumber", "defaultVal": 0.8, "required": false, "min": 0, "max": 10000}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('imgdiff_difference_area_generator_mapper', 'imgdiff_difference_area_generator_mapper', '生成图像对之间的差异区域', '1.4.4', 'image', 'image', NULL, '{"image_pair_similarity_filter_args": {"name": "image_pair_similarity_filter_args", "description": "参数: image_pair_similarity_filter_args", "type": "input", "defaultVal": "{}", "required": false}, "image_segment_mapper_args": {"name": "image_segment_mapper_args", "description": "参数: image_segment_mapper_args", "type": "input", "defaultVal": "{}", "required": false}, "image_text_matching_filter_args": {"name": "image_text_matching_filter_args", "description": "图文匹配过滤参数(JSON)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('imgdiff_difference_caption_generator_mapper', 'imgdiff_difference_caption_generator_mapper', '为图像差异区域生成描述标题', '1.4.4', 'image', 'image', NULL, '{"mllm_mapper_args": {"name": "mllm_mapper_args", "description": "多模态大模型参数(JSON)", "type": "input", "defaultVal": "{}", "required": false}, "image_text_matching_filter_args": {"name": "image_text_matching_filter_args", "description": "图文匹配过滤参数(JSON)", "type": "input", "defaultVal": "{}", "required": false}, "text_pair_similarity_filter_args": {"name": "text_pair_similarity_filter_args", "description": "参数: text_pair_similarity_filter_args", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('in_context_influence_filter', 'in_context_influence_filter', '根据文本在上下文中对验证集的影响保留样本', '1.4.4', 'text', 'text', NULL, '{"valid_dataset": {"name": "valid_dataset", "description": "参数: valid_dataset", "type": "input", "defaultVal": "None", "required": false}, "task_desc": {"name": "task_desc", "description": "参数: task_desc", "type": "input", "defaultVal": "", "required": false}, "valid_as_demo": {"name": "valid_as_demo", "description": "参数: valid_as_demo", "type": "switch", "defaultVal": false, "required": false}, "n_shot": {"name": "n_shot", "description": "参数: n_shot", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('instruction_following_difficulty_filter', 'instruction_following_difficulty_filter', '根据指令跟随难度分数过滤文本', '1.4.4', 'text', 'text', NULL, '{"hf_model": {"name": "hf_model", "description": "HuggingFace模型路径", "type": "input", "defaultVal": "Qwen/Qwen2.5-0.5B", "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "None", "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 1.0, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 100.0, "required": false, "min": 0, "max": 10000}, "query_template": {"name": "query_template", "description": "查询模板", "type": "input", "defaultVal": "", "required": false}, "response_template": {"name": "response_template", "description": "响应模板", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('key_value_grouper', 'key_value_grouper', '按键值对分组数据', '1.4.4', 'text', 'text', NULL, '{"group_by_keys": {"name": "group_by_keys", "description": "分组键列表", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('language_id_score_filter', 'language_id_score_filter', '过滤特定语言且置信度高于阈值的样本', '1.4.4', 'text', 'text', NULL, '{"lang": {"name": "lang", "description": "目标语言", "type": "input", "defaultVal": "", "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0.8, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('llm_analysis_filter', 'llm_analysis_filter', '利用LLMs分析和过滤数据样本的基本筛选器', '1.4.4', 'text', 'text', NULL, '{"api_or_hf_model": {"name": "api_or_hf_model", "description": "API或HuggingFace模型", "type": "input", "defaultVal": "gpt-4o", "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0.5, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 1.0, "required": false, "min": 0, "max": 10000}, "is_hf_model": {"name": "is_hf_model", "description": "参数: is_hf_model", "type": "switch", "defaultVal": false, "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "input_keys": {"name": "input_keys", "description": "参数: input_keys", "type": "input", "defaultVal": "[''text'']", "required": false}, "field_names": {"name": "field_names", "description": "参数: field_names", "type": "input", "defaultVal": "[''Text'']", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "field_template": {"name": "field_template", "description": "参数: field_template", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "enable_vllm": {"name": "enable_vllm", "description": "参数: enable_vllm", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "dim_required_keys": {"name": "dim_required_keys", "description": "参数: dim_required_keys", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('llm_difficulty_score_filter', 'llm_difficulty_score_filter', '过滤具有LLM估计高难度分数的样本', '1.4.4', 'text', 'text', NULL, '{"api_or_hf_model": {"name": "api_or_hf_model", "description": "API或HuggingFace模型", "type": "input", "defaultVal": "gpt-4o", "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0.5, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 1.0, "required": false, "min": 0, "max": 10000}, "is_hf_model": {"name": "is_hf_model", "description": "参数: is_hf_model", "type": "switch", "defaultVal": false, "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "input_keys": {"name": "input_keys", "description": "参数: input_keys", "type": "input", "defaultVal": "[''text'']", "required": false}, "field_names": {"name": "field_names", "description": "参数: field_names", "type": "input", "defaultVal": "[''Text'']", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "field_template": {"name": "field_template", "description": "参数: field_template", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "enable_vllm": {"name": "enable_vllm", "description": "参数: enable_vllm", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "dim_required_keys": {"name": "dim_required_keys", "description": "参数: dim_required_keys", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('llm_perplexity_filter', 'llm_perplexity_filter', '使用指定LLM计算困惑度并过滤', '1.4.4', 'text', 'text', NULL, '{"hf_model": {"name": "hf_model", "description": "HuggingFace模型路径", "type": "input", "defaultVal": "Qwen/Qwen2.5-0.5B", "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "None", "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 1.0, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 100.0, "required": false, "min": 0, "max": 10000}, "query_template": {"name": "query_template", "description": "查询模板", "type": "input", "defaultVal": "", "required": false}, "response_template": {"name": "response_template", "description": "响应模板", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('llm_quality_score_filter', 'llm_quality_score_filter', '过滤具有LLM估计高质量分数的样本', '1.4.4', 'text', 'text', NULL, '{"api_or_hf_model": {"name": "api_or_hf_model", "description": "API或HuggingFace模型", "type": "input", "defaultVal": "gpt-4o", "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0.5, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 1.0, "required": false, "min": 0, "max": 10000}, "is_hf_model": {"name": "is_hf_model", "description": "参数: is_hf_model", "type": "switch", "defaultVal": false, "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "input_keys": {"name": "input_keys", "description": "参数: input_keys", "type": "input", "defaultVal": "[''text'']", "required": false}, "field_names": {"name": "field_names", "description": "参数: field_names", "type": "input", "defaultVal": "[''Text'']", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "field_template": {"name": "field_template", "description": "参数: field_template", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "enable_vllm": {"name": "enable_vllm", "description": "参数: enable_vllm", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "dim_required_keys": {"name": "dim_required_keys", "description": "参数: dim_required_keys", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('llm_task_relevance_filter', 'llm_task_relevance_filter', '过滤与验证任务高相关性分数的样本', '1.4.4', 'text', 'text', NULL, '{"api_or_hf_model": {"name": "api_or_hf_model", "description": "API或HuggingFace模型", "type": "input", "defaultVal": "gpt-4o", "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0.5, "required": false, "min": 0, "max": 10000}, "is_hf_model": {"name": "is_hf_model", "description": "参数: is_hf_model", "type": "switch", "defaultVal": false, "required": false}, "valid_dataset": {"name": "valid_dataset", "description": "参数: valid_dataset", "type": "input", "defaultVal": "None", "required": false}, "task_desc": {"name": "task_desc", "description": "参数: task_desc", "type": "input", "defaultVal": "", "required": false}, "n_shot": {"name": "n_shot", "description": "参数: n_shot", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('maximum_line_length_filter', 'maximum_line_length_filter', '过滤最大行长在指定范围内的样本', '1.4.4', 'text', 'text', NULL, '{"min_len": {"name": "min_len", "description": "最小长度", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "max_len": {"name": "max_len", "description": "最大长度", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('meta_tags_aggregator', 'meta_tags_aggregator', '将类似的元标记合并到统一的标记中', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "meta_tag_key": {"name": "meta_tag_key", "description": "参数: meta_tag_key", "type": "input", "defaultVal": "dialog_sentiment_labels", "required": false}, "target_tags": {"name": "target_tags", "description": "参数: target_tags", "type": "input", "defaultVal": "", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "target_tag_template": {"name": "target_tag_template", "description": "参数: target_tag_template", "type": "input", "defaultVal": "", "required": false}, "tag_template": {"name": "tag_template", "description": "参数: tag_template", "type": "input", "defaultVal": "", "required": false}, "output_pattern": {"name": "output_pattern", "description": "输出匹配模式", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('mllm_mapper', 'mllm_mapper', '使用多模态大模型进行处理', '1.4.4', 'text', 'text', NULL, '{"hf_model": {"name": "hf_model", "description": "HuggingFace模型路径", "type": "input", "defaultVal": "llava-hf/llava-v1.6-vicuna-7b-hf", "required": false}, "max_new_tokens": {"name": "max_new_tokens", "description": "最大生成Token数", "type": "input", "defaultVal": "256", "required": false}, "temperature": {"name": "temperature", "description": "温度参数(控制创造性)", "type": "input", "defaultVal": "0.2", "required": false}, "top_p": {"name": "top_p", "description": "Top-p采样参数", "type": "input", "defaultVal": "", "required": false}, "num_beams": {"name": "num_beams", "description": "Beam Search束宽", "type": "input", "defaultVal": "1", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('most_relevant_entities_aggregator', 'most_relevant_entities_aggregator', '从文本中提取并排名与给定实体密切相关的实体', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "entity": {"name": "entity", "description": "参数: entity", "type": "input", "defaultVal": "", "required": false}, "query_entity_type": {"name": "query_entity_type", "description": "参数: query_entity_type", "type": "input", "defaultVal": "", "required": false}, "input_key": {"name": "input_key", "description": "参数: input_key", "type": "input", "defaultVal": "event_description", "required": false}, "output_key": {"name": "output_key", "description": "参数: output_key", "type": "input", "defaultVal": "most_relevant_entities", "required": false}, "max_token_num": {"name": "max_token_num", "description": "参数: max_token_num", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt_template": {"name": "system_prompt_template", "description": "参数: system_prompt_template", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "output_pattern": {"name": "output_pattern", "description": "输出匹配模式", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('naive_grouper', 'naive_grouper', '简单分组算子', '1.4.4', 'text', 'text', NULL, '{"video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('naive_reverse_grouper', 'naive_reverse_grouper', '简单反向分组算子', '1.4.4', 'text', 'text', NULL, '{"batch_meta_export_path": {"name": "batch_meta_export_path", "description": "批量元数据导出路径", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('nested_aggregator', 'nested_aggregator', '将多个示例中的嵌套内容聚合到单个摘要', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "input_key": {"name": "input_key", "description": "参数: input_key", "type": "input", "defaultVal": "event_description", "required": false}, "output_key": {"name": "output_key", "description": "参数: output_key", "type": "input", "defaultVal": "", "required": false}, "max_token_num": {"name": "max_token_num", "description": "参数: max_token_num", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "sub_doc_template": {"name": "sub_doc_template", "description": "参数: sub_doc_template", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('nlpaug_en_mapper', 'nlpaug_en_mapper', '使用nlpaug进行英文文本增强', '1.4.4', 'text', 'text', NULL, '{"sequential": {"name": "sequential", "description": "是否顺序执行", "type": "switch", "defaultVal": false, "required": false}, "aug_num": {"name": "aug_num", "description": "参数: aug_num", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "keep_original_sample": {"name": "keep_original_sample", "description": "是否保留原始样本", "type": "switch", "defaultVal": true, "required": false}, "delete_random_word": {"name": "delete_random_word", "description": "参数: delete_random_word", "type": "switch", "defaultVal": false, "required": false}, "swap_random_word": {"name": "swap_random_word", "description": "参数: swap_random_word", "type": "switch", "defaultVal": false, "required": false}, "spelling_error_word": {"name": "spelling_error_word", "description": "参数: spelling_error_word", "type": "switch", "defaultVal": false, "required": false}, "split_random_word": {"name": "split_random_word", "description": "参数: split_random_word", "type": "switch", "defaultVal": false, "required": false}, "keyboard_error_char": {"name": "keyboard_error_char", "description": "参数: keyboard_error_char", "type": "switch", "defaultVal": false, "required": false}, "ocr_error_char": {"name": "ocr_error_char", "description": "参数: ocr_error_char", "type": "switch", "defaultVal": false, "required": false}, "delete_random_char": {"name": "delete_random_char", "description": "参数: delete_random_char", "type": "switch", "defaultVal": false, "required": false}, "swap_random_char": {"name": "swap_random_char", "description": "参数: swap_random_char", "type": "switch", "defaultVal": false, "required": false}, "insert_random_char": {"name": "insert_random_char", "description": "参数: insert_random_char", "type": "switch", "defaultVal": false, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('nlpcda_zh_mapper', 'nlpcda_zh_mapper', '使用nlpcda进行中文文本增强', '1.4.4', 'text', 'text', NULL, '{"sequential": {"name": "sequential", "description": "是否顺序执行", "type": "switch", "defaultVal": false, "required": false}, "aug_num": {"name": "aug_num", "description": "参数: aug_num", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "keep_original_sample": {"name": "keep_original_sample", "description": "是否保留原始样本", "type": "switch", "defaultVal": true, "required": false}, "replace_similar_word": {"name": "replace_similar_word", "description": "参数: replace_similar_word", "type": "switch", "defaultVal": false, "required": false}, "replace_homophone_char": {"name": "replace_homophone_char", "description": "参数: replace_homophone_char", "type": "switch", "defaultVal": false, "required": false}, "delete_random_char": {"name": "delete_random_char", "description": "参数: delete_random_char", "type": "switch", "defaultVal": false, "required": false}, "swap_random_char": {"name": "swap_random_char", "description": "参数: swap_random_char", "type": "switch", "defaultVal": false, "required": false}, "replace_equivalent_num": {"name": "replace_equivalent_num", "description": "参数: replace_equivalent_num", "type": "switch", "defaultVal": false, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('optimize_prompt_mapper', 'optimize_prompt_mapper', '使用LLM优化提示词', '1.4.4', 'text', 'text', NULL, '{"api_or_hf_model": {"name": "api_or_hf_model", "description": "API或HuggingFace模型", "type": "input", "defaultVal": "Qwen/Qwen2.5-7B-Instruct", "required": false}, "gen_num": {"name": "gen_num", "description": "参数: gen_num", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "max_example_num": {"name": "max_example_num", "description": "参数: max_example_num", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "keep_original_sample": {"name": "keep_original_sample", "description": "是否保留原始样本", "type": "switch", "defaultVal": true, "required": false}, "retry_num": {"name": "retry_num", "description": "参数: retry_num", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "example_template": {"name": "example_template", "description": "参数: example_template", "type": "input", "defaultVal": "", "required": false}, "prompt_template": {"name": "prompt_template", "description": "参数: prompt_template", "type": "input", "defaultVal": "", "required": false}, "output_pattern": {"name": "output_pattern", "description": "输出匹配模式", "type": "input", "defaultVal": "", "required": false}, "enable_vllm": {"name": "enable_vllm", "description": "参数: enable_vllm", "type": "switch", "defaultVal": false, "required": false}, "is_hf_model": {"name": "is_hf_model", "description": "参数: is_hf_model", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "None", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "None", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('optimize_qa_mapper', 'optimize_qa_mapper', '使用LLM优化QA对', '1.4.4', 'text', 'text', NULL, '{"api_or_hf_model": {"name": "api_or_hf_model", "description": "API或HuggingFace模型", "type": "input", "defaultVal": "Qwen/Qwen2.5-7B-Instruct", "required": false}, "is_hf_model": {"name": "is_hf_model", "description": "参数: is_hf_model", "type": "switch", "defaultVal": true, "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "qa_pair_template": {"name": "qa_pair_template", "description": "问答对模板", "type": "input", "defaultVal": "", "required": false}, "output_pattern": {"name": "output_pattern", "description": "输出匹配模式", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "enable_vllm": {"name": "enable_vllm", "description": "参数: enable_vllm", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "None", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "None", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('optimize_query_mapper', 'optimize_query_mapper', '使用LLM优化查询', '1.4.4', 'text', 'text', NULL, '{"api_or_hf_model": {"name": "api_or_hf_model", "description": "API或HuggingFace模型", "type": "input", "defaultVal": "Qwen/Qwen2.5-7B-Instruct", "required": false}, "is_hf_model": {"name": "is_hf_model", "description": "参数: is_hf_model", "type": "switch", "defaultVal": true, "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "qa_pair_template": {"name": "qa_pair_template", "description": "问答对模板", "type": "input", "defaultVal": "", "required": false}, "output_pattern": {"name": "output_pattern", "description": "输出匹配模式", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "enable_vllm": {"name": "enable_vllm", "description": "参数: enable_vllm", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "None", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "None", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('optimize_response_mapper', 'optimize_response_mapper', '使用LLM优化回答', '1.4.4', 'text', 'text', NULL, '{"api_or_hf_model": {"name": "api_or_hf_model", "description": "API或HuggingFace模型", "type": "input", "defaultVal": "Qwen/Qwen2.5-7B-Instruct", "required": false}, "is_hf_model": {"name": "is_hf_model", "description": "参数: is_hf_model", "type": "switch", "defaultVal": true, "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "qa_pair_template": {"name": "qa_pair_template", "description": "问答对模板", "type": "input", "defaultVal": "", "required": false}, "output_pattern": {"name": "output_pattern", "description": "输出匹配模式", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "enable_vllm": {"name": "enable_vllm", "description": "参数: enable_vllm", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "None", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "None", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('pair_preference_mapper', 'pair_preference_mapper', '使用API模型进行配对偏好标注', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "output_pattern": {"name": "output_pattern", "description": "输出匹配模式", "type": "input", "defaultVal": "", "required": false}, "rejected_key": {"name": "rejected_key", "description": "参数: rejected_key", "type": "input", "defaultVal": "rejected_response", "required": false}, "reason_key": {"name": "reason_key", "description": "参数: reason_key", "type": "input", "defaultVal": "reason", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('perplexity_filter', 'perplexity_filter', '过滤困惑度在指定范围内的样本', '1.4.4', 'text', 'text', NULL, '{"lang": {"name": "lang", "description": "目标语言", "type": "input", "defaultVal": "en", "required": false}, "min_ppl": {"name": "min_ppl", "description": "参数: min_ppl", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "max_ppl": {"name": "max_ppl", "description": "参数: max_ppl", "type": "inputNumber", "defaultVal": 1500.0, "required": false, "min": 0, "max": 15000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('phrase_grounding_recall_filter', 'phrase_grounding_recall_filter', '基于短语定位召回率过滤图像文本对', '1.4.4', 'text', 'text', NULL, '{"hf_owlvit": {"name": "hf_owlvit", "description": "OWL-ViT检测模型路径", "type": "input", "defaultVal": "google/owlvit-base-patch32", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "min_recall": {"name": "min_recall", "description": "参数: min_recall", "type": "inputNumber", "defaultVal": 0.1, "required": false, "min": 0, "max": 10000}, "max_recall": {"name": "max_recall", "description": "参数: max_recall", "type": "inputNumber", "defaultVal": 1.0, "required": false, "min": 0, "max": 10000}, "horizontal_flip": {"name": "horizontal_flip", "description": "参数: horizontal_flip", "type": "switch", "defaultVal": false, "required": false}, "vertical_flip": {"name": "vertical_flip", "description": "参数: vertical_flip", "type": "switch", "defaultVal": false, "required": false}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "reduce_mode": {"name": "reduce_mode", "description": "参数: reduce_mode", "type": "input", "defaultVal": "avg", "required": false}, "iou_thr": {"name": "iou_thr", "description": "参数: iou_thr", "type": "inputNumber", "defaultVal": 0.5, "required": false, "min": 0, "max": 10000}, "large_area_ratio_thr": {"name": "large_area_ratio_thr", "description": "参数: large_area_ratio_thr", "type": "inputNumber", "defaultVal": 0.95, "required": false, "min": 0, "max": 10000}, "conf_thr": {"name": "conf_thr", "description": "参数: conf_thr", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('punctuation_normalization_mapper', 'punctuation_normalization_mapper', '', '1.4.4', 'text', 'text', NULL, '{"video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('python_file_mapper', 'python_file_mapper', '执行Python文件处理数据', '1.4.4', 'text', 'text', NULL, '{"file_path": {"name": "file_path", "description": "Python文件路径", "type": "input", "defaultVal": "", "required": false}, "function_name": {"name": "function_name", "description": "参数: function_name", "type": "input", "defaultVal": "process_single", "required": false}, "batched": {"name": "batched", "description": "参数: batched", "type": "switch", "defaultVal": false, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('python_lambda_mapper', 'python_lambda_mapper', '执行Python Lambda表达式处理数据', '1.4.4', 'text', 'text', NULL, '{"lambda_str": {"name": "lambda_str", "description": "Lambda表达式字符串", "type": "input", "defaultVal": "", "required": false}, "batched": {"name": "batched", "description": "参数: batched", "type": "switch", "defaultVal": false, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('query_intent_detection_mapper', 'query_intent_detection_mapper', '使用HF模型检测查询意图', '1.4.4', 'text', 'text', NULL, '{"hf_model": {"name": "hf_model", "description": "HuggingFace模型路径", "type": "input", "defaultVal": "bespin-global/klue-roberta-small-3i4k-intent-classification", "required": false}, "zh_to_en_hf_model": {"name": "zh_to_en_hf_model", "description": "参数: zh_to_en_hf_model", "type": "input", "defaultVal": "Helsinki-NLP/opus-mt-zh-en", "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "zh_to_en_model_params": {"name": "zh_to_en_model_params", "description": "参数: zh_to_en_model_params", "type": "input", "defaultVal": "{}", "required": false}, "label_key": {"name": "label_key", "description": "参数: label_key", "type": "input", "defaultVal": "query_intent_label", "required": false}, "score_key": {"name": "score_key", "description": "参数: score_key", "type": "input", "defaultVal": "query_intent_label_score", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('query_sentiment_detection_mapper', 'query_sentiment_detection_mapper', '使用HF模型检测查询情感', '1.4.4', 'text', 'text', NULL, '{"hf_model": {"name": "hf_model", "description": "HuggingFace模型路径", "type": "input", "defaultVal": "mrm8488/distilroberta-finetuned-financial-news-sentiment-analysis", "required": false}, "zh_to_en_hf_model": {"name": "zh_to_en_hf_model", "description": "参数: zh_to_en_hf_model", "type": "input", "defaultVal": "Helsinki-NLP/opus-mt-zh-en", "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "zh_to_en_model_params": {"name": "zh_to_en_model_params", "description": "参数: zh_to_en_model_params", "type": "input", "defaultVal": "{}", "required": false}, "label_key": {"name": "label_key", "description": "参数: label_key", "type": "input", "defaultVal": "query_sentiment_label", "required": false}, "score_key": {"name": "score_key", "description": "参数: score_key", "type": "input", "defaultVal": "query_sentiment_label_score", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('query_topic_detection_mapper', 'query_topic_detection_mapper', '使用HF模型检测查询主题', '1.4.4', 'text', 'text', NULL, '{"hf_model": {"name": "hf_model", "description": "HuggingFace模型路径", "type": "input", "defaultVal": "dstefa/roberta-base_topic_classification_nyt_news", "required": false}, "zh_to_en_hf_model": {"name": "zh_to_en_hf_model", "description": "参数: zh_to_en_hf_model", "type": "input", "defaultVal": "Helsinki-NLP/opus-mt-zh-en", "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "zh_to_en_model_params": {"name": "zh_to_en_model_params", "description": "参数: zh_to_en_model_params", "type": "input", "defaultVal": "{}", "required": false}, "label_key": {"name": "label_key", "description": "参数: label_key", "type": "input", "defaultVal": "query_topic_label", "required": false}, "score_key": {"name": "score_key", "description": "参数: score_key", "type": "input", "defaultVal": "query_topic_label_score", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('random_selector', 'random_selector', '随机选择样本', '1.4.4', 'text', 'text', NULL, '{"select_ratio": {"name": "select_ratio", "description": "参数: select_ratio", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "select_num": {"name": "select_num", "description": "参数: select_num", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('range_specified_field_selector', 'range_specified_field_selector', '根据指定数值字段范围选择样本', '1.4.4', 'text', 'text', NULL, '{"field_key": {"name": "field_key", "description": "参数: field_key", "type": "input", "defaultVal": "", "required": false}, "lower_percentile": {"name": "lower_percentile", "description": "参数: lower_percentile", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "upper_percentile": {"name": "upper_percentile", "description": "参数: upper_percentile", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "lower_rank": {"name": "lower_rank", "description": "参数: lower_rank", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "upper_rank": {"name": "upper_rank", "description": "参数: upper_rank", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('ray_bts_minhash_deduplicator', 'ray_bts_minhash_deduplicator', 'Ray分布式模式下的MinHash LSH去重器', '1.4.4', 'text', 'text', NULL, '{"tokenization": {"name": "tokenization", "description": "是否使用分词", "type": "input", "defaultVal": "space", "required": false}, "window_size": {"name": "window_size", "description": "滑动窗口大小", "type": "inputNumber", "defaultVal": 5, "required": false, "min": 0, "max": 10000}, "lowercase": {"name": "lowercase", "description": "是否转换为小写", "type": "switch", "defaultVal": true, "required": false}, "ignore_pattern": {"name": "ignore_pattern", "description": "忽略模式(正则)", "type": "input", "defaultVal": "", "required": false}, "num_permutations": {"name": "num_permutations", "description": "排列函数数量", "type": "inputNumber", "defaultVal": 256, "required": false, "min": 0, "max": 10000}, "jaccard_threshold": {"name": "jaccard_threshold", "description": "Jaccard相似度阈值", "type": "inputNumber", "defaultVal": 0.7, "required": false, "min": 0, "max": 10000}, "num_bands": {"name": "num_bands", "description": "LSH分桶数量", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "num_rows_per_band": {"name": "num_rows_per_band", "description": "每桶行数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "tokenizer_model": {"name": "tokenizer_model", "description": "分词器模型路径", "type": "input", "defaultVal": "", "required": false}, "union_find_parallel_num": {"name": "union_find_parallel_num", "description": "参数: union_find_parallel_num", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "union_threshold": {"name": "union_threshold", "description": "参数: union_threshold", "type": "inputNumber", "defaultVal": 256, "required": false, "min": 0, "max": 10000}, "max_pending_edge_buffer_task": {"name": "max_pending_edge_buffer_task", "description": "参数: max_pending_edge_buffer_task", "type": "inputNumber", "defaultVal": 20, "required": false, "min": 0, "max": 10000}, "num_edge_buffer_task_returns": {"name": "num_edge_buffer_task_returns", "description": "参数: num_edge_buffer_task_returns", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "max_pending_filter_tasks": {"name": "max_pending_filter_tasks", "description": "参数: max_pending_filter_tasks", "type": "inputNumber", "defaultVal": 20, "required": false, "min": 0, "max": 10000}, "num_filter_task_returns": {"name": "num_filter_task_returns", "description": "参数: num_filter_task_returns", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "merge_batch_size": {"name": "merge_batch_size", "description": "参数: merge_batch_size", "type": "inputNumber", "defaultVal": 1000, "required": false, "min": 0, "max": 10000}, "minhash_batch_size": {"name": "minhash_batch_size", "description": "参数: minhash_batch_size", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "memory_per_sample": {"name": "memory_per_sample", "description": "参数: memory_per_sample", "type": "inputNumber", "defaultVal": 0.1, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('ray_document_deduplicator', 'ray_document_deduplicator', 'Ray分布式模式下的精确匹配文档去重器', '1.4.4', 'text', 'text', NULL, '{"backend": {"name": "backend", "description": "参数: backend", "type": "input", "defaultVal": "ray_actor", "required": false}, "redis_address": {"name": "redis_address", "description": "参数: redis_address", "type": "input", "defaultVal": "redis://localhost:6379", "required": false}, "lowercase": {"name": "lowercase", "description": "是否转换为小写", "type": "switch", "defaultVal": false, "required": false}, "ignore_non_character": {"name": "ignore_non_character", "description": "是否忽略非字符", "type": "switch", "defaultVal": false, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('ray_image_deduplicator', 'ray_image_deduplicator', 'Ray分布式模式下的图像去重器', '1.4.4', 'image', 'image', NULL, '{"backend": {"name": "backend", "description": "参数: backend", "type": "input", "defaultVal": "ray_actor", "required": false}, "redis_address": {"name": "redis_address", "description": "参数: redis_address", "type": "input", "defaultVal": "redis://localhost:6379", "required": false}, "method": {"name": "method", "description": "参数: method", "type": "input", "defaultVal": "phash", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('ray_video_deduplicator', 'ray_video_deduplicator', 'Ray分布式模式下的视频去重器', '1.4.4', 'video', 'video', NULL, '{"backend": {"name": "backend", "description": "参数: backend", "type": "input", "defaultVal": "ray_actor", "required": false}, "redis_address": {"name": "redis_address", "description": "参数: redis_address", "type": "input", "defaultVal": "redis://localhost:6379", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('relation_identity_mapper', 'relation_identity_mapper', '使用LLM进行关系身份标注', '1.4.4', 'text', 'text', NULL, '{"api_model": {"name": "api_model", "description": "API模型名称", "type": "input", "defaultVal": "gpt-4o", "required": false}, "source_entity": {"name": "source_entity", "description": "参数: source_entity", "type": "input", "defaultVal": "", "required": false}, "target_entity": {"name": "target_entity", "description": "参数: target_entity", "type": "input", "defaultVal": "", "required": false}, "output_key": {"name": "output_key", "description": "参数: output_key", "type": "input", "defaultVal": "role_relation", "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "", "required": false}, "system_prompt_template": {"name": "system_prompt_template", "description": "参数: system_prompt_template", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "output_pattern_template": {"name": "output_pattern_template", "description": "参数: output_pattern_template", "type": "input", "defaultVal": "", "required": false}, "try_num": {"name": "try_num", "description": "最大重试次数", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "drop_text": {"name": "drop_text", "description": "参数: drop_text", "type": "switch", "defaultVal": false, "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "sampling_params": {"name": "sampling_params", "description": "采样参数(JSON格式)", "type": "input", "defaultVal": "{}", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('remove_bibliography_mapper', 'remove_bibliography_mapper', '删除文本末尾的参考文献', '1.4.4', 'text', 'text', NULL, '{"video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('remove_comments_mapper', 'remove_comments_mapper', '删除文本中的注释', '1.4.4', 'text', 'text', NULL, '{"doc_type": {"name": "doc_type", "description": "参数: doc_type", "type": "input", "defaultVal": "tex", "required": false}, "inline": {"name": "inline", "description": "参数: inline", "type": "switch", "defaultVal": true, "required": false}, "multiline": {"name": "multiline", "description": "参数: multiline", "type": "switch", "defaultVal": true, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('remove_header_mapper', 'remove_header_mapper', '删除文本头部信息', '1.4.4', 'text', 'text', NULL, '{"drop_no_head": {"name": "drop_no_head", "description": "参数: drop_no_head", "type": "switch", "defaultVal": true, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('remove_long_words_mapper', 'remove_long_words_mapper', '删除超过指定长度的单词', '1.4.4', 'text', 'text', NULL, '{"min_len": {"name": "min_len", "description": "最小长度", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "max_len": {"name": "max_len", "description": "最大长度", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('remove_non_chinese_character_mapper', 'remove_non_chinese_character_mapper', '删除非中文字符', '1.4.4', 'text', 'text', NULL, '{"keep_alphabet": {"name": "keep_alphabet", "description": "参数: keep_alphabet", "type": "switch", "defaultVal": true, "required": false}, "keep_number": {"name": "keep_number", "description": "参数: keep_number", "type": "switch", "defaultVal": true, "required": false}, "keep_punc": {"name": "keep_punc", "description": "参数: keep_punc", "type": "switch", "defaultVal": true, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('remove_repeat_sentences_mapper', 'remove_repeat_sentences_mapper', '删除重复的句子', '1.4.4', 'text', 'text', NULL, '{"lowercase": {"name": "lowercase", "description": "是否转换为小写", "type": "switch", "defaultVal": false, "required": false}, "ignore_special_character": {"name": "ignore_special_character", "description": "参数: ignore_special_character", "type": "switch", "defaultVal": true, "required": false}, "min_repeat_sentence_length": {"name": "min_repeat_sentence_length", "description": "参数: min_repeat_sentence_length", "type": "inputNumber", "defaultVal": 2, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('remove_specific_chars_mapper', 'remove_specific_chars_mapper', '删除指定的字符', '1.4.4', 'text', 'text', NULL, '{"chars_to_remove": {"name": "chars_to_remove", "description": "需要移除的字符列表", "type": "input", "defaultVal": "◆●■►▼▲▴∆▻▷❖♡□", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('remove_table_text_mapper', 'remove_table_text_mapper', '删除表格文本', '1.4.4', 'text', 'text', NULL, '{"min_col": {"name": "min_col", "description": "参数: min_col", "type": "inputNumber", "defaultVal": 2, "required": false, "min": 0, "max": 10000}, "max_col": {"name": "max_col", "description": "参数: max_col", "type": "inputNumber", "defaultVal": 20, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('remove_words_with_incorrect_substrings_mapper', 'remove_words_with_incorrect_substrings_mapper', '删除包含不正确子字符串的单词', '1.4.4', 'text', 'text', NULL, '{"lang": {"name": "lang", "description": "目标语言", "type": "input", "defaultVal": "en", "required": false}, "tokenization": {"name": "tokenization", "description": "是否使用分词", "type": "switch", "defaultVal": false, "required": false}, "substrings": {"name": "substrings", "description": "子字符串列表", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('replace_content_mapper', 'replace_content_mapper', '使用正则表达式替换文本内容', '1.4.4', 'text', 'text', NULL, '{"pattern": {"name": "pattern", "description": "正则表达式模式", "type": "input", "defaultVal": "", "required": false}, "repl": {"name": "repl", "description": "替换内容", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('sdxl_prompt2prompt_mapper', 'sdxl_prompt2prompt_mapper', '使用SDXL模型从提示词生成图像对', '1.4.4', 'text', 'text', NULL, '{"hf_diffusion": {"name": "hf_diffusion", "description": "Diffusion模型路径", "type": "input", "defaultVal": "stabilityai/stable-diffusion-xl-base-1.0", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "input", "defaultVal": "False", "required": false}, "torch_dtype": {"name": "torch_dtype", "description": "PyTorch数据类型", "type": "input", "defaultVal": "fp32", "required": false}, "num_inference_steps": {"name": "num_inference_steps", "description": "推理步数", "type": "inputNumber", "defaultVal": 50.0, "required": false, "min": 0, "max": 10000}, "guidance_scale": {"name": "guidance_scale", "description": "引导缩放系数", "type": "inputNumber", "defaultVal": 7.5, "required": false, "min": 0, "max": 10000}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "text_key_second": {"name": "text_key_second", "description": "第二文本字段名", "type": "input", "defaultVal": "", "required": false}, "output_dir": {"name": "output_dir", "description": "输出目录", "type": "input", "defaultVal": "C:\\Users\\Administrator/.cache\\data_juicer\\assets", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('sentence_augmentation_mapper', 'sentence_augmentation_mapper', '使用HF模型增强句子', '1.4.4', 'text', 'text', NULL, '{"hf_model": {"name": "hf_model", "description": "HuggingFace模型路径", "type": "input", "defaultVal": "Qwen/Qwen2-7B-Instruct", "required": false}, "system_prompt": {"name": "system_prompt", "description": "系统提示词", "type": "input", "defaultVal": "", "required": false}, "task_sentence": {"name": "task_sentence", "description": "任务描述句子", "type": "input", "defaultVal": "", "required": false}, "max_new_tokens": {"name": "max_new_tokens", "description": "最大生成Token数", "type": "input", "defaultVal": "256", "required": false}, "temperature": {"name": "temperature", "description": "温度参数(控制创造性)", "type": "input", "defaultVal": "0.2", "required": false}, "top_p": {"name": "top_p", "description": "Top-p采样参数", "type": "input", "defaultVal": "", "required": false}, "num_beams": {"name": "num_beams", "description": "Beam Search束宽", "type": "input", "defaultVal": "1", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "text_key_second": {"name": "text_key_second", "description": "第二文本字段名", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('sentence_split_mapper', 'sentence_split_mapper', '将文本分割为单独的句子', '1.4.4', 'text', 'text', NULL, '{"lang": {"name": "lang", "description": "目标语言", "type": "input", "defaultVal": "en", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('special_characters_filter', 'special_characters_filter', '过滤特殊字符比例在指定范围内的样本', '1.4.4', 'text', 'text', NULL, '{"min_ratio": {"name": "min_ratio", "description": "最小比例下限", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "max_ratio": {"name": "max_ratio", "description": "最大比例上限", "type": "inputNumber", "defaultVal": 0.25, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('specified_field_filter', 'specified_field_filter', '根据指定的字段信息筛选样本', '1.4.4', 'text', 'text', NULL, '{"field_key": {"name": "field_key", "description": "参数: field_key", "type": "input", "defaultVal": "", "required": false}, "target_value": {"name": "target_value", "description": "参数: target_value", "type": "input", "defaultVal": "[]", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('specified_numeric_field_filter', 'specified_numeric_field_filter', '根据指定的数值字段值筛选样本', '1.4.4', 'text', 'text', NULL, '{"field_key": {"name": "field_key", "description": "参数: field_key", "type": "input", "defaultVal": "", "required": false}, "min_value": {"name": "min_value", "description": "参数: min_value", "type": "inputNumber", "defaultVal": -9.223372036854776e+18, "required": false, "min": 0, "max": 10000}, "max_value": {"name": "max_value", "description": "参数: max_value", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('stopwords_filter', 'stopwords_filter', '过滤停用词比例在指定范围内的样本', '1.4.4', 'text', 'text', NULL, '{"lang": {"name": "lang", "description": "目标语言", "type": "input", "defaultVal": "en", "required": false}, "tokenization": {"name": "tokenization", "description": "是否使用分词", "type": "switch", "defaultVal": false, "required": false}, "min_ratio": {"name": "min_ratio", "description": "最小比例下限", "type": "inputNumber", "defaultVal": 0.3, "required": false, "min": 0, "max": 10000}, "max_ratio": {"name": "max_ratio", "description": "最大比例上限", "type": "inputNumber", "defaultVal": 1.0, "required": false, "min": 0, "max": 10000}, "stopwords_dir": {"name": "stopwords_dir", "description": "参数: stopwords_dir", "type": "input", "defaultVal": "C:\\Users\\Administrator/.cache\\data_juicer\\assets", "required": false}, "use_words_aug": {"name": "use_words_aug", "description": "参数: use_words_aug", "type": "switch", "defaultVal": false, "required": false}, "words_aug_group_sizes": {"name": "words_aug_group_sizes", "description": "参数: words_aug_group_sizes", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "words_aug_join_char": {"name": "words_aug_join_char", "description": "参数: words_aug_join_char", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('suffix_filter', 'suffix_filter', '过滤具有指定后缀的样本', '1.4.4', 'text', 'text', NULL, '{"suffixes": {"name": "suffixes", "description": "后缀列表", "type": "input", "defaultVal": "[]", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('tags_specified_field_selector', 'tags_specified_field_selector', '根据指定标签字段选择样本', '1.4.4', 'text', 'text', NULL, '{"field_key": {"name": "field_key", "description": "参数: field_key", "type": "input", "defaultVal": "", "required": false}, "target_tags": {"name": "target_tags", "description": "参数: target_tags", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('text_action_filter', 'text_action_filter', '过滤包含最少数量操作的文本', '1.4.4', 'text', 'text', NULL, '{"lang": {"name": "lang", "description": "目标语言", "type": "input", "defaultVal": "en", "required": false}, "min_action_num": {"name": "min_action_num", "description": "最小动作数量", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('text_chunk_mapper', 'text_chunk_mapper', '将文本切分为多个片段', '1.4.4', 'text', 'text', NULL, '{"max_len": {"name": "max_len", "description": "最大长度", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "split_pattern": {"name": "split_pattern", "description": "参数: split_pattern", "type": "input", "defaultVal": "\\n\\n", "required": false}, "overlap_len": {"name": "overlap_len", "description": "参数: overlap_len", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "tokenizer": {"name": "tokenizer", "description": "参数: tokenizer", "type": "input", "defaultVal": "", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('text_embd_similarity_filter', 'text_embd_similarity_filter', '过滤与验证文本嵌入相似度在指定范围内的文本', '1.4.4', 'text', 'text', NULL, '{"api_or_hf_model": {"name": "api_or_hf_model", "description": "API或HuggingFace模型", "type": "input", "defaultVal": "text-embedding-v4", "required": false}, "is_hf_model": {"name": "is_hf_model", "description": "参数: is_hf_model", "type": "switch", "defaultVal": false, "required": false}, "api_endpoint": {"name": "api_endpoint", "description": "API端点地址", "type": "input", "defaultVal": "embeddings", "required": false}, "response_path": {"name": "response_path", "description": "响应解析路径", "type": "input", "defaultVal": "data.0.embedding", "required": false}, "model_params": {"name": "model_params", "description": "模型额外参数(JSON格式)", "type": "input", "defaultVal": "None", "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "valid_dataset": {"name": "valid_dataset", "description": "参数: valid_dataset", "type": "input", "defaultVal": "None", "required": false}, "ebd_dim": {"name": "ebd_dim", "description": "参数: ebd_dim", "type": "inputNumber", "defaultVal": 4096, "required": false, "min": 0, "max": 40960}, "pooling": {"name": "pooling", "description": "参数: pooling", "type": "input", "defaultVal": "", "required": false}, "input_template": {"name": "input_template", "description": "输入模板", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('text_entity_dependency_filter', 'text_entity_dependency_filter', '根据实体依赖关系识别和过滤文本样本', '1.4.4', 'text', 'text', NULL, '{"lang": {"name": "lang", "description": "目标语言", "type": "input", "defaultVal": "en", "required": false}, "min_dependency_num": {"name": "min_dependency_num", "description": "参数: min_dependency_num", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "all", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('text_length_filter', 'text_length_filter', '过滤文本总长度在指定范围内的样本', '1.4.4', 'text', 'text', NULL, '{"min_len": {"name": "min_len", "description": "最小长度", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "max_len": {"name": "max_len", "description": "最大长度", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('text_pair_similarity_filter', 'text_pair_similarity_filter', '过滤文本对相似度在指定范围内的样本', '1.4.4', 'text', 'text', NULL, '{"hf_clip": {"name": "hf_clip", "description": "CLIP模型路径", "type": "input", "defaultVal": "openai/clip-vit-base-patch32", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "input", "defaultVal": "False", "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "text_key_second": {"name": "text_key_second", "description": "第二文本字段名", "type": "input", "defaultVal": "", "required": false}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('token_num_filter', 'token_num_filter', '过滤Token总数在指定范围内的样本', '1.4.4', 'text', 'text', NULL, '{"hf_tokenizer": {"name": "hf_tokenizer", "description": "HuggingFace分词器路径", "type": "input", "defaultVal": "EleutherAI/pythia-6.9b-deduped", "required": false}, "min_num": {"name": "min_num", "description": "参数: min_num", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "max_num": {"name": "max_num", "description": "参数: max_num", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('topk_specified_field_selector', 'topk_specified_field_selector', '根据指定字段选择Top-K样本', '1.4.4', 'text', 'text', NULL, '{"field_key": {"name": "field_key", "description": "参数: field_key", "type": "input", "defaultVal": "", "required": false}, "top_ratio": {"name": "top_ratio", "description": "参数: top_ratio", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "topk": {"name": "topk", "description": "参数: topk", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "reverse": {"name": "reverse", "description": "参数: reverse", "type": "switch", "defaultVal": true, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('vggt_mapper', 'vggt_mapper', '使用VGGT模型进行处理', '1.4.4', 'text', 'text', NULL, '{"vggt_model_path": {"name": "vggt_model_path", "description": "VGGT模型路径", "type": "input", "defaultVal": "facebook/VGGT-1B", "required": false}, "frame_num": {"name": "frame_num", "description": "帧数量", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "duration": {"name": "duration", "description": "参数: duration", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "tag_field_name": {"name": "tag_field_name", "description": "标签字段名", "type": "input", "defaultVal": "vggt_tags", "required": false}, "frame_dir": {"name": "frame_dir", "description": "参数: frame_dir", "type": "input", "defaultVal": "C:\\Users\\Administrator/.cache\\data_juicer\\assets", "required": false}, "if_output_camera_parameters": {"name": "if_output_camera_parameters", "description": "参数: if_output_camera_parameters", "type": "switch", "defaultVal": true, "required": false}, "if_output_depth_maps": {"name": "if_output_depth_maps", "description": "参数: if_output_depth_maps", "type": "switch", "defaultVal": true, "required": false}, "if_output_point_maps_from_projection": {"name": "if_output_point_maps_from_projection", "description": "参数: if_output_point_maps_from_projection", "type": "switch", "defaultVal": true, "required": false}, "if_output_point_maps_from_unprojection": {"name": "if_output_point_maps_from_unprojection", "description": "参数: if_output_point_maps_from_unprojection", "type": "switch", "defaultVal": true, "required": false}, "if_output_point_tracks": {"name": "if_output_point_tracks", "description": "参数: if_output_point_tracks", "type": "switch", "defaultVal": true, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_aesthetics_filter', 'video_aesthetics_filter', '过滤视频美学评分在指定范围内的样本', '1.4.4', 'video', 'video', NULL, '{"hf_scorer_model": {"name": "hf_scorer_model", "description": "评分模型路径", "type": "input", "defaultVal": "", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0.4, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 1.0, "required": false, "min": 0, "max": 10000}, "frame_sampling_method": {"name": "frame_sampling_method", "description": "参数: frame_sampling_method", "type": "input", "defaultVal": "uniform", "required": false}, "frame_num": {"name": "frame_num", "description": "帧数量", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "select", "defaultVal": "any", "required": false, "options": [{"label": "any", "value": "any"}, {"label": "all", "value": "all"}]}, "reduce_mode": {"name": "reduce_mode", "description": "参数: reduce_mode", "type": "select", "defaultVal": "avg", "required": false, "options": [{"label": "avg", "value": "avg"}, {"label": "max", "value": "max"}, {"label": "min", "value": "min"}]}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_aspect_ratio_filter', 'video_aspect_ratio_filter', '过滤视频长宽比在指定范围内的样本', '1.4.4', 'video', 'video', NULL, '{"min_ratio": {"name": "min_ratio", "description": "最小比例下限", "type": "input", "defaultVal": "9/21", "required": false}, "max_ratio": {"name": "max_ratio", "description": "最大比例上限", "type": "input", "defaultVal": "21/9", "required": false}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_captioning_from_audio_mapper', 'video_captioning_from_audio_mapper', '从视频音频生成字幕', '1.4.4', 'audio', 'audio', NULL, '{"keep_original_sample": {"name": "keep_original_sample", "description": "是否保留原始样本", "type": "switch", "defaultVal": true, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_captioning_from_frames_mapper', 'video_captioning_from_frames_mapper', '从视频帧生成字幕', '1.4.4', 'video', 'video', NULL, '{"hf_img2seq": {"name": "hf_img2seq", "description": "Image-to-Seq模型路径", "type": "input", "defaultVal": "Salesforce/blip2-opt-2.7b", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "caption_num": {"name": "caption_num", "description": "参数: caption_num", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "keep_candidate_mode": {"name": "keep_candidate_mode", "description": "参数: keep_candidate_mode", "type": "input", "defaultVal": "random_any", "required": false}, "keep_original_sample": {"name": "keep_original_sample", "description": "是否保留原始样本", "type": "switch", "defaultVal": true, "required": false}, "prompt": {"name": "prompt", "description": "提示词", "type": "input", "defaultVal": "", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "frame_sampling_method": {"name": "frame_sampling_method", "description": "参数: frame_sampling_method", "type": "input", "defaultVal": "all_keyframes", "required": false}, "frame_num": {"name": "frame_num", "description": "帧数量", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "horizontal_flip": {"name": "horizontal_flip", "description": "参数: horizontal_flip", "type": "switch", "defaultVal": false, "required": false}, "vertical_flip": {"name": "vertical_flip", "description": "参数: vertical_flip", "type": "switch", "defaultVal": false, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_captioning_from_summarizer_mapper', 'video_captioning_from_summarizer_mapper', '使用摘要模型从视频生成字幕', '1.4.4', 'video', 'video', NULL, '{"hf_summarizer": {"name": "hf_summarizer", "description": "摘要模型路径", "type": "input", "defaultVal": "", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "consider_video_caption_from_video": {"name": "consider_video_caption_from_video", "description": "参数: consider_video_caption_from_video", "type": "switch", "defaultVal": true, "required": false}, "consider_video_caption_from_audio": {"name": "consider_video_caption_from_audio", "description": "参数: consider_video_caption_from_audio", "type": "switch", "defaultVal": true, "required": false}, "consider_video_caption_from_frames": {"name": "consider_video_caption_from_frames", "description": "参数: consider_video_caption_from_frames", "type": "switch", "defaultVal": true, "required": false}, "consider_video_tags_from_audio": {"name": "consider_video_tags_from_audio", "description": "参数: consider_video_tags_from_audio", "type": "switch", "defaultVal": true, "required": false}, "consider_video_tags_from_frames": {"name": "consider_video_tags_from_frames", "description": "参数: consider_video_tags_from_frames", "type": "switch", "defaultVal": true, "required": false}, "vid_cap_from_vid_args": {"name": "vid_cap_from_vid_args", "description": "参数: vid_cap_from_vid_args", "type": "input", "defaultVal": "None", "required": false}, "vid_cap_from_frm_args": {"name": "vid_cap_from_frm_args", "description": "参数: vid_cap_from_frm_args", "type": "input", "defaultVal": "None", "required": false}, "vid_tag_from_aud_args": {"name": "vid_tag_from_aud_args", "description": "参数: vid_tag_from_aud_args", "type": "input", "defaultVal": "None", "required": false}, "vid_tag_from_frm_args": {"name": "vid_tag_from_frm_args", "description": "参数: vid_tag_from_frm_args", "type": "input", "defaultVal": "None", "required": false}, "keep_tag_num": {"name": "keep_tag_num", "description": "参数: keep_tag_num", "type": "inputNumber", "defaultVal": 5, "required": false, "min": 0, "max": 10000}, "keep_original_sample": {"name": "keep_original_sample", "description": "是否保留原始样本", "type": "switch", "defaultVal": true, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_captioning_from_video_mapper', 'video_captioning_from_video_mapper', '使用Video-BLIP模型从视频生成字幕', '1.4.4', 'video', 'video', NULL, '{"hf_video_blip": {"name": "hf_video_blip", "description": "Video-BLIP模型路径", "type": "input", "defaultVal": "kpyu/video-blip-opt-2.7b-ego4d", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "caption_num": {"name": "caption_num", "description": "参数: caption_num", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "keep_candidate_mode": {"name": "keep_candidate_mode", "description": "参数: keep_candidate_mode", "type": "input", "defaultVal": "random_any", "required": false}, "keep_original_sample": {"name": "keep_original_sample", "description": "是否保留原始样本", "type": "switch", "defaultVal": true, "required": false}, "prompt": {"name": "prompt", "description": "提示词", "type": "input", "defaultVal": "", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "frame_sampling_method": {"name": "frame_sampling_method", "description": "参数: frame_sampling_method", "type": "input", "defaultVal": "all_keyframes", "required": false}, "frame_num": {"name": "frame_num", "description": "帧数量", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "horizontal_flip": {"name": "horizontal_flip", "description": "参数: horizontal_flip", "type": "switch", "defaultVal": false, "required": false}, "vertical_flip": {"name": "vertical_flip", "description": "参数: vertical_flip", "type": "switch", "defaultVal": false, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_deduplicator', 'video_deduplicator', '通过视频精确匹配删除重复样本', '1.4.4', 'video', 'video', NULL, '{"consider_text": {"name": "consider_text", "description": "是否考虑文本内容", "type": "switch", "defaultVal": false, "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_duration_filter', 'video_duration_filter', '过滤视频持续时间在指定范围内的样本', '1.4.4', 'video', 'video', NULL, '{"min_duration": {"name": "min_duration", "description": "最小时长(秒)", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "max_duration": {"name": "max_duration", "description": "最大时长(秒)", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "select", "defaultVal": "any", "required": false, "options": [{"label": "any", "value": "any"}, {"label": "all", "value": "all"}]}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_extract_frames_mapper', 'video_extract_frames_mapper', '从视频中提取帧', '1.4.4', 'video', 'video', NULL, '{"frame_sampling_method": {"name": "frame_sampling_method", "description": "参数: frame_sampling_method", "type": "select", "defaultVal": "all_keyframes", "required": false, "options": [{"label": "all_keyframes", "value": "all_keyframes"}, {"label": "uniform", "value": "uniform"}]}, "output_format": {"name": "output_format", "description": "参数: output_format", "type": "select", "defaultVal": "path", "required": false, "options": [{"label": "path", "value": "path"}, {"label": "bytes", "value": "bytes"}]}, "frame_num": {"name": "frame_num", "description": "帧数量", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "duration": {"name": "duration", "description": "参数: duration", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "frame_dir": {"name": "frame_dir", "description": "参数: frame_dir", "type": "input", "defaultVal": "", "required": false}, "frame_key": {"name": "frame_key", "description": "参数: frame_key", "type": "input", "defaultVal": "video_frames", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_face_blur_mapper', 'video_face_blur_mapper', '对视频中的人脸进行模糊处理', '1.4.4', 'video', 'video', NULL, '{"cv_classifier": {"name": "cv_classifier", "description": "OpenCV分类器路径", "type": "input", "defaultVal": "", "required": false}, "blur_type": {"name": "blur_type", "description": "参数: blur_type", "type": "input", "defaultVal": "gaussian", "required": false}, "radius": {"name": "radius", "description": "参数: radius", "type": "inputNumber", "defaultVal": 2.0, "required": false, "min": 0, "max": 10000}, "save_dir": {"name": "save_dir", "description": "保存目录", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_ffmpeg_wrapped_mapper', 'video_ffmpeg_wrapped_mapper', '使用FFmpeg视频滤镜处理视频文件', '1.4.4', 'video', 'video', NULL, '{"filter_name": {"name": "filter_name", "description": "FFmpeg滤镜名称", "type": "input", "defaultVal": "", "required": false}, "filter_kwargs": {"name": "filter_kwargs", "description": "滤镜参数", "type": "input", "defaultVal": "None", "required": false}, "global_args": {"name": "global_args", "description": "全局参数", "type": "input", "defaultVal": "", "required": false}, "capture_stderr": {"name": "capture_stderr", "description": "是否捕获标准错误输出", "type": "switch", "defaultVal": true, "required": false}, "overwrite_output": {"name": "overwrite_output", "description": "是否覆盖输出文件", "type": "switch", "defaultVal": true, "required": false}, "save_dir": {"name": "save_dir", "description": "保存目录", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_frames_text_similarity_filter', 'video_frames_text_similarity_filter', '根据视频帧与文本相似度过滤视频', '1.4.4', 'video', 'video', NULL, '{"hf_clip": {"name": "hf_clip", "description": "CLIP模型路径", "type": "input", "defaultVal": "openai/clip-vit-base-patch32", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "input", "defaultVal": "False", "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0.1, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 1.0, "required": false, "min": 0, "max": 10000}, "frame_sampling_method": {"name": "frame_sampling_method", "description": "参数: frame_sampling_method", "type": "input", "defaultVal": "all_keyframes", "required": false}, "frame_num": {"name": "frame_num", "description": "帧数量", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "horizontal_flip": {"name": "horizontal_flip", "description": "参数: horizontal_flip", "type": "switch", "defaultVal": false, "required": false}, "vertical_flip": {"name": "vertical_flip", "description": "参数: vertical_flip", "type": "switch", "defaultVal": false, "required": false}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "select", "defaultVal": "any", "required": false, "options": [{"label": "any", "value": "any"}, {"label": "all", "value": "all"}]}, "reduce_mode": {"name": "reduce_mode", "description": "参数: reduce_mode", "type": "select", "defaultVal": "avg", "required": false, "options": [{"label": "avg", "value": "avg"}, {"label": "max", "value": "max"}, {"label": "min", "value": "min"}]}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_hand_reconstruction_mapper', 'video_hand_reconstruction_mapper', '视频手部重建算子', '1.4.4', 'video', 'video', NULL, '{"wilor_model_path": {"name": "wilor_model_path", "description": "WiLoR手部模型路径", "type": "input", "defaultVal": "wilor_final.ckpt", "required": false}, "wilor_model_config": {"name": "wilor_model_config", "description": "参数: wilor_model_config", "type": "input", "defaultVal": "model_config.yaml", "required": false}, "detector_model_path": {"name": "detector_model_path", "description": "参数: detector_model_path", "type": "input", "defaultVal": "detector.pt", "required": false}, "mano_right_path": {"name": "mano_right_path", "description": "参数: mano_right_path", "type": "input", "defaultVal": "path_to_mano_right_pkl", "required": false}, "frame_num": {"name": "frame_num", "description": "帧数量", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "duration": {"name": "duration", "description": "参数: duration", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "batch_size": {"name": "batch_size", "description": "参数: batch_size", "type": "inputNumber", "defaultVal": 16, "required": false, "min": 0, "max": 10000}, "tag_field_name": {"name": "tag_field_name", "description": "标签字段名", "type": "input", "defaultVal": "hand_reconstruction_tags", "required": false}, "frame_dir": {"name": "frame_dir", "description": "参数: frame_dir", "type": "input", "defaultVal": "C:\\Users\\Administrator/.cache\\data_juicer\\assets", "required": false}, "if_save_visualization": {"name": "if_save_visualization", "description": "参数: if_save_visualization", "type": "switch", "defaultVal": true, "required": false}, "save_visualization_dir": {"name": "save_visualization_dir", "description": "参数: save_visualization_dir", "type": "input", "defaultVal": "C:\\Users\\Administrator/.cache\\data_juicer\\assets", "required": false}, "if_save_mesh": {"name": "if_save_mesh", "description": "参数: if_save_mesh", "type": "switch", "defaultVal": true, "required": false}, "save_mesh_dir": {"name": "save_mesh_dir", "description": "参数: save_mesh_dir", "type": "input", "defaultVal": "C:\\Users\\Administrator/.cache\\data_juicer\\assets", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_motion_score_filter', 'video_motion_score_filter', '过滤视频运动分数在指定范围内的样本', '1.4.4', 'video', 'video', NULL, '{"min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0.25, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "sampling_fps": {"name": "sampling_fps", "description": "采样帧率(fps)", "type": "inputNumber", "defaultVal": 2.0, "required": false, "min": 0, "max": 10000}, "size": {"name": "size", "description": "尺寸", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "max_size": {"name": "max_size", "description": "最大大小", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "divisible": {"name": "divisible", "description": "整除数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "relative": {"name": "relative", "description": "是否使用相对值", "type": "switch", "defaultVal": false, "required": false}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_motion_score_raft_filter', 'video_motion_score_raft_filter', '使用RAFT算法过滤视频运动分数', '1.4.4', 'video', 'video', NULL, '{"min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 1.0, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "sampling_fps": {"name": "sampling_fps", "description": "采样帧率(fps)", "type": "inputNumber", "defaultVal": 2.0, "required": false, "min": 0, "max": 10000}, "size": {"name": "size", "description": "尺寸", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "max_size": {"name": "max_size", "description": "最大大小", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "divisible": {"name": "divisible", "description": "整除数", "type": "inputNumber", "defaultVal": 8, "required": false, "min": 0, "max": 10000}, "relative": {"name": "relative", "description": "是否使用相对值", "type": "switch", "defaultVal": false, "required": false}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_nsfw_filter', 'video_nsfw_filter', '过滤视频NSFW评分在指定范围内的样本', '1.4.4', 'video', 'video', NULL, '{"hf_nsfw_model": {"name": "hf_nsfw_model", "description": "NSFW检测模型路径", "type": "input", "defaultVal": "Falconsai/nsfw_image_detection", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "min_score": {"name": "min_score", "description": "最低评分阈值", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "max_score": {"name": "max_score", "description": "最高评分阈值", "type": "inputNumber", "defaultVal": 0.5, "required": false, "min": 0, "max": 10000}, "frame_sampling_method": {"name": "frame_sampling_method", "description": "参数: frame_sampling_method", "type": "input", "defaultVal": "all_keyframes", "required": false}, "frame_num": {"name": "frame_num", "description": "帧数量", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "reduce_mode": {"name": "reduce_mode", "description": "参数: reduce_mode", "type": "input", "defaultVal": "avg", "required": false}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_ocr_area_ratio_filter', 'video_ocr_area_ratio_filter', '过滤视频OCR区域占比在指定范围内的样本', '1.4.4', 'video', 'video', NULL, '{"min_area_ratio": {"name": "min_area_ratio", "description": "参数: min_area_ratio", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "max_area_ratio": {"name": "max_area_ratio", "description": "参数: max_area_ratio", "type": "inputNumber", "defaultVal": 1.0, "required": false, "min": 0, "max": 10000}, "frame_sample_num": {"name": "frame_sample_num", "description": "参数: frame_sample_num", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "languages_to_detect": {"name": "languages_to_detect", "description": "参数: languages_to_detect", "type": "input", "defaultVal": "[''ch_sim'', ''en'']", "required": false}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_remove_watermark_mapper', 'video_remove_watermark_mapper', '去除视频中的水印', '1.4.4', 'video', 'video', NULL, '{"roi_strings": {"name": "roi_strings", "description": "感兴趣区域文本列表", "type": "input", "defaultVal": "[''0,0,0.1,0.1'']", "required": false}, "roi_type": {"name": "roi_type", "description": "参数: roi_type", "type": "input", "defaultVal": "ratio", "required": false}, "roi_key": {"name": "roi_key", "description": "参数: roi_key", "type": "input", "defaultVal": "", "required": false}, "frame_num": {"name": "frame_num", "description": "帧数量", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "min_frame_threshold": {"name": "min_frame_threshold", "description": "参数: min_frame_threshold", "type": "inputNumber", "defaultVal": 7, "required": false, "min": 0, "max": 10000}, "detection_method": {"name": "detection_method", "description": "参数: detection_method", "type": "input", "defaultVal": "pixel_value", "required": false}, "save_dir": {"name": "save_dir", "description": "保存目录", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_resize_aspect_ratio_mapper', 'video_resize_aspect_ratio_mapper', '按长宽比调整视频尺寸', '1.4.4', 'video', 'video', NULL, '{"min_ratio": {"name": "min_ratio", "description": "最小比例下限", "type": "input", "defaultVal": "9/21", "required": false}, "max_ratio": {"name": "max_ratio", "description": "最大比例上限", "type": "input", "defaultVal": "21/9", "required": false}, "strategy": {"name": "strategy", "description": "参数: strategy", "type": "input", "defaultVal": "increase", "required": false}, "save_dir": {"name": "save_dir", "description": "保存目录", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_resize_resolution_mapper', 'video_resize_resolution_mapper', '按分辨率调整视频尺寸', '1.4.4', 'video', 'video', NULL, '{"min_width": {"name": "min_width", "description": "最小宽度", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "max_width": {"name": "max_width", "description": "最大宽度", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "min_height": {"name": "min_height", "description": "最小高度", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "max_height": {"name": "max_height", "description": "最大高度", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "force_original_aspect_ratio": {"name": "force_original_aspect_ratio", "description": "参数: force_original_aspect_ratio", "type": "input", "defaultVal": "disable", "required": false}, "force_divisible_by": {"name": "force_divisible_by", "description": "参数: force_divisible_by", "type": "inputNumber", "defaultVal": 2, "required": false, "min": 0, "max": 10000}, "save_dir": {"name": "save_dir", "description": "保存目录", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_resolution_filter', 'video_resolution_filter', '过滤视频分辨率在指定范围内的样本', '1.4.4', 'video', 'video', NULL, '{"min_width": {"name": "min_width", "description": "最小宽度", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "max_width": {"name": "max_width", "description": "最大宽度", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "min_height": {"name": "min_height", "description": "最小高度", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "max_height": {"name": "max_height", "description": "最大高度", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_split_by_duration_mapper', 'video_split_by_duration_mapper', '按持续时间分割视频', '1.4.4', 'video', 'video', NULL, '{"split_duration": {"name": "split_duration", "description": "参数: split_duration", "type": "inputNumber", "defaultVal": 10.0, "required": false, "min": 0, "max": 10000}, "min_last_split_duration": {"name": "min_last_split_duration", "description": "参数: min_last_split_duration", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "keep_original_sample": {"name": "keep_original_sample", "description": "是否保留原始样本", "type": "switch", "defaultVal": true, "required": false}, "save_dir": {"name": "save_dir", "description": "保存目录", "type": "input", "defaultVal": "", "required": false}, "video_backend": {"name": "video_backend", "description": "参数: video_backend", "type": "input", "defaultVal": "ffmpeg", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_split_by_key_frame_mapper', 'video_split_by_key_frame_mapper', '按关键帧分割视频', '1.4.4', 'video', 'video', NULL, '{"keep_original_sample": {"name": "keep_original_sample", "description": "是否保留原始样本", "type": "switch", "defaultVal": true, "required": false}, "save_dir": {"name": "save_dir", "description": "保存目录", "type": "input", "defaultVal": "", "required": false}, "video_backend": {"name": "video_backend", "description": "参数: video_backend", "type": "input", "defaultVal": "ffmpeg", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_split_by_scene_mapper', 'video_split_by_scene_mapper', '按场景分割视频', '1.4.4', 'video', 'video', NULL, '{"detector": {"name": "detector", "description": "参数: detector", "type": "input", "defaultVal": "ContentDetector", "required": false}, "threshold": {"name": "threshold", "description": "参数: threshold", "type": "inputNumber", "defaultVal": 27.0, "required": false, "min": 0, "max": 10000}, "min_scene_len": {"name": "min_scene_len", "description": "参数: min_scene_len", "type": "inputNumber", "defaultVal": 15, "required": false, "min": 0, "max": 10000}, "show_progress": {"name": "show_progress", "description": "参数: show_progress", "type": "switch", "defaultVal": false, "required": false}, "save_dir": {"name": "save_dir", "description": "保存目录", "type": "input", "defaultVal": "", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_tagging_from_audio_mapper', 'video_tagging_from_audio_mapper', '根据视频音频内容生成标签', '1.4.4', 'audio', 'audio', NULL, '{"hf_ast": {"name": "hf_ast", "description": "AST音频模型路径", "type": "input", "defaultVal": "MIT/ast-finetuned-audioset-10-10-0.4593", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "tag_field_name": {"name": "tag_field_name", "description": "标签字段名", "type": "input", "defaultVal": "video_audio_tags", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_tagging_from_frames_filter', 'video_tagging_from_frames_filter', '根据视频帧内容过滤标签', '1.4.4', 'video', 'video', NULL, '{"tags": {"name": "tags", "description": "参数: tags", "type": "input", "defaultVal": "[''people'']", "required": false}, "contain": {"name": "contain", "description": "参数: contain", "type": "input", "defaultVal": "any", "required": false}, "frame_sampling_method": {"name": "frame_sampling_method", "description": "参数: frame_sampling_method", "type": "input", "defaultVal": "all_keyframes", "required": false}, "frame_num": {"name": "frame_num", "description": "帧数量", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "tag_field_name": {"name": "tag_field_name", "description": "标签字段名", "type": "input", "defaultVal": "video_frame_tags", "required": false}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_tagging_from_frames_mapper', 'video_tagging_from_frames_mapper', '根据视频帧内容生成标签', '1.4.4', 'video', 'video', NULL, '{"frame_sampling_method": {"name": "frame_sampling_method", "description": "参数: frame_sampling_method", "type": "input", "defaultVal": "all_keyframes", "required": false}, "frame_num": {"name": "frame_num", "description": "帧数量", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "tag_field_name": {"name": "tag_field_name", "description": "标签字段名", "type": "input", "defaultVal": "video_frame_tags", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_watermark_filter', 'video_watermark_filter', '过滤含有水印的视频样本', '1.4.4', 'video', 'video', NULL, '{"hf_watermark_model": {"name": "hf_watermark_model", "description": "水印检测模型路径", "type": "input", "defaultVal": "amrul-hzz/watermark_detector", "required": false}, "trust_remote_code": {"name": "trust_remote_code", "description": "是否信任远程代码", "type": "switch", "defaultVal": false, "required": false}, "prob_threshold": {"name": "prob_threshold", "description": "参数: prob_threshold", "type": "inputNumber", "defaultVal": 0.8, "required": false, "min": 0, "max": 10000}, "frame_sampling_method": {"name": "frame_sampling_method", "description": "参数: frame_sampling_method", "type": "input", "defaultVal": "all_keyframes", "required": false}, "frame_num": {"name": "frame_num", "description": "帧数量", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "reduce_mode": {"name": "reduce_mode", "description": "参数: reduce_mode", "type": "input", "defaultVal": "avg", "required": false}, "any_or_all": {"name": "any_or_all", "description": "匹配策略(任意/全部)", "type": "input", "defaultVal": "any", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('video_whole_body_pose_estimation_mapper', 'video_whole_body_pose_estimation_mapper', '视频全身姿态估计算子', '1.4.4', 'video', 'video', NULL, '{"onnx_det_model": {"name": "onnx_det_model", "description": "ONNX检测模型路径", "type": "input", "defaultVal": "yolox_l.onnx", "required": false}, "onnx_pose_model": {"name": "onnx_pose_model", "description": "参数: onnx_pose_model", "type": "input", "defaultVal": "dw-ll_ucoco_384.onnx", "required": false}, "frame_num": {"name": "frame_num", "description": "帧数量", "type": "inputNumber", "defaultVal": 3, "required": false, "min": 0, "max": 10000}, "duration": {"name": "duration", "description": "参数: duration", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "tag_field_name": {"name": "tag_field_name", "description": "标签字段名", "type": "input", "defaultVal": "pose_estimation_tags", "required": false}, "frame_dir": {"name": "frame_dir", "description": "参数: frame_dir", "type": "input", "defaultVal": "C:\\Users\\Administrator/.cache\\data_juicer\\assets", "required": false}, "if_save_visualization": {"name": "if_save_visualization", "description": "参数: if_save_visualization", "type": "switch", "defaultVal": false, "required": false}, "save_visualization_dir": {"name": "save_visualization_dir", "description": "参数: save_visualization_dir", "type": "input", "defaultVal": "C:\\Users\\Administrator/.cache\\data_juicer\\assets", "required": false}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('whitespace_normalization_mapper', 'whitespace_normalization_mapper', '规范化文本中的空白字符', '1.4.4', 'text', 'text', NULL, '{"video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('word_repetition_filter', 'word_repetition_filter', '过滤单词级n-gram重复率在指定范围内的样本', '1.4.4', 'text', 'text', NULL, '{"lang": {"name": "lang", "description": "目标语言", "type": "input", "defaultVal": "en", "required": false}, "tokenization": {"name": "tokenization", "description": "是否使用分词", "type": "switch", "defaultVal": false, "required": false}, "rep_len": {"name": "rep_len", "description": "N-gram重复长度", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "min_ratio": {"name": "min_ratio", "description": "最小比例下限", "type": "inputNumber", "defaultVal": 0.0, "required": false, "min": 0, "max": 10000}, "max_ratio": {"name": "max_ratio", "description": "最大比例上限", "type": "inputNumber", "defaultVal": 0.5, "required": false, "min": 0, "max": 10000}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system'),
|
|
('words_num_filter', 'words_num_filter', '过滤总词数在指定范围内的样本', '1.4.4', 'text', 'text', NULL, '{"lang": {"name": "lang", "description": "目标语言", "type": "input", "defaultVal": "en", "required": false}, "tokenization": {"name": "tokenization", "description": "是否使用分词", "type": "switch", "defaultVal": false, "required": false}, "min_num": {"name": "min_num", "description": "参数: min_num", "type": "inputNumber", "defaultVal": 10, "required": false, "min": 0, "max": 10000}, "max_num": {"name": "max_num", "description": "参数: max_num", "type": "inputNumber", "defaultVal": 99999, "required": false, "min": 0, "max": 999990}, "video_key": {"name": "video_key", "description": "视频字段名", "type": "input", "defaultVal": "videos", "required": false}, "audio_key": {"name": "audio_key", "description": "音频字段名", "type": "input", "defaultVal": "audios", "required": false}, "text_key": {"name": "text_key", "description": "文本字段名", "type": "input", "defaultVal": "text", "required": false}, "accelerator": {"name": "accelerator", "description": "加速设备(cpu/gpu)", "type": "input", "defaultVal": "", "required": false}, "num_proc": {"name": "num_proc", "description": "并行进程数", "type": "inputNumber", "defaultVal": 1, "required": false, "min": 0, "max": 10000}, "cpu_required": {"name": "cpu_required", "description": "每个Worker所需CPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "gpu_required": {"name": "gpu_required", "description": "每个Worker所需GPU数", "type": "inputNumber", "defaultVal": 0, "required": false, "min": 0, "max": 10000}, "mem_required": {"name": "mem_required", "description": "内存资源限制", "type": "input", "defaultVal": null, "required": false}, "work_dir": {"name": "work_dir", "description": "算子工作目录", "type": "input", "defaultVal": "", "required": false}, "skip_op_error": {"name": "skip_op_error", "description": "是否跳过算子执行异常", "type": "switch", "defaultVal": true, "required": false}, "turbo": {"name": "turbo", "description": "Turbo加速开关", "type": "switch", "defaultVal": false, "required": false}, "history_key": {"name": "history_key", "description": "对话历史字段名", "type": "input", "defaultVal": "history", "required": false}, "prompt_key": {"name": "prompt_key", "description": "Prompt字段名", "type": "input", "defaultVal": "prompt", "required": false}, "query_key": {"name": "query_key", "description": "Query字段名", "type": "input", "defaultVal": "query", "required": false}, "response_key": {"name": "response_key", "description": "Response字段名", "type": "input", "defaultVal": "response", "required": false}, "instruction_key": {"name": "instruction_key", "description": "指令字段名", "type": "input", "defaultVal": "instruction", "required": false}, "index_key": {"name": "index_key", "description": "索引字段名", "type": "input", "defaultVal": "", "required": false}}', '', 8192, NULL, false, 'system', 'system')
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
-- Data-Juicer 算子分类关联
|
|
INSERT INTO t_operator_category_relation (category_id, operator_id)
|
|
SELECT c.id, o.id
|
|
FROM t_operator_category c
|
|
CROSS JOIN t_operator o
|
|
WHERE c.value IN ('data-juicer', 'cleaning')
|
|
AND o.id IN ('alphanumeric_filter', 'audio_add_gaussian_noise_mapper', 'audio_duration_filter', 'audio_ffmpeg_wrapped_mapper', 'audio_nmf_snr_filter', 'audio_size_filter', 'average_line_length_filter', 'calibrate_qa_mapper', 'calibrate_query_mapper', 'calibrate_response_mapper', 'character_repetition_filter', 'chinese_convert_mapper', 'clean_copyright_mapper', 'clean_email_mapper', 'clean_html_mapper', 'clean_ip_mapper', 'clean_links_mapper', 'detect_character_attributes_mapper', 'detect_character_locations_mapper', 'detect_main_character_mapper', 'dialog_intent_detection_mapper', 'dialog_sentiment_detection_mapper', 'dialog_sentiment_intensity_mapper', 'dialog_topic_detection_mapper', 'document_deduplicator', 'document_minhash_deduplicator', 'document_simhash_deduplicator', 'download_file_mapper', 'entity_attribute_aggregator', 'expand_macro_mapper', 'extract_entity_attribute_mapper', 'extract_entity_relation_mapper', 'extract_event_mapper', 'extract_keyword_mapper', 'extract_nickname_mapper', 'extract_support_text_mapper', 'extract_tables_from_html_mapper', 'fix_unicode_mapper', 'flagged_words_filter', 'frequency_specified_field_selector', 'general_field_filter', 'general_fused_op', 'generate_qa_from_examples_mapper', 'generate_qa_from_text_mapper', 'human_preference_annotation_mapper', 'image_aesthetics_filter', 'image_aspect_ratio_filter', 'image_blur_mapper', 'image_captioning_from_gpt4v_mapper', 'image_captioning_mapper', 'image_deduplicator', 'image_detection_yolo_mapper', 'image_diffusion_mapper', 'image_face_blur_mapper', 'image_face_count_filter', 'image_face_ratio_filter', 'image_nsfw_filter', 'image_pair_similarity_filter', 'image_remove_background_mapper', 'image_segment_mapper', 'image_shape_filter', 'image_size_filter', 'image_tagging_mapper', 'image_text_matching_filter', 'image_text_similarity_filter', 'image_watermark_filter', 'imgdiff_difference_area_generator_mapper', 'imgdiff_difference_caption_generator_mapper', 'in_context_influence_filter', 'instruction_following_difficulty_filter', 'key_value_grouper', 'language_id_score_filter', 'llm_analysis_filter', 'llm_difficulty_score_filter', 'llm_perplexity_filter', 'llm_quality_score_filter', 'llm_task_relevance_filter', 'maximum_line_length_filter', 'meta_tags_aggregator', 'mllm_mapper', 'most_relevant_entities_aggregator', 'naive_grouper', 'naive_reverse_grouper', 'nested_aggregator', 'nlpaug_en_mapper', 'nlpcda_zh_mapper', 'optimize_prompt_mapper', 'optimize_qa_mapper', 'optimize_query_mapper', 'optimize_response_mapper', 'pair_preference_mapper', 'perplexity_filter', 'phrase_grounding_recall_filter', 'punctuation_normalization_mapper', 'python_file_mapper', 'python_lambda_mapper', 'query_intent_detection_mapper', 'query_sentiment_detection_mapper', 'query_topic_detection_mapper', 'random_selector', 'range_specified_field_selector', 'ray_bts_minhash_deduplicator', 'ray_document_deduplicator', 'ray_image_deduplicator', 'ray_video_deduplicator', 'relation_identity_mapper', 'remove_bibliography_mapper', 'remove_comments_mapper', 'remove_header_mapper', 'remove_long_words_mapper', 'remove_non_chinese_character_mapper', 'remove_repeat_sentences_mapper', 'remove_specific_chars_mapper', 'remove_table_text_mapper', 'remove_words_with_incorrect_substrings_mapper', 'replace_content_mapper', 'sdxl_prompt2prompt_mapper', 'sentence_augmentation_mapper', 'sentence_split_mapper', 'special_characters_filter', 'specified_field_filter', 'specified_numeric_field_filter', 'stopwords_filter', 'suffix_filter', 'tags_specified_field_selector', 'text_action_filter', 'text_chunk_mapper', 'text_embd_similarity_filter', 'text_entity_dependency_filter', 'text_length_filter', 'text_pair_similarity_filter', 'token_num_filter', 'topk_specified_field_selector', 'vggt_mapper', 'video_aesthetics_filter', 'video_aspect_ratio_filter', 'video_captioning_from_audio_mapper', 'video_captioning_from_frames_mapper', 'video_captioning_from_summarizer_mapper', 'video_captioning_from_video_mapper', 'video_deduplicator', 'video_duration_filter', 'video_extract_frames_mapper', 'video_face_blur_mapper', 'video_ffmpeg_wrapped_mapper', 'video_frames_text_similarity_filter', 'video_hand_reconstruction_mapper', 'video_motion_score_filter', 'video_motion_score_raft_filter', 'video_nsfw_filter', 'video_ocr_area_ratio_filter', 'video_remove_watermark_mapper', 'video_resize_aspect_ratio_mapper', 'video_resize_resolution_mapper', 'video_resolution_filter', 'video_split_by_duration_mapper', 'video_split_by_key_frame_mapper', 'video_split_by_scene_mapper', 'video_tagging_from_audio_mapper', 'video_tagging_from_frames_filter', 'video_tagging_from_frames_mapper', 'video_watermark_filter', 'video_whole_body_pose_estimation_mapper', 'whitespace_normalization_mapper', 'word_repetition_filter', 'words_num_filter')
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
-- ============================================================
|
|
-- 2. 缺失算子的分类关联 INSERT INTO t_operator_category_relation
|
|
-- ============================================================
|
|
|
|
-- ImgAdvertisementImagesCleaner: 图片 + Python + 清洗 + 系统预置 + DeepData
|
|
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
|
SELECT c.id, o.id
|
|
FROM t_operator_category c
|
|
CROSS JOIN t_operator o
|
|
WHERE c.id IN ('de36b61c-9e8a-4422-8c31-d30585c7100f', '9eda9d5d-072b-499b-916c-797a0a8750e1', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34', '8c09476a-a922-418f-a908-733f8a0de521')
|
|
AND o.id IN ('ImgAdvertisementImagesCleaner')
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
-- SlideFormatter: 图片 + Python + 系统预置 + DeepData
|
|
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
|
SELECT c.id, o.id
|
|
FROM t_operator_category c
|
|
CROSS JOIN t_operator o
|
|
WHERE c.id IN ('de36b61c-9e8a-4422-8c31-d30585c7100f', '9eda9d5d-072b-499b-916c-797a0a8750e1', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34')
|
|
AND o.id IN ('SlideFormatter')
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
-- QAConditionEvaluator: 文本 + Python + 系统预置 + DeepData
|
|
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
|
SELECT c.id, o.id
|
|
FROM t_operator_category c
|
|
CROSS JOIN t_operator o
|
|
WHERE c.id IN ('d8a5df7a-52a9-42c2-83c4-01062e60f597', '9eda9d5d-072b-499b-916c-797a0a8750e1', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34')
|
|
AND o.id IN ('QAConditionEvaluator')
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
-- TextQualityEvaluation: 文本 + Python + 系统预置 + DeepData
|
|
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
|
SELECT c.id, o.id
|
|
FROM t_operator_category c
|
|
CROSS JOIN t_operator o
|
|
WHERE c.id IN ('d8a5df7a-52a9-42c2-83c4-01062e60f597', '9eda9d5d-072b-499b-916c-797a0a8750e1', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34')
|
|
AND o.id IN ('TextQualityEvaluation')
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
-- KnowledgeRelationSlice: 文本 + Python + 清洗 + 系统预置 + DeepData
|
|
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
|
SELECT c.id, o.id
|
|
FROM t_operator_category c
|
|
CROSS JOIN t_operator o
|
|
WHERE c.id IN ('d8a5df7a-52a9-42c2-83c4-01062e60f597', '9eda9d5d-072b-499b-916c-797a0a8750e1', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34', '8c09476a-a922-418f-a908-733f8a0de521')
|
|
AND o.id IN ('KnowledgeRelationSlice')
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
-- DuplicateSentencesCleaner: 文本 + Python + 清洗 + 系统预置 + DeepData
|
|
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
|
SELECT c.id, o.id
|
|
FROM t_operator_category c
|
|
CROSS JOIN t_operator o
|
|
WHERE c.id IN ('d8a5df7a-52a9-42c2-83c4-01062e60f597', '9eda9d5d-072b-499b-916c-797a0a8750e1', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34', '8c09476a-a922-418f-a908-733f8a0de521')
|
|
AND o.id IN ('DuplicateSentencesCleaner')
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
-- TextToWord: 文本 + Python + 清洗 + 系统预置 + DeepData
|
|
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
|
SELECT c.id, o.id
|
|
FROM t_operator_category c
|
|
CROSS JOIN t_operator o
|
|
WHERE c.id IN ('d8a5df7a-52a9-42c2-83c4-01062e60f597', '9eda9d5d-072b-499b-916c-797a0a8750e1', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34', '8c09476a-a922-418f-a908-733f8a0de521')
|
|
AND o.id IN ('TextToWord')
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
-- Segmentation: 文本 + Python + 系统预置 + DeepData
|
|
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
|
SELECT c.id, o.id
|
|
FROM t_operator_category c
|
|
CROSS JOIN t_operator o
|
|
WHERE c.id IN ('d8a5df7a-52a9-42c2-83c4-01062e60f597', '9eda9d5d-072b-499b-916c-797a0a8750e1', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34')
|
|
AND o.id IN ('Segmentation')
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
-- AnnotationSlicer: 图片 + Python + 系统预置 + DeepData
|
|
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
|
SELECT c.id, o.id
|
|
FROM t_operator_category c
|
|
CROSS JOIN t_operator o
|
|
WHERE c.id IN ('de36b61c-9e8a-4422-8c31-d30585c7100f', '9eda9d5d-072b-499b-916c-797a0a8750e1', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34')
|
|
AND o.id IN ('AnnotationSlicer')
|
|
ON CONFLICT DO NOTHING;
|
|
|
|
-- SimpleSlicer: 图片 + Python + 系统预置 + DeepData
|
|
INSERT INTO t_operator_category_relation(category_id, operator_id)
|
|
SELECT c.id, o.id
|
|
FROM t_operator_category c
|
|
CROSS JOIN t_operator o
|
|
WHERE c.id IN ('de36b61c-9e8a-4422-8c31-d30585c7100f', '9eda9d5d-072b-499b-916c-797a0a8750e1', '96a3b07a-3439-4557-a835-525faad60ca3', '431e7798-5426-4e1a-aae6-b9905a836b34')
|
|
AND o.id IN ('SimpleSlicer')
|
|
ON CONFLICT DO NOTHING;
|