-- ================================================================
-- Data-Juicer 算子初始化脚本 (独立注入)
-- 从 data-juicer 官方 Operators Zoo 提取所有算子及其参数
-- 参考: https://github.com/datajuicer/data-juicer/blob/main/docs/Operators.md
-- 版本: 1.4.4
-- ================================================================

-- ================================================================
-- 插入 Data-Juicer 算子 (所有 t_operator 字段)
-- ================================================================
INSERT INTO t_operator
(id, name, description, version, inputs, outputs, runtime, settings, file_name, file_size, is_star, created_by, updated_by)
VALUES

-- ===== aggregator (4个) =====
('entity_attribute_aggregator', '实体属性聚合器',
 '汇总一组文档中实体的给定属性。Summarizes a given attribute of an entity from a set of documents.',
 '1.4.4', 'text', 'text', NULL,
'{"api_prompt":{"name":"API提示词","description":"用于汇总实体属性的LLM提示词","type":"textarea","defaultVal":""},"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"使用的LLM模型","type":"input","defaultVal":"gpt-3.5-turbo"}}',
 '', 8192, false, 'system', 'system'),

('meta_tags_aggregator', '元标签聚合器',
 '将类似的元标记合并到一个统一的标记中。Merge similar meta tags into a single, unified tag.',
 '1.4.4', 'text', 'text', NULL,
'{"similarity_threshold":{"name":"相似度阈值","description":"判定标签相似的阈值","type":"slider","defaultVal":0.8,"min":0,"max":1,"step":0.01}}',
 '', 5120, false, 'system', 'system'),

('most_relevant_entities_aggregator', '最相关实体聚合器',
 '从提供的文本中提取与给定实体密切相关的实体并对其进行排名。Extracts and ranks entities closely related to a given entity.',
 '1.4.4', 'text', 'text', NULL,
'{"entity_name":{"name":"实体名称","description":"目标实体名称","type":"input","defaultVal":""},"top_k":{"name":"Top-K","description":"返回的最相关实体数量","type":"inputNumber","defaultVal":10,"min":1}}',
 '', 5120, false, 'system', 'system'),

('nested_aggregator', '嵌套聚合器',
 '将多个示例中的嵌套内容聚合到单个摘要中。Aggregates nested content from multiple samples into a single summary.',
 '1.4.4', 'text', 'text', NULL,
'{"api_prompt":{"name":"API提示词","description":"嵌套内容聚合的LLM提示词","type":"textarea","defaultVal":""}}',
 '', 5120, false, 'system', 'system'),

-- ===== deduplicator (10个) =====
('document_deduplicator', '文档去重器',
 '使用完全匹配在文档级别删除重复的样本。Deduplicates samples at the document level using exact matching.',
 '1.4.4', 'text', 'text', NULL,
'{"hash_method":{"name":"哈希方法","description":"用于精确匹配的哈希方法","type":"select","defaultVal":"md5","options":[{"label":"MD5","value":"md5"},{"label":"SHA256","value":"sha256"}]},"lowercase":{"name":"转小写","description":"是否在哈希前将文本转为小写","type":"switch","defaultVal":true,"checkedLabel":"是","unCheckedLabel":"否"},"ignore_non_character":{"name":"忽略非字符","description":"是否忽略非字符字符","type":"switch","defaultVal":true,"checkedLabel":"是","unCheckedLabel":"否"}}',
 '', 8192, false, 'system', 'system'),

('document_minhash_deduplicator', '文档MinHash去重器',
 '使用MinHash LSH在文档级别删除重复样本。Deduplicates samples at the document level using MinHash LSH.',
 '1.4.4', 'text', 'text', NULL,
'{"window_size":{"name":"窗口大小","description":"MinHash的窗口大小","type":"inputNumber","defaultVal":5,"min":1},"num_perm":{"name":"排列数","description":"MinHash的排列函数数量","type":"inputNumber","defaultVal":256,"min":1},"jaccard_threshold":{"name":"Jaccard阈值","description":"Jaccard相似度阈值","type":"slider","defaultVal":0.8,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('document_simhash_deduplicator', '文档SimHash去重器',
 '使用SimHash在文档级别删除重复的样本。Deduplicates samples at the document level using SimHash.',
 '1.4.4', 'text', 'text', NULL,
'{"window_size":{"name":"窗口大小","description":"SimHash的窗口大小(n-gram)","type":"inputNumber","defaultVal":6,"min":1},"hamming_distance":{"name":"汉明距离","description":"SimHash的汉明距离阈值","type":"inputNumber","defaultVal":4,"min":0,"max":64}}',
 '', 8192, false, 'system', 'system'),

('image_deduplicator', '图像去重器',
 '通过图像的精确匹配在文档级别删除重复的样本。Deduplicates by exact matching of images.',
 '1.4.4', 'image', 'image', NULL,
 NULL,
 '', 8192, false, 'system', 'system'),

('video_deduplicator', '视频去重器',
 '使用视频的精确匹配在文档级别删除重复的样本。Deduplicates by exact matching of videos.',
 '1.4.4', 'video', 'video', NULL,
 NULL,
 '', 8192, false, 'system', 'system'),

('ray_bts_minhash_deduplicator', 'Ray BTS MinHash去重器',
 '在Ray分布式模式下运行的MinHashLSH重复数据删除程序。A distributed MinHashLSH deduplicator with load balancing.',
 '1.4.4', 'text', 'text', NULL,
'{"window_size":{"name":"窗口大小","description":"MinHash的窗口大小","type":"inputNumber","defaultVal":5,"min":1},"num_perm":{"name":"排列数","description":"MinHash的排列函数数量","type":"inputNumber","defaultVal":256,"min":1},"jaccard_threshold":{"name":"Jaccard阈值","description":"Jaccard相似度阈值","type":"slider","defaultVal":0.8,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('ray_document_deduplicator', 'Ray文档去重器',
 '在Ray分布式模式下使用精确匹配在文档级别删除重复的样本。Deduplicates using exact matching in Ray distributed mode.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 8192, false, 'system', 'system'),

('ray_image_deduplicator', 'Ray图像去重器',
 '在Ray分布式模式下使用图像的精确匹配在文档级别删除重复样本。Deduplicates images in Ray distributed mode.',
 '1.4.4', 'image', 'image', NULL,
 NULL,
 '', 8192, false, 'system', 'system'),

('ray_video_deduplicator', 'Ray视频去重器',
 '在Ray分布式模式下使用视频的精确匹配在文档级删除重复样本。Deduplicates videos in Ray distributed mode.',
 '1.4.4', 'video', 'video', NULL,
 NULL,
 '', 8192, false, 'system', 'system'),

('document_line_deduplicator', '文档行级去重器',
 '在文档间按行级别去重。Deduplicates at the line level across documents.',
 '1.4.4', 'text', 'text', NULL,
'{"window_size":{"name":"窗口大小","description":"行级去重的窗口大小","type":"inputNumber","defaultVal":1,"min":1}}',
 '', 5120, false, 'system', 'system'),

-- ===== filter (text模态) =====
('alphanumeric_filter', '字母数字比例过滤器',
 '过滤器，以保持具有特定范围内的字母/数字比率的样本。Filter to keep samples with alphabet/numeric ratio within a specific range.',
 '1.4.4', 'text', 'text', NULL,
'{"min":{"name":"最小比例","description":"字母/数字比例的下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大比例","description":"字母/数字比例的上限","type":"slider","defaultVal":0.25,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('average_line_length_filter', '平均行长过滤器',
 '过滤器，以保持平均线长度在特定范围内的样本。Filter to keep samples with average line length within a specific range.',
 '1.4.4', 'text', 'text', NULL,
'{"min":{"name":"最小平均行长","description":"平均行长下限","type":"inputNumber","defaultVal":0,"min":0},"max":{"name":"最大平均行长","description":"平均行长上限","type":"inputNumber","defaultVal":10000,"min":0}}',
 '', 8192, false, 'system', 'system'),

('character_repetition_filter', '字符重复过滤器',
 '过滤器将具有字符级n-gram重复比的样本保持在特定范围内。Filter to keep samples with character-level n-gram repetition ratio.',
 '1.4.4', 'text', 'text', NULL,
'{"rep_len":{"name":"N-gram长度","description":"字符级n-gram的长度","type":"inputNumber","defaultVal":10,"min":1},"min":{"name":"最小重复率","description":"字符重复率下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大重复率","description":"字符重复率上限","type":"slider","defaultVal":0.25,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('flagged_words_filter', '标记词过滤器',
 '过滤器将标记词比率的样本保留在指定范围内。Filter to keep samples with flagged-word ratio in a specified range.',
 '1.4.4', 'text', 'text', NULL,
'{"lang":{"name":"语言","description":"标记词字典使用的语言","type":"select","defaultVal":"en","options":[{"label":"English","value":"en"},{"label":"Chinese","value":"zh"},{"label":"Russian","value":"ru"},{"label":"German","value":"de"}]},"tokenization":{"name":"分词","description":"是否使用模型分词","type":"switch","defaultVal":false,"checkedLabel":"是","unCheckedLabel":"否"},"min":{"name":"最小标记词比例","description":"标记词比例下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大标记词比例","description":"标记词比例上限","type":"slider","defaultVal":0.25,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('general_field_filter', '通用字段过滤器',
 '根据常规字段筛选条件保留样本。Filter to keep samples based on a general field filter condition.',
 '1.4.4', 'text', 'text', NULL,
'{"field_key":{"name":"字段名","description":"用于过滤的字段名","type":"input","defaultVal":""},"condition":{"name":"条件","description":"过滤条件表达式","type":"input","defaultVal":""}}',
 '', 5120, false, 'system', 'system'),

('maximum_line_length_filter', '最大行长过滤器',
 '筛选器将最大行长度的样本保持在指定范围内。Filter to keep samples with a maximum line length within a specified range.',
 '1.4.4', 'text', 'text', NULL,
'{"min":{"name":"最小行长","description":"最大行长的下限","type":"inputNumber","defaultVal":0,"min":0},"max":{"name":"最大行长","description":"最大行长的上限","type":"inputNumber","defaultVal":10000,"min":0}}',
 '', 8192, false, 'system', 'system'),

('perplexity_filter', '困惑度过滤器',
 '过滤以保持困惑分数在指定范围内的样本。Filter to keep samples with perplexity score in a specified range.',
 '1.4.4', 'text', 'text', NULL,
'{"max_perp":{"name":"最大困惑度","description":"困惑度上限","type":"inputNumber","defaultVal":1500,"min":0}}',
 '', 8192, false, 'system', 'system'),

('special_characters_filter', '特殊字符过滤器',
 '过滤器，以将具有特殊字符比率的样本保持在特定范围内。Filter to keep samples with special-character ratio within a specific range.',
 '1.4.4', 'text', 'text', NULL,
'{"min":{"name":"最小特殊字符比例","description":"特殊字符比例下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大特殊字符比例","description":"特殊字符比例上限","type":"slider","defaultVal":0.25,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('stopwords_filter', '停用词过滤器',
 '过滤器将停用词比率的样本保持在指定范围内。Filter to keep samples with stopword ratio within a specified range.',
 '1.4.4', 'text', 'text', NULL,
'{"lang":{"name":"语言","description":"停用词使用的语言","type":"select","defaultVal":"en","options":[{"label":"English","value":"en"},{"label":"Chinese","value":"zh"}]},"tokenization":{"name":"分词","description":"是否使用模型分词","type":"switch","defaultVal":false,"checkedLabel":"是","unCheckedLabel":"否"},"min":{"name":"最小停用词比例","description":"停用词比例下限","type":"slider","defaultVal":0.3,"min":0,"max":1,"step":0.01},"max":{"name":"最大停用词比例","description":"停用词比例上限","type":"slider","defaultVal":1.0,"min":0,"max":1,"step":0.01},"use_words_aug":{"name":"词增强","description":"是否进行词增强(适用于中文)","type":"switch","defaultVal":false,"checkedLabel":"是","unCheckedLabel":"否"}}',
 '', 8192, false, 'system', 'system'),

('suffix_filter', '后缀过滤器',
 '过滤器以保留具有指定后缀的样本。Filter to keep samples with specified suffix.',
 '1.4.4', 'text', 'text', NULL,
'{"suffixes":{"name":"后缀列表","description":"需要保留的后缀列表(逗号分隔)","type":"input","defaultVal":""}}',
 '', 5120, false, 'system', 'system'),

('text_action_filter', '文本动作过滤器',
 '过滤以保留包含最少数量操作的文本。Filter to keep texts that contain a minimum number of actions.',
 '1.4.4', 'text', 'text', NULL,
'{"lang":{"name":"语言","description":"用于动作检测的语言","type":"select","defaultVal":"en","options":[{"label":"English","value":"en"},{"label":"Chinese","value":"zh"}]},"min_action_num":{"name":"最小动作数","description":"文本中最少需要的动作数","type":"inputNumber","defaultVal":0,"min":0}}',
 '', 8192, false, 'system', 'system'),

('text_length_filter', '文本长度过滤器',
 '过滤以保持文本总长度在特定范围内的样本。Filter to keep samples with total text length within a specific range.',
 '1.4.4', 'text', 'text', NULL,
'{"min_len":{"name":"最小文本长度","description":"文本长度下限","type":"inputNumber","defaultVal":10,"min":0},"max_len":{"name":"最大文本长度","description":"文本长度上限","type":"inputNumber","defaultVal":100000,"min":0}}',
 '', 8192, false, 'system', 'system'),

('token_num_filter', 'Token数量过滤器',
 '筛选器将总令牌数的样本保留在指定范围内。Filter to keep samples with a total token number within a specified range.',
 '1.4.4', 'text', 'text', NULL,
'{"hf_tokenizer":{"name":"Tokenizer名称","description":"使用的Hugging Face tokenizer","type":"input","defaultVal":"EleutherAI/pythia-6.9b-deduped"},"min_num":{"name":"最小Token数","description":"Token数量下限","type":"inputNumber","defaultVal":10,"min":0},"max_num":{"name":"最大Token数","description":"Token数量上限","type":"inputNumber","defaultVal":100000,"min":0}}',
 '', 8192, false, 'system', 'system'),

('word_repetition_filter', '单词重复过滤器',
 '过滤器将单词级n-gram重复比率的样本保持在特定范围内。Filter to keep samples with word-level n-gram repetition ratio.',
 '1.4.4', 'text', 'text', NULL,
'{"rep_len":{"name":"N-gram长度","description":"单词级n-gram的长度","type":"inputNumber","defaultVal":2,"min":1},"min":{"name":"最小重复率","description":"单词重复率下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大重复率","description":"单词重复率上限","type":"slider","defaultVal":0.25,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('words_num_filter', '词数过滤器',
 '过滤器将样本的总字数保持在指定范围内。Filter to keep samples with a total word count within a specified range.',
 '1.4.4', 'text', 'text', NULL,
'{"min":{"name":"最小词数","description":"词数下限","type":"inputNumber","defaultVal":10,"min":0},"max":{"name":"最大词数","description":"词数上限","type":"inputNumber","defaultVal":10000,"min":0}}',
 '', 8192, false, 'system', 'system'),

('language_id_score_filter', '语种识别过滤器',
 '过滤器以保留置信度高于阈值的特定语言的样本。Filter to keep samples in a specific language with confidence above threshold.',
 '1.4.4', 'text', 'text', NULL,
'{"lang":{"name":"目标语言","description":"需要保留的语言","type":"select","defaultVal":"en","options":[{"label":"English","value":"en"},{"label":"Chinese","value":"zh"},{"label":"French","value":"fr"},{"label":"Spanish","value":"es"},{"label":"German","value":"de"},{"label":"Japanese","value":"ja"},{"label":"Korean","value":"ko"}]},"min_score":{"name":"最低置信度","description":"语种识别的最低置信度","type":"slider","defaultVal":0.8,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('text_entity_dependency_filter', '文本实体依赖过滤器',
 '根据实体依赖关系识别和过滤文本样本。Identify and filter text samples based on entity dependencies.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 8192, false, 'system', 'system'),

('specified_field_filter', '指定字段过滤器',
 '根据指定的字段信息筛选样本。Filter samples based on the specified field information.',
 '1.4.4', 'text', 'text', NULL,
'{"field_key":{"name":"字段名","description":"用于过滤的字段名","type":"input","defaultVal":""},"value":{"name":"过滤值","description":"字段需要匹配的值","type":"input","defaultVal":""}}',
 '', 5120, false, 'system', 'system'),

('specified_numeric_field_filter', '指定数值字段过滤器',
 '根据指定的数值字段值筛选样本。Filter samples based on a specified numeric field value.',
 '1.4.4', 'text', 'text', NULL,
'{"field_key":{"name":"字段名","description":"数值字段名","type":"input","defaultVal":""},"min":{"name":"最小值","description":"数值下限","type":"inputNumber","defaultVal":0},"max":{"name":"最大值","description":"数值上限","type":"inputNumber","defaultVal":10000},"min_op":{"name":"最小值开闭区间","description":"最小值是否包含","type":"switch","defaultVal":true,"checkedLabel":"包含","unCheckedLabel":"不包含"},"max_op":{"name":"最大值开闭区间","description":"最大值是否包含","type":"switch","defaultVal":true,"checkedLabel":"包含","unCheckedLabel":"不包含"}}',
 '', 8192, false, 'system', 'system'),

('in_context_influence_filter', '上下文影响过滤器',
 '根据文本在上下文中对验证集的影响来保留文本。Filter to keep texts based on their in-context influence on a validation set.',
 '1.4.4', 'text', 'text', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于计算上下文影响的HF模型","type":"input","defaultVal":"meta-llama/Llama-2-7b-hf"},"min":{"name":"最小影响分数","description":"影响分数下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大影响分数","description":"影响分数上限","type":"slider","defaultVal":1.0,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('llm_condition_filter', 'LLM条件过滤器',
 '按用户给定的自然语言条件进行过滤(LLM是/否)。Filter by user-given natural language condition (LLM yes/no).',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"使用的LLM模型","type":"input","defaultVal":"gpt-3.5-turbo"},"prompt":{"name":"条件提示词","description":"自然语言过滤条件","type":"textarea","defaultVal":""}}',
 '', 8192, false, 'system', 'system'),

('llm_difficulty_score_filter', 'LLM难度得分过滤器',
 '过滤器以保留由LLM估计的高难度分数的样本。Filter to keep samples with high difficulty scores estimated by an LLM.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"使用的LLM模型","type":"input","defaultVal":"gpt-3.5-turbo"},"min":{"name":"最小难度分数","description":"难度分数下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大难度分数","description":"难度分数上限","type":"slider","defaultVal":1.0,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('llm_quality_score_filter', 'LLM质量得分过滤器',
 '过滤器，以保留具有语言模型估计的高质量分数的样本。Filter to keep samples with a high quality score estimated by a language model.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"使用的LLM模型","type":"input","defaultVal":"gpt-3.5-turbo"},"min":{"name":"最小质量分数","description":"质量分数下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大质量分数","description":"质量分数上限","type":"slider","defaultVal":1.0,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('llm_task_relevance_filter', 'LLM任务相关性过滤器',
 '过滤器以保留与LLM估计的验证任务具有高相关性分数的样本。Filter to keep samples with high relevance scores.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"使用的LLM模型","type":"input","defaultVal":"gpt-3.5-turbo"},"validation_tasks":{"name":"验证任务","description":"任务描述","type":"textarea","defaultVal":""},"min":{"name":"最小相关性","description":"相关性分数下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大相关性","description":"相关性分数上限","type":"slider","defaultVal":1.0,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('llm_analysis_filter', 'LLM分析过滤器',
 '用于利用LLMs分析和过滤数据样本的基本筛选器类。Base filter class for leveraging LLMs to analyze and filter data samples.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"使用的LLM模型","type":"input","defaultVal":"gpt-3.5-turbo"},"prompt":{"name":"分析提示词","description":"LLM分析提示词","type":"textarea","defaultVal":""}}',
 '', 8192, false, 'system', 'system'),

('llm_perplexity_filter', 'LLM困惑度过滤器',
 '过滤器将困惑分数的样本保留在指定范围内，使用指定的LLM计算。Filter to keep samples with perplexity scores computed using a specified LLM.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"使用的LLM模型","type":"input","defaultVal":"gpt-3.5-turbo"},"min":{"name":"最小困惑度","description":"困惑度下限","type":"slider","defaultVal":0.0,"min":0,"max":100,"step":0.1},"max":{"name":"最大困惑度","description":"困惑度上限","type":"slider","defaultVal":50.0,"min":0,"max":100,"step":0.1}}',
 '', 8192, false, 'system', 'system'),

('instruction_following_difficulty_filter', '指令跟随难度过滤器',
 '过滤以保持文本基于他们的指令跟随难度(IFD)分数。Filter to keep texts based on instruction following difficulty score.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"使用的LLM模型","type":"input","defaultVal":"gpt-3.5-turbo"},"min":{"name":"最小难度分数","description":"难度分数下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大难度分数","description":"难度分数上限","type":"slider","defaultVal":1.0,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('text_embd_similarity_filter', '文本嵌入相似度过滤器',
 '过滤器，以保留与一组给定验证文本的平均嵌入相似度在特定范围内的文本。Filter to keep texts whose average embedding similarity falls within a specific range.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"Embedding API密钥","type":"input","defaultVal":""},"model":{"name":"Embedding模型","description":"使用的Embedding模型","type":"input","defaultVal":"text-embedding-ada-002"},"validation_texts":{"name":"验证文本","description":"用于计算相似度的参考文本","type":"textarea","defaultVal":""},"min":{"name":"最小相似度","description":"相似度下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大相似度","description":"相似度上限","type":"slider","defaultVal":1.0,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('text_pair_similarity_filter', '文本对相似度过滤器',
 '过滤以将具有相似性的文本对保持在特定范围内。Filter to keep text pairs with similarities within a specific range.',
 '1.4.4', 'text', 'text', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于文本相似度计算的HF模型","type":"input","defaultVal":""},"min":{"name":"最小相似度","description":"相似度下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大相似度","description":"相似度上限","type":"slider","defaultVal":1.0,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

-- ===== filter (audio模态) =====
('audio_duration_filter', '音频时长过滤器',
 '保留音频持续时间在指定范围内的数据样本。Keep data samples whose audio durations are within a specified range.',
 '1.4.4', 'audio', 'audio', NULL,
'{"min_duration":{"name":"最小时长(秒)","description":"音频时长下限","type":"inputNumber","defaultVal":0,"min":0},"max_duration":{"name":"最大时长(秒)","description":"音频时长上限","type":"inputNumber","defaultVal":3600,"min":0}}',
 '', 8192, false, 'system', 'system'),

('audio_nmf_snr_filter', '音频NMF信噪比过滤器',
 '保留音频信噪比(SNR)在指定范围内的数据样本。Keep data samples whose audio SNRs are within a specified range.',
 '1.4.4', 'audio', 'audio', NULL,
'{"min":{"name":"最小信噪比(dB)","description":"信噪比下限","type":"inputNumber","defaultVal":0,"min":0},"max":{"name":"最大信噪比(dB)","description":"信噪比上限","type":"inputNumber","defaultVal":100,"min":0}}',
 '', 8192, false, 'system', 'system'),

('audio_size_filter', '音频大小过滤器',
 '根据音频文件的大小保留数据样本。Keep data samples based on the size of their audio files.',
 '1.4.4', 'audio', 'audio', NULL,
'{"min":{"name":"最小文件大小(MB)","description":"文件大小下限","type":"inputNumber","defaultVal":0,"min":0},"max":{"name":"最大文件大小(MB)","description":"文件大小上限","type":"inputNumber","defaultVal":500,"min":0}}',
 '', 5120, false, 'system', 'system'),

-- ===== filter (image模态) =====
('image_aesthetics_filter', '图像美学过滤器',
 '过滤以保持美学分数在特定范围内的样品。Filter to keep samples with aesthetics scores within a specific range.',
 '1.4.4', 'image', 'image', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于美学评分的HF模型","type":"input","defaultVal":"ChristophSchuhmann/improved_aesthetic_predictor"},"min":{"name":"最小美学分数","description":"美学分数下限","type":"slider","defaultVal":0.0,"min":0,"max":10,"step":0.1},"max":{"name":"最大美学分数","description":"美学分数上限","type":"slider","defaultVal":10.0,"min":0,"max":10,"step":0.1}}',
 '', 8192, false, 'system', 'system'),

('image_aspect_ratio_filter', '图像长宽比过滤器',
 '过滤器，以保持样本的图像纵横比在特定范围内。Filter to keep samples with image aspect ratio within a specific range.',
 '1.4.4', 'image', 'image', NULL,
'{"min":{"name":"最小长宽比","description":"长宽比下限","type":"slider","defaultVal":0.5,"min":0.01,"max":10,"step":0.01},"max":{"name":"最大长宽比","description":"长宽比上限","type":"slider","defaultVal":2.0,"min":0.01,"max":10,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('image_face_count_filter', '图像人脸计数过滤器',
 '过滤以保持样本的面数在特定范围内。Filter to keep samples with the number of faces within a specific range.',
 '1.4.4', 'image', 'image', NULL,
'{"min":{"name":"最小人脸数","description":"人脸数量下限","type":"inputNumber","defaultVal":0,"min":0},"max":{"name":"最大人脸数","description":"人脸数量上限","type":"inputNumber","defaultVal":100,"min":0}}',
 '', 8192, false, 'system', 'system'),

('image_face_ratio_filter', '图像人脸占比过滤器',
 '过滤以保持面面积比在特定范围内的样本。Filter to keep samples with face area ratios within a specific range.',
 '1.4.4', 'image', 'image', NULL,
'{"min":{"name":"最小人脸占比","description":"人脸面积占比下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大人脸占比","description":"人脸面积占比上限","type":"slider","defaultVal":1.0,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('image_nsfw_filter', '图像NSFW过滤器',
 '过滤器保留其图像的NSFW分数在指定范围内的样本。Filter to keep samples whose images have NSFW scores in a specified range.',
 '1.4.4', 'image', 'image', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于NSFW检测的HF模型","type":"input","defaultVal":"Falconsai/nsfw_image_detection"},"min":{"name":"最小NSFW分数","description":"NSFW分数下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大NSFW分数","description":"NSFW分数上限","type":"slider","defaultVal":0.5,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('image_shape_filter', '图像形状过滤器',
 '过滤器，以保持样本的图像形状(宽度，高度)在特定的范围内。Filter to keep samples with image shape within specific ranges.',
 '1.4.4', 'image', 'image', NULL,
'{"min_width":{"name":"最小宽度","description":"图像宽度下限(像素)","type":"inputNumber","defaultVal":0,"min":0},"max_width":{"name":"最大宽度","description":"图像宽度上限(像素)","type":"inputNumber","defaultVal":10000,"min":0},"min_height":{"name":"最小高度","description":"图像高度下限(像素)","type":"inputNumber","defaultVal":0,"min":0},"max_height":{"name":"最大高度","description":"图像高度上限(像素)","type":"inputNumber","defaultVal":10000,"min":0}}',
 '', 8192, false, 'system', 'system'),

('image_size_filter', '图像大小过滤器',
 '保留图像大小在特定范围内的数据样本。Keep data samples whose image size is within a specific range.',
 '1.4.4', 'image', 'image', NULL,
'{"min":{"name":"最小文件大小(MB)","description":"图像文件大小下限","type":"inputNumber","defaultVal":0,"min":0},"max":{"name":"最大文件大小(MB)","description":"图像文件大小上限","type":"inputNumber","defaultVal":100,"min":0}}',
 '', 5120, false, 'system', 'system'),

('image_pair_similarity_filter', '图像对相似度过滤器',
 '过滤器将图像之间具有相似性的图像对保持在特定范围内。Filter to keep image pairs with similarities within a specific range.',
 '1.4.4', 'image', 'image', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于图像相似度计算的HF模型","type":"input","defaultVal":"openai/clip-vit-base-patch32"},"min":{"name":"最小相似度","description":"相似度下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大相似度","description":"相似度上限","type":"slider","defaultVal":1.0,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('image_text_matching_filter', '图文匹配过滤器',
 '过滤器将图像文本匹配分数的样本保持在特定范围内。Filter to keep samples with image-text matching scores within a specific range.',
 '1.4.4', 'multimodal', 'multimodal', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于图文匹配的HF模型","type":"input","defaultVal":"openai/clip-vit-base-patch32"},"min":{"name":"最小匹配分数","description":"匹配分数下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大匹配分数","description":"匹配分数上限","type":"slider","defaultVal":1.0,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('image_text_similarity_filter', '图文相似度过滤器',
 '过滤器将具有图像-文本相似性的样本保持在指定范围内。Filter to keep samples with image-text similarity within a specified range.',
 '1.4.4', 'multimodal', 'multimodal', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于图文相似度计算的HF模型","type":"input","defaultVal":"openai/clip-vit-base-patch32"},"min":{"name":"最小相似度","description":"相似度下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大相似度","description":"相似度上限","type":"slider","defaultVal":1.0,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('image_watermark_filter', '图像水印过滤器',
 '过滤器以保持其图像没有水印的样本具有高概率。Filter to keep samples whose images have no watermark with high probability.',
 '1.4.4', 'image', 'image', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于水印检测的HF模型","type":"input","defaultVal":"ondatel/watermark_model"},"threshold":{"name":"水印阈值","description":"高于此值判定为有水印","type":"slider","defaultVal":0.5,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('image_subplot_filter', '图像子图过滤器',
 '过滤器来检测和删除包含子图的图像样本。Filter to detect and remove samples with images containing subplots.',
 '1.4.4', 'image', 'image', NULL,
'{"threshold":{"name":"子图检测阈值","description":"子图检测的置信度阈值","type":"slider","defaultVal":0.5,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

-- ===== filter (video模态) =====
('video_duration_filter', '视频时长过滤器',
 '保留视频持续时间在指定范围内的数据样本。Keep data samples whose videos durations are within a specified range.',
 '1.4.4', 'video', 'video', NULL,
'{"min_duration":{"name":"最小时长(秒)","description":"视频时长下限","type":"inputNumber","defaultVal":0,"min":0},"max_duration":{"name":"最大时长(秒)","description":"视频时长上限","type":"inputNumber","defaultVal":3600,"min":0}}',
 '', 8192, false, 'system', 'system'),

('video_aspect_ratio_filter', '视频长宽比过滤器',
 '过滤器将视频纵横比的样本保持在特定范围内。Filter to keep samples with video aspect ratio within a specific range.',
 '1.4.4', 'video', 'video', NULL,
'{"min":{"name":"最小长宽比","description":"视频长宽比下限","type":"slider","defaultVal":0.5,"min":0.01,"max":10,"step":0.01},"max":{"name":"最大长宽比","description":"视频长宽比上限","type":"slider","defaultVal":2.0,"min":0.01,"max":10,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('video_resolution_filter', '视频分辨率过滤器',
 '保留视频分辨率在指定范围内的数据样本。Keep data samples whose videos resolutions are within a specified range.',
 '1.4.4', 'video', 'video', NULL,
'{"min_width":{"name":"最小宽度","description":"视频宽度下限(像素)","type":"inputNumber","defaultVal":0,"min":0},"max_width":{"name":"最大宽度","description":"视频宽度上限(像素)","type":"inputNumber","defaultVal":4096,"min":0},"min_height":{"name":"最小高度","description":"视频高度下限(像素)","type":"inputNumber","defaultVal":0,"min":0},"max_height":{"name":"最大高度","description":"视频高度上限(像素)","type":"inputNumber","defaultVal":4096,"min":0}}',
 '', 8192, false, 'system', 'system'),

('video_motion_score_filter', '视频运动得分过滤器',
 '过滤器将来自OpenCV的视频运动分数的样本保持在特定范围内。Filter to keep samples with video motion scores from OpenCV.',
 '1.4.4', 'video', 'video', NULL,
'{"min":{"name":"最小运动分数","description":"运动分数下限","type":"slider","defaultVal":0.0,"min":0,"max":100,"step":0.1},"max":{"name":"最大运动分数","description":"运动分数上限","type":"slider","defaultVal":100.0,"min":0,"max":100,"step":0.1}}',
 '', 8192, false, 'system', 'system'),

('video_nsfw_filter', '视频NSFW过滤器',
 '过滤器以保留其视频的NSFW分数在指定范围内的样本。Filter to keep samples whose videos have NSFW scores in a specified range.',
 '1.4.4', 'video', 'video', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于NSFW检测的HF模型","type":"input","defaultVal":""},"min":{"name":"最小NSFW分数","description":"NSFW分数下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大NSFW分数","description":"NSFW分数上限","type":"slider","defaultVal":0.5,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('video_aesthetics_filter', '视频美学过滤器',
 '过滤器将视频中指定帧的美学得分数据样本保留在特定范围内。Filter to keep samples with aesthetics scores for specified frames.',
 '1.4.4', 'video', 'video', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于美学评分的HF模型","type":"input","defaultVal":""},"min":{"name":"最小美学分数","description":"美学分数下限","type":"slider","defaultVal":0.0,"min":0,"max":10,"step":0.1},"max":{"name":"最大美学分数","description":"美学分数上限","type":"slider","defaultVal":10.0,"min":0,"max":10,"step":0.1}}',
 '', 8192, false, 'system', 'system'),

('video_watermark_filter', '视频水印过滤器',
 '过滤器以保持其视频具有高概率没有水印的样本。Filter to keep samples whose videos have no watermark with high probability.',
 '1.4.4', 'video', 'video', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于水印检测的HF模型","type":"input","defaultVal":""},"threshold":{"name":"水印阈值","description":"高于此值判定为有水印","type":"slider","defaultVal":0.5,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('video_ocr_area_ratio_filter', '视频OCR面积占比过滤器',
 '保留检测到的视频中指定帧的文本面积比率在指定范围内的数据样本。Keep data samples whose detected text area ratios.',
 '1.4.4', 'video', 'video', NULL,
'{"min":{"name":"最小OCR面积比","description":"OCR文本面积占比下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大OCR面积比","description":"OCR文本面积占比上限","type":"slider","defaultVal":1.0,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('video_frames_text_similarity_filter', '视频帧文本相似度过滤器',
 '根据视频帧图像和文本之间的相似性进行过滤。Filter to keep samples based on similarity between video frames and text.',
 '1.4.4', 'multimodal', 'multimodal', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于视频帧文本相似度计算的HF模型","type":"input","defaultVal":"openai/clip-vit-base-patch32"},"min":{"name":"最小相似度","description":"相似度下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大相似度","description":"相似度上限","type":"slider","defaultVal":1.0,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('video_tagging_from_frames_filter', '视频帧标签过滤器',
 '过滤器以保留其视频包含指定标签的样本。Filter to keep samples whose videos contain specified tags.',
 '1.4.4', 'video', 'video', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于视频帧标签生成的HF模型","type":"input","defaultVal":""},"tags":{"name":"指定标签","description":"需要保留的视频标签(逗号分隔)","type":"input","defaultVal":""}}',
 '', 8192, false, 'system', 'system'),

('video_face_ratio_filter', '视频人脸占比过滤器',
 '保留视频的脸帧比在指定范围内的数据样本。Keep data samples whose videos face-to-frame ratios are within a specified range.',
 '1.4.4', 'video', 'video', NULL,
'{"min":{"name":"最小人脸占比","description":"人脸面积占比下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01},"max":{"name":"最大人脸占比","description":"人脸面积占比上限","type":"slider","defaultVal":1.0,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('video_motion_score_raft_filter', '视频RAFT运动得分过滤器',
 '过滤器将视频运动分数的样本保持在指定范围内。Filter to keep samples with video motion scores from RAFT.',
 '1.4.4', 'video', 'video', NULL,
'{"min":{"name":"最小运动分数","description":"运动分数下限","type":"slider","defaultVal":0.0,"min":0,"max":100,"step":0.1},"max":{"name":"最大运动分数","description":"运动分数上限","type":"slider","defaultVal":100.0,"min":0,"max":100,"step":0.1}}',
 '', 8192, false, 'system', 'system'),

('phrase_grounding_recall_filter', '短语定位召回过滤器',
 '根据从图像中的文本中提取的短语接地召回来过滤以保留样本。Filter to keep samples based on phrase grounding recall.',
 '1.4.4', 'multimodal', 'multimodal', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于短语定位的HF模型","type":"input","defaultVal":""},"min_recall":{"name":"最小召回率","description":"召回率下限","type":"slider","defaultVal":0.0,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('video_motion_score_ptlflow_filter', '视频PtlFlow运动得分过滤器',
 '过滤器以将来自ptlflow的具有视频运动分数的样本保持在指定范围内。Filter to keep samples with video motion scores from ptlflow.',
 '1.4.4', 'video', 'video', NULL,
'{"min":{"name":"最小运动分数","description":"运动分数下限","type":"slider","defaultVal":0.0,"min":0,"max":100,"step":0.1},"max":{"name":"最大运动分数","description":"运动分数上限","type":"slider","defaultVal":100.0,"min":0,"max":100,"step":0.1}}',
 '', 8192, false, 'system', 'system'),

-- ===== grouper (3个) =====
('key_value_grouper', '键值分组器',
 '根据指定键中的值将样本分组为批处理。Groups samples into batches based on values in specified keys.',
 '1.4.4', 'text', 'text', NULL,
'{"group_key":{"name":"分组键","description":"用于分组的字段名","type":"input","defaultVal":""}}',
 '', 5120, false, 'system', 'system'),

('naive_grouper', '朴素分组器',
 '将数据集中的所有样本分组为单个批处理样本。Group all samples in a dataset into a single batched sample.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

('naive_reverse_grouper', '朴素反向分组器',
 '将批处理的样品分成单个样品。Split batched samples into individual samples.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

-- ===== selector (6个) =====
('frequency_specified_field_selector', '指定字段频率选择器',
 '选择器根据指定字段的频率过滤样本。Selector to filter samples based on the frequency of a specified field.',
 '1.4.4', 'text', 'text', NULL,
'{"field_key":{"name":"字段名","description":"用于频率统计的字段","type":"input","defaultVal":""},"top_k":{"name":"Top-K","description":"选择频率最高的K个","type":"inputNumber","defaultVal":100,"min":1}}',
 '', 5120, false, 'system', 'system'),

('random_selector', '随机选择器',
 '从数据集中随机选择样本子集。Randomly selects a subset of samples from the dataset.',
 '1.4.4', 'text', 'text', NULL,
'{"num_samples":{"name":"样本数量","description":"随机选择的样本数量","type":"inputNumber","defaultVal":1000,"min":1},"seed":{"name":"随机种子","description":"随机种子","type":"inputNumber","defaultVal":42,"min":0}}',
 '', 5120, false, 'system', 'system'),

('range_specified_field_selector', '指定字段范围选择器',
 '根据指定字段的排序值选择采样范围。Selects a range of samples based on the sorted values of a specified field.',
 '1.4.4', 'text', 'text', NULL,
'{"field_key":{"name":"字段名","description":"用于排序的字段","type":"input","defaultVal":""},"start":{"name":"起始索引","description":"起始索引(包含)","type":"inputNumber","defaultVal":0,"min":0},"end":{"name":"结束索引","description":"结束索引(不包含)","type":"inputNumber","defaultVal":1000,"min":0}}',
 '', 5120, false, 'system', 'system'),

('tags_specified_field_selector', '指定字段标签选择器',
 '选择器根据指定字段的标签过滤样本。Selector to filter samples based on the tags of a specified field.',
 '1.4.4', 'text', 'text', NULL,
'{"field_key":{"name":"字段名","description":"用于标签匹配的字段","type":"input","defaultVal":""},"tags":{"name":"标签列表","description":"需要匹配的标签(逗号分隔)","type":"input","defaultVal":""}}',
 '', 5120, false, 'system', 'system'),

('topk_specified_field_selector', '指定字段TopK选择器',
 '根据指定字段的排序值选择顶部样本。Selects top samples based on the sorted values of a specified field.',
 '1.4.4', 'text', 'text', NULL,
'{"field_key":{"name":"字段名","description":"用于排序的字段","type":"input","defaultVal":""},"top_k":{"name":"Top-K","description":"选择排序值最高的K个样本","type":"inputNumber","defaultVal":100,"min":1},"ascending":{"name":"升序","description":"是否升序排序","type":"switch","defaultVal":false,"checkedLabel":"升序","unCheckedLabel":"降序"}}',
 '', 5120, false, 'system', 'system'),

('image_ohem_selector', '图像难例选择器',
 '保留损失最大的样本（在线难例挖掘）。Keep the highest-loss image samples (Online Hard Example Mining).',
 '1.4.4', 'image', 'image', NULL,
'{"top_k":{"name":"Top-K","description":"选择损失最高的K个样本","type":"inputNumber","defaultVal":100,"min":1},"loss_field":{"name":"损失字段","description":"包含损失值的字段名","type":"input","defaultVal":"loss"}}',
 '', 8192, false, 'system', 'system'),

-- ===== formatter (8个) =====
('csv_formatter', 'CSV格式化器',
 '类用于加载和格式化csv类型的文件。The class is used to load and format csv-type files.',
 '1.4.4', 'text', 'text', NULL,
'{"sep":{"name":"分隔符","description":"CSV文件的分隔符","type":"input","defaultVal":","}}',
 '', 5120, false, 'system', 'system'),

('empty_formatter', '空格式化器',
 '类用于创建空数据。The class is used to create empty data.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

('json_formatter', 'JSON格式化器',
 '加载json类型的文件。Load json-type files.',
 '1.4.4', 'text', 'text', NULL,
'{"jsonl":{"name":"JSONL模式","description":"是否按行读取JSON","type":"switch","defaultVal":true,"checkedLabel":"是","unCheckedLabel":"否"}}',
 '', 5120, false, 'system', 'system'),

('local_formatter', '本地格式化器',
 '类用于从本地文件或本地目录加载数据集。The class is used to load a dataset from local files or local directory.',
 '1.4.4', 'text', 'text', NULL,
'{"load_patterns":{"name":"加载模式","description":"文件加载模式(如*.txt)","type":"input","defaultVal":"*"}}',
 '', 5120, false, 'system', 'system'),

('parquet_formatter', 'Parquet格式化器',
 '该类用于加载和格式化镶木地板类型的文件。The class is used to load and format parquet-type files.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

('remote_formatter', '远程格式化器',
 '该类用于从huggingface hub的存储库加载数据集。The class is used to load a dataset from repository of huggingface hub.',
 '1.4.4', 'text', 'text', NULL,
'{"hf_repo":{"name":"HuggingFace仓库","description":"HuggingFace数据集仓库名称","type":"input","defaultVal":""},"split":{"name":"数据集分割","description":"数据集的分割(如train/test)","type":"input","defaultVal":"train"}}',
 '', 5120, false, 'system', 'system'),

('text_formatter', '文本格式化器',
 '类用于加载和格式化文本类型文件。The class is used to load and format text-type files.',
 '1.4.4', 'text', 'text', NULL,
'{"encoding":{"name":"编码","description":"文本文件的编码","type":"input","defaultVal":"utf-8"}}',
 '', 5120, false, 'system', 'system'),

('tsv_formatter', 'TSV格式化器',
 '该类用于加载和格式化tsv类型的文件。The class is used to load and format tsv-type files.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

-- ===== mapper (精选核心算子, 138个中选取最常用的) =====
('calibrate_qa_mapper', 'QA校准映射器',
 '使用API模型根据参考文本校准问答对。Calibrates question-answer pairs based on reference text using an API model.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"使用的LLM模型","type":"input","defaultVal":"gpt-3.5-turbo"},"prompt":{"name":"校准提示词","description":"QA校准的提示词模板","type":"textarea","defaultVal":""}}',
 '', 8192, false, 'system', 'system'),

('calibrate_query_mapper', '查询校准映射器',
 '基于参考文本校准问答对中的查询。Calibrate query in question-answer pairs based on reference text.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"使用的LLM模型","type":"input","defaultVal":"gpt-3.5-turbo"}}',
 '', 5120, false, 'system', 'system'),

('calibrate_response_mapper', '回复校准映射器',
 '根据参考文本校准问答对中的回答。Calibrate response in question-answer pairs based on reference text.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"使用的LLM模型","type":"input","defaultVal":"gpt-3.5-turbo"}}',
 '', 5120, false, 'system', 'system'),

('chinese_convert_mapper', '中文简繁转换映射器',
 '映射器在繁体、简体和日文汉字之间转换中文文本。Mapper to convert Chinese text between Traditional, Simplified, and Japanese Kanji.',
 '1.4.4', 'text', 'text', NULL,
'{"convert_type":{"name":"转换类型","description":"转换方向","type":"select","defaultVal":"s2t","options":[{"label":"简体转繁体","value":"s2t"},{"label":"繁体转简体","value":"t2s"},{"label":"简体转日文汉字","value":"s2jp"},{"label":"日文汉字转简体","value":"jp2s"}]}}',
 '', 5120, false, 'system', 'system'),

('clean_copyright_mapper', '版权清洗映射器',
 '清除文本示例开头的版权注释。Cleans copyright comments at the beginning of text samples.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

('clean_email_mapper', '邮箱清洗映射器',
 '使用正则表达式从文本示例中清除电子邮件地址。Cleans email addresses from text samples using a regular expression.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

('clean_html_mapper', 'HTML清洗映射器',
 '从文本示例中清除HTML代码，将HTML转换为纯文本。Cleans HTML code from text samples, converting HTML to plain text.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

('clean_ip_mapper', 'IP清洗映射器',
 '从文本示例中清除IPv4和IPv6地址。Cleans IPv4 and IPv6 addresses from text samples.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

('clean_links_mapper', '链接清洗映射器',
 '映射器来清理链接，如文本示例中的http/https/ftp。Mapper to clean links like http/https/ftp in text samples.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

('expand_macro_mapper', '宏展开映射器',
 '展开LaTeX示例文档主体中的宏定义。Expands macro definitions in the document body of LaTeX samples.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

('fix_unicode_mapper', 'Unicode修复映射器',
 '修复文本示例中的unicode错误。Fixes unicode errors in text samples.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),


-- ===== mapper (精选核心算子) =====
('generate_qa_from_examples_mapper', '示例生成QA映射器',
 '使用HuggingFace模型从示例生成问题和答案对。Generates question and answer pairs from examples using a Hugging Face model.',
 '1.4.4', 'text', 'text', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于QA生成的HF模型","type":"input","defaultVal":""},"few_shot_num":{"name":"示例数量","description":"用于few-shot的示例数量","type":"inputNumber","defaultVal":5,"min":0}}',
 '', 8192, false, 'system', 'system'),

('generate_qa_from_text_mapper', '文本生成QA映射器',
 '使用指定的模型从文本生成问题和答案对。Generates question and answer pairs from text using a specified model.',
 '1.4.4', 'text', 'text', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于QA生成的HF模型","type":"input","defaultVal":""},"max_new_tokens":{"name":"最大生成Token数","description":"最大生成的token数量","type":"inputNumber","defaultVal":512,"min":1}}',
 '', 8192, false, 'system', 'system'),

-- ===== Mapper (Image/Video/Multimodal) =====
('image_blur_mapper', '图像模糊映射器',
 '使用指定的概率和模糊类型对数据集中的图像进行模糊处理。Blurs images with a specified probability and blur type.',
 '1.4.4', 'image', 'image', NULL,
'{"prob":{"name":"概率","description":"执行模糊的概率","type":"slider","defaultVal":0.1,"min":0,"max":1,"step":0.01},"blur_type":{"name":"模糊类型","description":"高斯模糊或运动模糊","type":"select","defaultVal":"gaussian","options":[{"label":"高斯","value":"gaussian"},{"label":"运动","value":"motion"}]}}',
 '', 5120, false, 'system', 'system'),

('image_captioning_mapper', '图像描述映射器',
 '使用HuggingFace模型生成图像标题，并将其附加到样本中。Generates image captions using a Hugging Face model.',
 '1.4.4', 'multimodal', 'multimodal', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于图像描述的HF模型","type":"input","defaultVal":"Salesforce/blip-image-captioning-base"},"max_new_tokens":{"name":"最大Token数","description":"生成文本的最大长度","type":"inputNumber","defaultVal":64,"min":1}}',
 '', 8192, false, 'system', 'system'),

('image_detection_yolo_mapper', 'YOLO图像检测映射器',
 '使用YOLO对图像执行对象检测，并返回边界框和类标签。Perform object detection using YOLO.',
 '1.4.4', 'image', 'image', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"YOLO模型路径","type":"input","defaultVal":"ultralytics/yolov8n.pt"},"conf":{"name":"置信度阈值","description":"检测框的置信度阈值","type":"slider","defaultVal":0.25,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('image_diffusion_mapper', '图像扩散生成映射器',
 '使用基于提供的字幕的扩散模型生成图像。Generate images using a diffusion model based on provided captions.',
 '1.4.4', 'multimodal', 'multimodal', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"扩散模型路径","type":"input","defaultVal":"runwayml/stable-diffusion-v1-5"},"num_inference_steps":{"name":"推理步数","description":"生成图像的推理步数","type":"inputNumber","defaultVal":50,"min":1}}',
 '', 8192, false, 'system', 'system'),

('image_face_blur_mapper', '图像人脸模糊映射器',
 '映射器模糊图像中检测到的人脸。Mapper to blur faces detected in images.',
 '1.4.4', 'image', 'image', NULL,
'{"blur_strength":{"name":"模糊强度","description":"高斯模糊的强度","type":"inputNumber","defaultVal":10,"min":0}}',
 '', 5120, false, 'system', 'system'),

('image_remove_background_mapper', '图像去背景映射器',
 '映射器删除图像的背景。Mapper to remove the background of images.',
 '1.4.4', 'image', 'image', NULL,
'{"threshold":{"name":"透明度阈值","description":"Alpha通道阈值","type":"inputNumber","defaultVal":0.5,"min":0,"max":1,"step":0.01}}',
 '', 5120, false, 'system', 'system'),
 
 ('image_segment_mapper', '图像分割映射器',
 '对图像执行segment-anything操作并返回边界框。Perform segment-anything on images and return bounding boxes.',
 '1.4.4', 'image', 'image', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"SAM模型路径","type":"input","defaultVal":""},"points_per_side":{"name":"每边点数","description":"SAM模型每边采样点数","type":"inputNumber","defaultVal":32,"min":1}}',
 '', 8192, false, 'system', 'system'),

('image_tagging_mapper', '图像标签生成映射器',
 'Generates image tags for each image in the sample.',
 '1.4.4', 'image', 'image', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"用于图像打标模型","type":"input","defaultVal":""}}',
 '', 5120, false, 'system', 'system'),

('image_blur_mapper', '图像模糊映射器',
 'Blurs images in the dataset with a specified probability and blur type.',
 '1.4.4', 'image', 'image', NULL,
'{"prob":{"name":"模糊概率","description":"执行模糊操作的概率","type":"slider","defaultVal":0.1,"min":0,"max":1,"step":0.01},"blur_type":{"name":"模糊类型","description":"高斯模糊或运动模糊","type":"select","defaultVal":"gaussian","options":[{"label":"高斯","value":"gaussian"},{"label":"运动","value":"motion"}]}}',
 '', 5120, false, 'system', 'system'),

('image_captioning_mapper', '图像描述生成映射器',
 'Generates image captions using a Hugging Face model.',
 '1.4.4', 'image', 'image,text', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"图像描述生成模型","type":"input","defaultVal":"Salesforce/blip-image-captioning-base"},"max_new_tokens":{"name":"最大生成Token数","description":"生成描述的最大长度","type":"inputNumber","defaultVal":64,"min":1}}',
 '', 8192, false, 'system', 'system'),

('image_detection_yolo_mapper', 'YOLO目标检测映射器',
 'Perform object detection using YOLO on images.',
 '1.4.4', 'image', 'image,annotations', NULL,
'{"hf_model":{"name":"YOLO模型","description":"YOLO模型路径或HuggingFace ID","type":"input","defaultVal":"ultralytics/yolov8n.pt"},"conf":{"name":"置信度阈值","description":"检测框置信度阈值","type":"slider","defaultVal":0.25,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('image_diffusion_mapper', '图像扩散生成映射器',
 'Generate images using a diffusion model based on provided captions.',
 '1.4.4', 'text', 'image', NULL,
'{"hf_model":{"name":"扩散模型","description":"Stable Diffusion模型路径","type":"input","defaultVal":"runwayml/stable-diffusion-v1-5"},"num_inference_steps":{"name":"推理步数","description":"生成图像的推理步数","type":"inputNumber","defaultVal":50,"min":1}}',
 '', 8192, false, 'system', 'system'),

('image_face_blur_mapper', '图像人脸模糊映射器',
 'Mapper to blur faces detected in images.',
 '1.4.4', 'image', 'image', NULL,
'{"blur_strength":{"name":"模糊强度","description":"高斯模糊的强度","type":"inputNumber","defaultVal":10,"min":0}}',
 '', 5120, false, 'system', 'system'),

('image_remove_background_mapper', '图像去背景映射器',
 'Mapper to remove the background of images.',
 '1.4.4', 'image', 'image', NULL,
'{"threshold":{"name":"透明度阈值","description":"Alpha通道阈值，低于此值视为背景","type":"slider","defaultVal":0.5,"min":0,"max":1,"step":0.01}}',
 '', 5120, false, 'system', 'system'),

('image_segment_mapper', '图像分割映射器(SAM)',
 'Perform segment-anything on images and return the bounding boxes.',
 '1.4.4', 'image', 'image,segmentation', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"SAM模型路径","type":"input","defaultVal":""},"points_per_side":{"name":"每边点数","description":"SAM每边采样点数","type":"inputNumber","defaultVal":32,"min":1}}',
 '', 8192, false, 'system', 'system'),

('image_sdxl_prompt2prompt_mapper', 'SDXL Prompt转Prompt映射器',
 'Generates pairs of similar images using the SDXL model.',
 '1.4.4', 'text', 'text,image', NULL,
'{"hf_model":{"name":"SDXL模型","description":"Stable Diffusion XL模型","type":"input","defaultVal":""},"guidance_scale":{"name":"Guidance Scale","description":"生成的指导比例","type":"slider","defaultVal":7.5,"min":1,"max":20,"step":0.1}}',
 '', 8192, false, 'system', 'system'),

('mllm_mapper', '多模态大模型映射器',
 'Mapper to use MLLMs for visual question answering tasks.',
 '1.4.4', 'image,text', 'text', NULL,
'{"hf_model":{"name":"MLLM模型","description":"多模态大模型路径","type":"input","defaultVal":""},"max_new_tokens":{"name":"最大Token数","description":"最大生成的token数量","type":"inputNumber","defaultVal":512,"min":1}}',
 '', 8192, false, 'system', 'system'),

('nlpaug_en_mapper', '英文文本增强映射器',
 'Augments English text samples using various methods.',
 '1.4.4', 'text', 'text', NULL,
'{"action":{"name":"增强操作","description":"增强类型(插入/替换/删除等)","type":"select","defaultVal":"substitute","options":[{"label":"替换","value":"substitute"},{"label":"插入","value":"insert"},{"label":"删除","value":"delete"}]},"aug_max":{"name":"最大增强比例","description":"被增强文本的最大比例","type":"slider","defaultVal":0.1,"min":0,"max":1,"step":0.01},"aug_min":{"name":"最小增强比例","description":"被增强文本的最小比例","type":"slider","defaultVal":0.01,"min":0,"max":1,"step":0.01}}',
 '', 8192, false, 'system', 'system'),

('nlpcda_zh_mapper', '中文文本增强映射器',
 'Augments Chinese text samples using the nlpcda library.',
 '1.4.4', 'text', 'text', NULL,
'{"action":{"name":"增强操作","description":"增强类型","type":"select","defaultVal":"substitute","options":[{"label":"替换","value":"substitute"},{"label":"插入","value":"insert"}]},"create_num":{"name":"增强数量","description":"每条样本生成的增强版本数量","type":"inputNumber","defaultVal":1,"min":1}}',
 '', 8192, false, 'system', 'system'),

('optimize_prompt_mapper', 'Prompt优化映射器',
 'Optimize prompts based on existing ones in the same batch.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"用于优化的LLM模型","type":"input","defaultVal":"gpt-3.5-turbo"}}',
 '', 8192, false, 'system', 'system'),

('optimize_qa_mapper', 'QA优化映射器',
 'Mapper to optimize question-answer pairs.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"用于优化的LLM模型","type":"input","defaultVal":"gpt-3.5-turbo"}}',
 '', 8192, false, 'system', 'system'),

('optimize_query_mapper', '查询优化映射器',
 'Optimize queries in question-answer pairs to make them more specific.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"用于优化的LLM模型","type":"input","defaultVal":"gpt-3.5-turbo"}}',
 '', 5120, false, 'system', 'system'),

('optimize_response_mapper', '回复优化映射器',
 'Optimize response in question-answer pairs to be more detailed.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"用于优化的LLM模型","type":"input","defaultVal":"gpt-3.5-turbo"}}',
 '', 5120, false, 'system', 'system'),

('pair_preference_mapper', '成对偏好构建映射器',
 'Mapper to construct paired preference samples.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"用于生成拒绝响应的模型","type":"input","defaultVal":"gpt-3.5-turbo"}}',
 '', 8192, false, 'system', 'system'),

('pii_redaction_mapper', 'PII匿名化映射器',
 'Redact PII in text.',
 '1.4.4', 'text', 'text', NULL,
'{"anonymization_type":{"name":"匿名化类型","description":"替换或掩盖","type":"select","defaultVal":"mask","options":[{"label":"掩盖","value":"mask"},{"label":"替换","value":"replace"}]}}',
 '', 5120, false, 'system', 'system'),

('punctuation_normalization_mapper', '标点符号规范化映射器',
 'Normalizes unicode punctuations to their English equivalents.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

('python_file_mapper', 'Python脚本映射器',
 'Executes a Python function defined in a file on input data.',
 '1.4.4', 'text', 'text', NULL,
'{"func_file":{"name":"脚本文件","description":"包含处理函数的Python文件路径","type":"input","defaultVal":""},"func_name":{"name":"函数名","description":"要调用的函数名称","type":"input","defaultVal":"process"}}',
 '', 5120, false, 'system', 'system'),

('python_lambda_mapper', 'Python Lambda映射器',
 'Mapper for applying a Python lambda function to data samples.',
 '1.4.4', 'text', 'text', NULL,
'{"lambda_func":{"name":"Lambda表达式","description":"Python lambda表达式字符串","type":"textarea","defaultVal":"lambda x: x"}}',
 '', 5120, false, 'system', 'system'),

('query_intent_detection_mapper', '查询意图检测映射器',
 'Predicts the user intent label for a given query.',
 '1.4.4', 'text', 'text', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"意图分类模型","type":"input","defaultVal":""}}',
 '', 8192, false, 'system', 'system'),

('query_sentiment_detection_mapper', '查询情绪检测映射器',
 'Predicts user sentiment label in a query.',
 '1.4.4', 'text', 'text', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"情绪分类模型","type":"input","defaultVal":""}}',
 '', 8192, false, 'system', 'system'),

('query_topic_detection_mapper', '查询主题检测映射器',
 'Predicts the topic label for a given query.',
 '1.4.4', 'text', 'text', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"主题分类模型","type":"input","defaultVal":""}}',
 '', 8192, false, 'system', 'system'),

('relation_identity_mapper', '实体关系识别映射器',
 'Identify the relation between two entities in a given text.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"用于关系识别的LLM","type":"input","defaultVal":"gpt-3.5-turbo"}}',
 '', 8192, false, 'system', 'system'),

('remove_bibliography_mapper', '参考文献去除映射器',
 'Removes bibliography sections at the end of LaTeX documents.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

('remove_comments_mapper', '注释去除映射器',
 'Removes comments from documents.',
 '1.4.4', 'text', 'text', NULL,
'{"comment_type":{"name":"注释类型","description":"要移除的注释类型","type":"select","defaultVal":"all","options":[{"label":"全部","value":"all"},{"label":"行内","value":"inline"},{"label":"块","value":"block"}]}}',
 '', 5120, false, 'system', 'system'),

('remove_header_mapper', '头部去除映射器',
 'Removes headers at the beginning of documents.',
 '1.4.4', 'text', 'text', NULL,
'{"header_section":{"name":"头部Section","description":"要移除的Section名称","type":"input","defaultVal":""}}',
 '', 5120, false, 'system', 'system'),

('remove_long_words_mapper', '长词去除映射器',
 'Mapper to remove long words within a specific range.',
 '1.4.4', 'text', 'text', NULL,
'{"min_len":{"name":"最小长度","description":"单词长度下限","type":"inputNumber","defaultVal":20,"min":1},"max_len":{"name":"最大长度","description":"单词长度上限","type":"inputNumber","defaultVal":1000,"min":1}}',
 '', 5120, false, 'system', 'system'),

('remove_non_chinese_character_mapper', '非中文字符去除映射器',
 'Removes non-Chinese characters from text samples.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

('remove_repeat_sentences_mapper', '重复句子去除映射器',
 'Mapper to remove repeat sentences in text samples.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

('remove_specific_chars_mapper', '特定字符去除映射器',
 'Removes specific characters from text samples.',
 '1.4.4', 'text', 'text', NULL,
'{"chars_to_remove":{"name":"要移除的字符","description":"需要移除的字符集合","type":"input","defaultVal":""}}',
 '', 5120, false, 'system', 'system'),

('remove_table_text_mapper', '表格文本去除映射器',
 'Mapper to remove table texts from text samples.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

('remove_words_with_incorrect_substrings_mapper', '含错子串词去除映射器',
 'Mapper to remove words containing specified incorrect substrings.',
 '1.4.4', 'text', 'text', NULL,
'{"substrings":{"name":"错误子串列表","description":"包含这些子串的单词将被移除","type":"input","defaultVal":""}}',
 '', 5120, false, 'system', 'system'),

('replace_content_mapper', '内容替换映射器',
 'Replaces content matching a regex pattern.',
 '1.4.4', 'text', 'text', NULL,
'{"regex":{"name":"正则表达式","description":"匹配要替换内容的正则","type":"input","defaultVal":""},"replacement":{"name":"替换内容","description":"用于替换的字符串","type":"input","defaultVal":""}}',
 '', 5120, false, 'system', 'system'),

('s3_download_file_mapper', 'S3下载映射器',
 'Mapper to download files from S3.',
 '1.4.4', 'text', 'text', NULL,
'{"s3_endpoint":{"name":"S3端点","description":"S3服务地址","type":"input","defaultVal":""},"aws_access_key":{"name":"Access Key","description":"AWS访问密钥","type":"input","defaultVal":""},"aws_secret_key":{"name":"Secret Key","description":"AWS密钥","type":"input","defaultVal":""}}',
 '', 5120, false, 'system', 'system'),

('s3_upload_file_mapper', 'S3上传映射器',
 'Mapper to upload local files to S3.',
 '1.4.4', 'text', 'text', NULL,
'{"s3_endpoint":{"name":"S3端点","description":"S3服务地址","type":"input","defaultVal":""},"aws_access_key":{"name":"Access Key","description":"AWS访问密钥","type":"input","defaultVal":""},"aws_secret_key":{"name":"Secret Key","description":"AWS密钥","type":"input","defaultVal":""}}',
 '', 5120, false, 'system', 'system'),

('sdxl_prompt2prompt_mapper', 'SDXL提示词映射器',
 'Generates pairs of similar images using the SDXL model.',
 '1.4.4', 'text', 'text', NULL,
'{"hf_model":{"name":"SDXL模型","description":"模型路径","type":"input","defaultVal":""},"guidance_scale":{"name":"Guidance Scale","description":"生成引导比例","type":"slider","defaultVal":7.5,"min":1,"max":20,"step":0.1}}',
 '', 5120, false, 'system', 'system'),

('sentence_augmentation_mapper', '句子增强映射器',
 'Augments sentences by generating enhanced versions using a Hugging Face model.',
 '1.4.4', 'text', 'text', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"增强模型","type":"input","defaultVal":""}}',
 '', 8192, false, 'system', 'system'),

('sentence_split_mapper', '句子分割映射器',
 'Splits text samples into individual sentences.',
 '1.4.4', 'text', 'text', NULL,
'{"language":{"name":"语言","description":"句子分割使用的语言","type":"select","defaultVal":"english","options":[{"label":"English","value":"english"},{"label":"Chinese","value":"chinese"}]}}',
 '', 5120, false, 'system', 'system'),

('text_chunk_mapper', '文本分块映射器',
 'Split input text into chunks based on specified criteria.',
 '1.4.4', 'text', 'text', NULL,
'{"chunk_size":{"name":"块大小","description":"每个chunk的最大token数","type":"inputNumber","defaultVal":1024,"min":1},"chunk_overlap":{"name":"重叠大小","description":"chunk之间的重叠token数","type":"inputNumber","defaultVal":0,"min":0}}',
 '', 8192, false, 'system', 'system'),

('whitespace_normalization_mapper', '空白符规范化映射器',
 'Normalizes various types of whitespace characters to standard spaces.',
 '1.4.4', 'text', 'text', NULL,
 NULL,
 '', 5120, false, 'system', 'system'),

-- ===== Video Mappers =====
('video_extract_frames_mapper', '视频抽帧映射器',
 'Mapper to extract frames from video files.',
 '1.4.4', 'video', 'video,image', NULL,
'{"num_frames":{"name":"抽取帧数","description":"从视频中抽取的帧数","type":"inputNumber","defaultVal":1,"min":1},"strategy":{"name":"抽取策略","description":"帧抽取策略","type":"select","defaultVal":"uniform","options":[{"label":"均匀","value":"uniform"},{"label":"关键帧","value":"keyframe"}]}}',
 '', 8192, false, 'system', 'system'),

('video_face_blur_mapper', '视频人脸模糊映射器',
 'Mapper to blur faces detected in videos.',
 '1.4.4', 'video', 'video', NULL,
'{"blur_strength":{"name":"模糊强度","description":"高斯模糊强度","type":"inputNumber","defaultVal":10,"min":0}}',
 '', 5120, false, 'system', 'system'),

('video_remove_watermark_mapper', '视频去水印映射器',
 'Remove watermarks from videos based on specified regions.',
 '1.4.4', 'video', 'video', NULL,
'{"region":{"name":"水印区域","description":"水印位置坐标(x,y,w,h)","type":"input","defaultVal":"0,0,100,100"}}',
 '', 5120, false, 'system', 'system'),

('video_resize_aspect_ratio_mapper', '视频长宽比调整映射器',
 'Resizes videos to fit within a specified aspect ratio range.',
 '1.4.4', 'video', 'video', NULL,
'{"min_aspect_ratio":{"name":"最小长宽比","description":"最小允许的长宽比","type":"slider","defaultVal":0.5,"min":0,"max":5,"step":0.1},"max_aspect_ratio":{"name":"最大长宽比","description":"最大允许的长宽比","type":"slider","defaultVal":2.0,"min":0,"max":5,"step":0.1}}',
 '', 5120, false, 'system', 'system'),

('video_resize_resolution_mapper', '视频分辨率调整映射器',
 'Resizes video resolution based on specified constraints.',
 '1.4.4', 'video', 'video', NULL,
'{"target_width":{"name":"目标宽度","description":"目标视频宽度","type":"inputNumber","defaultVal":1920,"min":1},"target_height":{"name":"目标高度","description":"目标视频高度","type":"inputNumber","defaultVal":1080,"min":1}}',
 '', 5120, false, 'system', 'system'),

('video_split_by_duration_mapper', '视频按时长分割映射器',
 'Splits videos into segments based on a specified duration.',
 '1.4.4', 'video', 'video', NULL,
'{"duration":{"name":"分割时长(秒)","description":"每段视频的时长","type":"inputNumber","defaultVal":10,"min":1}}',
 '', 5120, false, 'system', 'system'),

('video_split_by_key_frame_mapper', '视频按关键帧分割映射器',
 'Splits a video into segments based on key frames.',
 '1.4.4', 'video', 'video', NULL,
'{"threshold":{"name":"差异阈值","description":"关键帧检测的差异阈值","type":"slider","defaultVal":30.0,"min":0,"max":100,"step":0.1}}',
 '', 5120, false, 'system', 'system'),

('video_split_by_scene_mapper', '视频按场景分割映射器',
 'Splits videos into scene clips based on detected scene changes.',
 '1.4.4', 'video', 'video', NULL,
'{"threshold":{"name":"场景切换阈值","description":"检测场景切换的阈值","type":"slider","defaultVal":30.0,"min":0,"max":100,"step":0.1}}',
 '', 5120, false, 'system', 'system'),

('video_tagging_from_audio_mapper', '视频音频标签映射器',
 'Generates video tags from audio streams using the Audio Spectrogram Transformer.',
 '1.4.4', 'video', 'video,text', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"音频分类模型","type":"input","defaultVal":""}}',
 '', 8192, false, 'system', 'system'),

('video_tagging_from_frames_mapper', '视频帧标签映射器',
 'Generates video tags from frames extracted from videos.',
 '1.4.4', 'video', 'video,text', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"图像分类模型","type":"input","defaultVal":""}}',
 '', 8192, false, 'system', 'system'),

('video_captioning_from_audio_mapper', '视频音频字幕映射器',
 'Mapper to caption a video according to its audio streams.',
 '1.4.4', 'video', 'video,text', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"语音转文本模型","type":"input","defaultVal":""}}',
 '', 8192, false, 'system', 'system'),

('video_captioning_from_frames_mapper', '视频帧字幕映射器',
 'Generates video captions from sampled frames.',
 '1.4.4', 'video', 'video,text', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"图文模型","type":"input","defaultVal":""}}',
 '', 8192, false, 'system', 'system'),

('video_captioning_from_summarizer_mapper', '视频摘要字幕映射器',
 'Mapper to generate video captions by summarizing several kinds of generated texts.',
 '1.4.4', 'video', 'video,text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"用于摘要的LLM","type":"input","defaultVal":"gpt-3.5-turbo"}}',
 '', 8192, false, 'system', 'system'),

('video_captioning_from_video_mapper', '视频端到端字幕映射器',
 'Generates video captions using a Hugging Face video-to-text model.',
 '1.4.4', 'video', 'video,text', NULL,
'{"hf_model":{"name":"HuggingFace模型","description":"视频到文本模型","type":"input","defaultVal":""}}',
 '', 8192, false, 'system', 'system'),

('llm_extract_mapper', 'LLM结构化提取映射器',
 'Extract structured fields from text using an LLM.',
 '1.4.4', 'text', 'text', NULL,
'{"api_key":{"name":"API密钥","description":"LLM API密钥","type":"input","defaultVal":""},"model":{"name":"模型名称","description":"用于提取的LLM","type":"input","defaultVal":"gpt-3.5-turbo"},"prompt":{"name":"提取提示词","description":"定义提取格式和字段的提示词","type":"textarea","defaultVal":""}}',
 '', 8192, false, 'system', 'system'),

-- ===== pipeline (3个) =====
('llm_ray_vllm_engine_pipeline', 'LLM Ray vLLM引擎管道',
 'Pipeline to generate response using vLLM engine on Ray.',
 '1.4.4', 'text', 'text', NULL,
'{"vllm_model":{"name":"vLLM模型","description":"要加载的模型名称","type":"input","defaultVal":""},"tensor_parallel_size":{"name":"张量并行度","description":"vLLM的张量并行大小","type":"inputNumber","defaultVal":1,"min":1}}',
 '', 8192, false, 'system', 'system'),

('ray_repartition_pipeline', 'Ray重分区管道',
 'Repartition a Ray Dataset into a target number of blocks.',
 '1.4.4', 'text', 'text', NULL,
'{"target_num_blocks":{"name":"目标块数","description":"重分区后的块数量","type":"inputNumber","defaultVal":10,"min":1}}',
 '', 5120, false, 'system', 'system'),

('vlm_ray_vllm_engine_pipeline', 'VLM Ray vLLM引擎管道',
 'Pipeline to generate response using vLLM engine on Ray.',
 '1.4.4', 'image', 'image', NULL,
'{"vllm_model":{"name":"vLLM模型","description":"多模态模型名称","type":"input","defaultVal":""},"tensor_parallel_size":{"name":"张量并行度","description":"vLLM的张量并行大小","type":"inputNumber","defaultVal":1,"min":1}}',
 '', 8192, false, 'system', 'system')

-- End of Operator Inserts
;

-- ================================================================
-- 插入 Data-Juicer 算子的分类关联关系 (t_operator_category_relation)
-- ================================================================
INSERT INTO t_operator_category_relation(category_id, operator_id)
VALUES
-- aggregator 关联 (功能: 功能)
('8c09476a-a922-418f-a908-733f8a0de521', 'entity_attribute_aggregator'),
('8c09476a-a922-418f-a908-733f8a0de521', 'meta_tags_aggregator'),
('8c09476a-a922-418f-a908-733f8a0de521', 'most_relevant_entities_aggregator'),
('8c09476a-a922-418f-a908-733f8a0de521', 'nested_aggregator'),

-- deduplicator 关联
('8c09476a-a922-418f-a908-733f8a0de521', 'document_deduplicator'),
('8c09476a-a922-418f-a908-733f8a0de521', 'document_minhash_deduplicator'),
('8c09476a-a922-418f-a908-733f8a0de521', 'document_simhash_deduplicator'),
('8c09476a-a922-418f-a908-733f8a0de521', 'image_deduplicator'),
('8c09476a-a922-418f-a908-733f8a0de521', 'video_deduplicator'),
('8c09476a-a922-418f-a908-733f8a0de521', 'ray_bts_minhash_deduplicator'),

-- filter (text) 关联
('8c09476a-a922-418f-a908-733f8a0de521', 'alphanumeric_filter'),
('8c09476a-a922-418f-a908-733f8a0de521', 'average_line_length_filter'),
('8c09476a-a922-418f-a908-733f8a0de521', 'character_repetition_filter'),
('8c09476a-a922-418f-a908-733f8a0de521', 'flagged_words_filter'),
('8c09476a-a922-418f-a908-733f8a0de521', 'text_length_filter'),
('8c09476a-a922-418f-a908-733f8a0de521', 'token_num_filter'),
('8c09476a-a922-418f-a908-733f8a0de521', 'language_id_score_filter'),
('8c09476a-a922-418f-a908-733f8a0de521', 'stopwords_filter'),
('8c09476a-a922-418f-a908-733f8a0de521', 'llm_condition_filter'),

-- filter (image/video/audio) 关联
('8c09476a-a922-418f-a908-733f8a0de521', 'audio_duration_filter'),
('8c09476a-a922-418f-a908-733f8a0de521', 'image_aesthetics_filter'),
('8c09476a-a922-418f-a908-733f8a0de521', 'image_nsfw_filter'),
('8c09476a-a922-418f-a908-733f8a0de521', 'video_duration_filter'),
('8c09476a-a922-418f-a908-733f8a0de521', 'video_nsfw_filter'),

-- grouper 关联
('8c09476a-a922-418f-a908-733f8a0de521', 'key_value_grouper'),
('8c09476a-a922-418f-a908-733f8a0de521', 'naive_grouper'),

-- selector 关联
('8c09476a-a922-418f-a908-733f8a0de521', 'random_selector'),
('8c09476a-a922-418f-a908-733f8a0de521', 'topk_specified_field_selector'),

-- formatter 关联
('8c09476a-a922-418f-a908-733f8a0de521', 'csv_formatter'),
('8c09476a-a922-418f-a908-733f8a0de521', 'json_formatter'),

-- mapper 关联 (部分精选)
('8c09476a-a922-418f-a908-733f8a0de521', 'generate_qa_from_text_mapper'),
('8c09476a-a922-418f-a908-733f8a0de521', 'chinese_convert_mapper'),
('8c09476a-a922-418f-a908-733f8a0de521', 'image_captioning_mapper'),
('8c09476a-a922-418f-a908-733f8a0de521', 'image_diffusion_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'image_detection_yolo_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'image_face_blur_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'image_remove_background_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'image_segment_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'image_tagging_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'image_blur_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'image_sdxl_prompt2prompt_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'mllm_mapper'),
    
    -- Text Mappers 关联
    ('8c09476a-a922-418f-a908-733f8a0de521', 'nlpaug_en_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'nlpcda_zh_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'optimize_prompt_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'optimize_qa_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'optimize_query_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'optimize_response_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'pair_preference_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'pii_redaction_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'punctuation_normalization_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'python_file_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'python_lambda_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'query_intent_detection_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'query_sentiment_detection_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'query_topic_detection_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'relation_identity_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'remove_bibliography_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'remove_comments_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'remove_header_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'remove_long_words_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'remove_non_chinese_character_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'remove_repeat_sentences_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'remove_specific_chars_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'remove_table_text_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'remove_words_with_incorrect_substrings_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'replace_content_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 's3_download_file_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 's3_upload_file_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'sentence_augmentation_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'sentence_split_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'text_chunk_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'whitespace_normalization_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'extract_keyword_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'generate_qa_from_examples_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'llm_extract_mapper'),
    
    -- Video Mappers 关联
    ('8c09476a-a922-418f-a908-733f8a0de521', 'video_extract_frames_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'video_face_blur_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'video_remove_watermark_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'video_resize_aspect_ratio_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'video_resize_resolution_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'video_split_by_duration_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'video_split_by_key_frame_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'video_split_by_scene_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'video_tagging_from_audio_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'video_tagging_from_frames_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'video_captioning_from_audio_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'video_captioning_from_frames_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'video_captioning_from_summarizer_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'video_captioning_from_video_mapper'),
    
    -- Audio Mappers 关联
    ('8c09476a-a922-418f-a908-733f8a0de521', 'audio_add_gaussian_noise_mapper'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'audio_ffmpeg_wrapped_mapper'),
    
    -- Pipeline 关联 (管道)
    ('8c09476a-a922-418f-a908-733f8a0de521', 'llm_ray_vllm_engine_pipeline'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'ray_repartition_pipeline'),
    ('8c09476a-a922-418f-a908-733f8a0de521', 'vlm_ray_vllm_engine_pipeline')

ON CONFLICT DO NOTHING;

-- 提交事务
COMMIT;