deepdata: core product code (backend, frontend, runtime, deepflow module)

runtime/ops/filter/__init__.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+
+import sys
+from pathlib import Path
+from deepdata.common.utils.custom_importer import CustomImporter
+
+
+def _configure_importer():
+    base_path = Path(__file__).resolve().parent
+    sys.meta_path.append(CustomImporter(base_path))
+
+
+_configure_importer()
+
+
+def _import_operators():
+    from . import file_with_high_repeat_phrase_rate_filter
+    from . import file_with_high_repeat_word_rate_filter
+    from . import file_with_high_special_char_rate_filter
+    from . import remove_file_with_many_sensitive_words
+    from . import remove_file_with_short_or_long_length
+    from . import remove_duplicate_file
+    from . import img_blurred_images_cleaner
+    from . import img_duplicated_images_cleaner
+    from . import img_similar_images_cleaner
+    from . import img_advertisement_images_cleaner
+
+
+_import_operators()

runtime/ops/filter/file_with_high_repeat_phrase_rate_filter/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from deepdata.core.base_op import OPERATORS
+
+OPERATORS.register_module(module_name='FileWithHighRepeatPhraseRateFilter',
+                          module_path="ops.filter.file_with_high_repeat_phrase_rate_filter.process")

runtime/ops/filter/file_with_high_repeat_phrase_rate_filter/metadata.yml

@@ -0,0 +1,31 @@
+name: '文档词重复率检查'
+description: '去除重复词过多的文档。'
+language: 'Python'
+vendor: 'Huawei'
+raw_id: 'FileWithHighRepeatPhraseRateFilter'
+version: '1.0.0'
+types:
+  - 'cleanse'
+modal: 'text'
+effect:
+  before: '机器机器机器机器机器机器机器机器机器机器学习学习学习学习学习'
+  after: ''
+inputs: 'text'
+outputs: 'text'
+settings:
+  repeatPhraseRatio:
+    name: 文档词重复率
+    description: 某个词的统计数/文档总词数 > 设定值，该文档被去除。
+    type: slider
+    defaultVal: 0.5
+    min: 0
+    max: 1
+    step: 0.1
+  hitStopwords:
+    name: 去除停用词
+    description: 统计重复词时，选择是否要去除停用词。
+    type: switch
+    defaultVal: false
+    required: true
+    checkedLabel: 去除
+    unCheckedLabel: 不去除

runtime/ops/filter/file_with_high_repeat_phrase_rate_filter/process.py

@@ -0,0 +1,74 @@
+#!/user/bin/python
+# -- encoding: utf-8 --
+
+"""
+Description: 词重复率过高文档过滤插件
+Create: 2023/11/7 9:26
+"""
+import re
+import time
+
+from collections import Counter
+from pathlib import Path
+from typing import Dict, Any
+from loguru import logger
+
+import jieba
+from deepdata.core.base_op import Filter
+
+
+class FileWithHighRepeatPhraseRateFilter(Filter):
+    """词重复率过高文档过滤插件"""
+    PUNCTUATION_PATTERN = re.compile(r'^[\u3000-\u303F\uff00-\uffef\s\W_]+$')
+
+    def __init__(self, *args, **kwargs):
+        super(FileWithHighRepeatPhraseRateFilter, self).__init__(*args, **kwargs)
+        self._min_threshold = kwargs.get("repeatPhraseRatio", 0.5)  # 重复词符占全文的比例阈值，默认值为0.5
+        self._hit_stopword_trigger = kwargs.get("hitStopwords", False)  # 计算重复词率时是否去除停用词，默认为False不去除，True为去除
+        self._file_path = Path(__file__).parent / 'resources' / 'hit_stopwords.txt'
+        self._hit_stopwords = []
+        if self._hit_stopword_trigger:
+            with open(self._file_path, 'r', encoding='utf-8') as f:
+                self._hit_stopwords = f.read().splitlines()
+
+    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
+        start = time.time()
+        self.read_file_first(sample)
+        sample[self.text_key] = self._file_with_high_repeat_phrase_rate_filter(sample[self.text_key],
+                                                                               sample[self.filename_key])
+        logger.info(f"fileName: {sample[self.filename_key]}, "
+                    f"method: FileWithHighRepeatPhraseRateFilter costs {(time.time() - start):6f} s")
+        return sample
+
+    def _tokenize_by_jieba(self, text: str):
+        """基于jieba对输入文本进行切分
+
+        Args:
+            text: 输入文档内容
+        Returns:
+            words_list: 切割后的词列表
+        """
+
+        for word in jieba.lcut(text):
+            if not self.PUNCTUATION_PATTERN.match(word) and word not in self._hit_stopwords:
+                yield word
+
+    def _file_with_high_repeat_phrase_rate_filter(self, input_data: str, file_name):
+        if len(input_data) < 2:  # 词语长度至少2个字符
+            return input_data
+        words_list = self._tokenize_by_jieba(input_data)
+        words_count = dict(Counter(words_list))
+        words_count_max, words_total_count = 0, 0
+        for words in words_count:
+            # 只统计中文、字母，且长度大于1的词语
+            if len(words) > 1 and words.isalpha():
+                words_count_max = max(words_count_max, words_count.get(words))
+                words_total_count += words_count.get(words)
+        output_data = input_data
+        repeat_phrase_rate = words_count_max / words_total_count if words_total_count > 0 else 0
+        if repeat_phrase_rate >= self._min_threshold:
+            # 只要有一个词重复率高于阈值，就会过滤文档
+            output_data = ""
+            logger.info(f"The repeat phrase rate of the input data is {repeat_phrase_rate}. "
+                        f"Threshold is {self._min_threshold}. The document {file_name} is filtered.")
+        return output_data

runtime/ops/filter/file_with_high_repeat_word_rate_filter/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from deepdata.core.base_op import OPERATORS
+
+OPERATORS.register_module(module_name='FileWithHighRepeatWordRateFilter',
+                          module_path="ops.filter.file_with_high_repeat_word_rate_filter.process")

runtime/ops/filter/file_with_high_repeat_word_rate_filter/metadata.yml

@@ -0,0 +1,25 @@
+name: '文档字重复率检查'
+name_en: 'Word Repetition Rate Check'
+description: '去除重复字过多的文档。'
+description_en: 'Filters out files that contain excessive repeated words.'
+language: 'python'
+vendor: 'huawei'
+raw_id: 'FileWithHighRepeatWordRateFilter'
+version: '1.0.0'
+types:
+  - 'cleanse'
+modal: 'text'
+effect:
+  before: '机器学学学学学学学学学学学学学学学学学学学学学学学学学学学学学学习'
+  after: ''
+inputs: 'text'
+outputs: 'text'
+settings:
+  repeatWordRatio:
+    name: 文档字重复率
+    description: 某个字的统计数/文档总字数 > 设定值，该文档被去除。
+    type: slider
+    defaultVal: 0.5
+    min: 0
+    max: 1
+    step: 0.1

runtime/ops/filter/file_with_high_repeat_word_rate_filter/process.py

@@ -0,0 +1,52 @@
+#!/user/bin/python
+# -- encoding: utf-8 --
+
+"""
+Description: 检查文档字重复率插件
+Create: 2023/11/7 9:26
+"""
+import re
+import time
+
+from collections import Counter
+from typing import Dict, Any
+from loguru import logger
+
+from deepdata.core.base_op import Filter
+
+
+class FileWithHighRepeatWordRateFilter(Filter):
+    """检查文档字重复率插件"""
+
+    def __init__(self, *args, **kwargs):
+        super(FileWithHighRepeatWordRateFilter, self).__init__(*args, **kwargs)
+        self._min_threshold = kwargs.get("repeatWordRatio", 0.5)  # 重复字符占整行的比例阈值，默认值为0.5
+
+    @staticmethod
+    def _extract_word(input_data):
+        # 只统计中文字的重复率
+        extracted_word = re.sub(r'[^\u4e00-\u9fff]', '', input_data)
+        return extracted_word
+
+    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
+        start = time.time()
+        self.read_file_first(sample)
+        sample[self.text_key] = self._file_with_high_repeat_word_rate_filter(sample[self.text_key],
+                                                                             sample[self.filename_key])
+        logger.info(f"fileName: {sample[self.filename_key]}, "
+                    f"method: FileWithHighRepeatWordRateFilter costs {(time.time() - start):6f} s")
+        return sample
+
+    def _file_with_high_repeat_word_rate_filter(self, input_data: str, file_name):
+        tmp = self._extract_word(input_data)
+        if not tmp:
+            return input_data
+        output_data = input_data
+        words_count = Counter(tmp)
+        max_value = max(words_count.values())
+        repeat_word_rate = max_value / len(tmp)
+        if repeat_word_rate >= self._min_threshold:
+            output_data = ""
+            logger.info(f"The repeat word rate of the input data is {repeat_word_rate}. "
+                        f"Threshold is {self._min_threshold}. The document %s is filtered.")
+        return output_data

runtime/ops/filter/file_with_high_special_char_rate_filter/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from deepdata.core.base_op import OPERATORS
+
+OPERATORS.register_module(module_name='FileWithHighSpecialCharRateFilter',
+                          module_path="ops.filter.file_with_high_special_char_rate_filter.process")

runtime/ops/filter/file_with_high_special_char_rate_filter/metadata.yml

@@ -0,0 +1,25 @@
+name: '文档特殊字符率检查'
+name_en: 'Special Character Rate Check'
+description: '去除特殊字符过多的文档。'
+description_en: 'Filters out files that contain excessive special characters.'
+language: 'python'
+vendor: 'huawei'
+raw_id: 'FileWithHighSpecialCharRateFilter'
+version: '1.0.0'
+types:
+  - 'cleanse'
+modal: 'text'
+effect:
+  before: '你好！@！@#！￥！@#'
+  after: ''
+inputs: 'text'
+outputs: 'text'
+settings:
+  specialCharRatio:
+    name: 文档特殊字符率
+    description: 特殊字符的统计数/文档总字数 > 设定值，该文档被去除。
+    type: slider
+    defaultVal: 0.3
+    min: 0
+    max: 1
+    step: 0.1

runtime/ops/filter/file_with_high_special_char_rate_filter/process.py

@@ -0,0 +1,50 @@
+#!/user/bin/python
+# -- encoding: utf-8 --
+
+"""
+Description: 文档特殊字符率检查
+Create: 2023/11/7 9:26
+"""
+import time
+
+from pathlib import Path
+from typing import Dict, Any
+from loguru import logger
+
+from deepdata.core.base_op import Filter
+
+
+class FileWithHighSpecialCharRateFilter(Filter):
+    """检查文档特殊字符率"""
+
+    def __init__(self, *args, **kwargs):
+        super(FileWithHighSpecialCharRateFilter, self).__init__(*args, **kwargs)
+        self._min_threshold = kwargs.get("specialCharRatio", 0.3)  # 特殊字符占全文比例阈值，默认值为0.3
+        self._file_path = Path(__file__).parent / 'resources' / 'special_token.txt'
+        with open(self._file_path, 'r', encoding='utf-8') as f:
+            self._special_token = set(f.read().splitlines())
+
+    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
+        start = time.time()
+        self.read_file_first(sample)
+        sample[self.text_key] = self._file_with_high_special_char_rate_filter(sample[self.text_key],
+                                                                              sample[self.filename_key])
+        logger.info(f"fileName: {sample[self.filename_key]}, "
+                    f"method: FileWithHighSpecialCharRateFilter costs {(time.time() - start):6f} s")
+        return sample
+
+    def _file_with_high_special_char_rate_filter(self, input_data: str, file_name):
+        if not input_data:
+            return ""
+
+        output_data = input_data
+        total = 0
+        for token in self._special_token:
+            total += input_data.count(token)
+
+        special_char_rate = total / len(input_data)
+        if special_char_rate >= self._min_threshold:
+            logger.info(f"The special char rate of the input data is {special_char_rate}. "
+                        f"Threshold is {self._min_threshold}. The document {file_name} is filtered.")
+            output_data = ""
+        return output_data

runtime/ops/filter/file_with_high_special_char_rate_filter/resources/special_token.txt

@@ -0,0 +1,50 @@
+~
+·
+！
+@
+#
+￥
+%
+…
+&
+*
+（
+）
+—
++
+-
+=
+{
+}
+|
+【
+】
+、
+：
+“
+；
+‘
+《
+》
+？
+，
+。
+`
+!
+$
+^
+(
+)
+_
+[
+]
+\
+:
+"
+;
+'
+<
+>
+?
+,
+/

runtime/ops/filter/img_advertisement_images_cleaner/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from deepdata.core.base_op import OPERATORS
+
+OPERATORS.register_module(module_name='ImgAdvertisementImagesCleaner',
+                          module_path="ops.filter.img_advertisement_images_cleaner.process")

runtime/ops/filter/img_advertisement_images_cleaner/metadata.yml

@@ -0,0 +1,16 @@
+name: '广告图片过滤'
+name_en: 'Ad Image Filter'
+description: '去除包含二维码的图片。'
+description_en: 'Removes images containing QR codes.'
+language: 'python'
+vendor: 'huawei'
+raw_id: 'ImgAdvertisementImagesCleaner'
+version: '1.0.0'
+types:
+  - 'cleanse'
+modal: 'image'
+effect:
+  before: ''
+  after: ''
+inputs: 'image'
+outputs: 'image'

runtime/ops/filter/img_advertisement_images_cleaner/process.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+"""
+Description:
+Create: 2024/1/22 20:49
+"""
+import time
+from typing import Dict, Any
+
+import cv2
+import numpy as np
+from loguru import logger
+
+from deepdata.common.utils import bytes_transform
+from deepdata.core.base_op import Filter
+
+from .wechat_qrcode_model import WechatQRCodeModel
+
+
+class ImgAdvertisementImagesCleaner(Filter):
+    """去除广告图片的插件，当前仅支持去除二维码"""
+
+    def __init__(self, *args, **kwargs):
+        super(ImgAdvertisementImagesCleaner, self).__init__(*args, **kwargs)
+        self.img_resize = 1000  # 大图片的最长边压缩为1000
+        self.use_model = True
+        self.model = self.get_model(*args, **kwargs)
+
+    @staticmethod
+    def _detect_qr_code_using_anchor_point(img):
+        # 有些二维码和边缘紧贴，无法识别出整个矩形，所以我们先对图片大小进行扩展
+        expand_length = 10
+        edge = expand_length // 2
+        h, w = img.shape[:2]
+        image_extend = np.zeros((img.shape[0] + expand_length, img.shape[1] + expand_length, 3), np.uint8)
+        image_extend[:] = 255
+        image_extend[edge:edge + h, edge:edge + w] = img
+
+        # 转灰度、二值化、找轮廓
+        gray = cv2.cvtColor(image_extend, cv2.COLOR_BGR2GRAY)
+        # 中值滤波
+        blur_image = cv2.medianBlur(gray, 5)
+        _, thresh = cv2.threshold(blur_image, 127, 255, cv2.THRESH_BINARY)
+        contours, hir = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
+
+        # 三个“回”字特征轮廓存储
+        parent_contours_list = []
+        hir_list = hir[0]
+        for i, item in enumerate(hir_list):
+            # 判断A轮廓是否有B轮廓
+            if item[2] == -1:
+                continue
+            else:
+                hir_b_index = item[2]
+            # 判断B轮廓是否有C轮廓
+            if hir_list[hir_b_index][2] == -1:
+                continue
+            hir_c_index = hir_list[hir_b_index][2]
+            # 计算A轮廓的周长和C轮廓周长的比值
+            hir_c_arc_length = cv2.arcLength(contours[hir_c_index], True)
+            if hir_c_arc_length:
+                error = cv2.arcLength(contours[i], True) / hir_c_arc_length
+                # 二维码每一个“回”的黑白框框的比例大概为1:1:3:1:1
+                # 理论上，A轮廓周长为28，C轮廓周长为12，A/C = error = 2.3333
+                if 1.5 <= error <= 3:
+                    parent_contours_list.append(contours[i])
+
+        # 若找到3个以上“回”字，该图片含有二维码
+        return len(parent_contours_list) >= 3
+
+    @staticmethod
+    def _detect_qr_code_using_wechat_model(img, file_name, model):
+        res = ""
+        try:
+            res, points = model.detectAndDecode(img)
+        except UnicodeDecodeError as ex:
+            res = ex.object.decode('ISO-8859-1').split(" ")[0]
+        except Exception as err:
+            logger.exception(f"fileName: {file_name}, method: ImgAdvertisementImagesCleaner. "
+                             f"An error occurred when using the WeChat model to detect the QR code. "
+                             f"The error is: {err}")
+        if res:
+            return True
+        return False
+
+    def init_model(self, *args, **kwargs):
+        return WechatQRCodeModel(*args, **kwargs).wechat_qr_model
+
+    def resize_img(self, image):
+        """图片等比压缩"""
+        height, width = image.shape[:2]  # 获取原图像的水平方向尺寸和垂直方向尺寸。
+        temp = max(height, width)
+        # 若图片最长边大于限值，对图片进行压缩，否则返回原图
+        if temp >= self.img_resize:
+            mul_temp = temp / self.img_resize
+            if height > width:
+                res = cv2.resize(image, (int(width / mul_temp), self.img_resize), interpolation=cv2.INTER_AREA)
+            elif height < width:
+                res = cv2.resize(image, (self.img_resize, int(height / mul_temp)), interpolation=cv2.INTER_AREA)
+            else:
+                res = cv2.resize(image, (self.img_resize, self.img_resize), interpolation=cv2.INTER_AREA)
+            return res
+        return image
+
+    def execute(self, sample: Dict[str, Any]):
+        start = time.time()
+        self.read_file_first(sample)
+        file_name = sample[self.filename_key]
+        file_type = "." + sample[self.filetype_key]
+        img_bytes = sample[self.data_key]
+        if img_bytes:
+            data = bytes_transform.bytes_to_numpy(img_bytes)
+            image = self._detect_advertisement_img(data, file_name, self.model)
+            sample[self.data_key] = bytes_transform.numpy_to_bytes(image, file_type)
+            logger.info(f"fileName: {file_name}, "
+                        f"method: ImgAdvertisementImagesCleaner costs {(time.time() - start):6f} s")
+        return sample
+
+    def _detect_advertisement_img(self, img, file_name, model):
+        """检测含有二维码的图片"""
+        img_resize = self.resize_img(img)
+        if self._detect_qr_code_using_wechat_model(img_resize, file_name, model) \
+                or self._detect_qr_code_using_anchor_point(img_resize):
+            logger.info(f"fileName: {file_name}, method: ImgAdvertisementImagesCleaner. "
+                        "The image contains advertisement. The image is filtered out.")
+            return np.array([])
+        return img

runtime/ops/filter/img_advertisement_images_cleaner/wechat_qrcode_model.py

@@ -0,0 +1,23 @@
+# -- encoding: utf-8 --
+
+import gc
+import os
+from pathlib import Path
+
+import cv2
+
+
+class WechatQRCodeModel:
+
+    def __init__(self, *args, **kwargs):
+        models_path = os.getenv("MODELS_PATH", "/home/models")
+        self.resources_path = str(Path(models_path, 'img_QRcode_detect', 'resources'))
+        self.wechat_qr_model = cv2.wechat_qrcode_WeChatQRCode(
+            str(Path(self.resources_path, 'detect.prototxt')),
+            str(Path(self.resources_path, 'detect.caffemodel')),
+            str(Path(self.resources_path, 'sr.prototxt')),
+            str(Path(self.resources_path, 'sr.caffemodel')))
+
+    def __del__(self):
+        del self.wechat_qr_model
+        gc.collect()

runtime/ops/filter/img_blurred_images_cleaner/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from deepdata.core.base_op import OPERATORS
+
+OPERATORS.register_module(module_name='ImgBlurredImagesCleaner',
+                          module_path="ops.filter.img_blurred_images_cleaner.process")

runtime/ops/filter/img_blurred_images_cleaner/metadata.yml

@@ -0,0 +1,25 @@
+name: '模糊图片过滤'
+name_en: 'Fuzzy Image Filter'
+description: '去除模糊的图片。'
+description_en: 'Filters out fuzzy images.'
+language: 'python'
+vendor: 'huawei'
+raw_id: 'ImgBlurredImagesCleaner'
+version: '1.0.0'
+types:
+  - 'cleanse'
+modal: 'image'
+effect:
+  before: ''
+  after: ''
+inputs: 'image'
+outputs: 'image'
+settings:
+  blurredThreshold:
+    name: 梯度函数值
+    description: 梯度函数值取值越小，图片模糊度越高。
+    type: slider
+    defaultVal: 1000
+    min: 1
+    max: 10000
+    step: 1

runtime/ops/filter/img_blurred_images_cleaner/process.py

@@ -0,0 +1,51 @@
+# -- encoding: utf-8 --
+
+"""
+Description:
+Create: 2025/01/17
+"""
+import time
+
+from typing import Dict, Any
+
+import cv2
+import numpy as np
+from loguru import logger
+
+
+from deepdata.common.utils import bytes_transform
+from deepdata.core.base_op import Filter
+
+
+class ImgBlurredImagesCleaner(Filter):
+    """过滤模糊度低于阈值的图片插件"""
+
+    def __init__(self, *args, **kwargs):
+        super(ImgBlurredImagesCleaner, self).__init__(*args, **kwargs)
+        # 设置模糊度阈值
+        self._blurred_threshold = kwargs.get("blurredThreshold", 1000)
+
+    def execute(self, sample: Dict[str, Any]):
+        start = time.time()
+        self.read_file_first(sample)
+        img_bytes = sample[self.data_key]
+        file_name = sample[self.filename_key]
+        file_type = "." + sample[self.filetype_key]
+        if img_bytes:
+            data = bytes_transform.bytes_to_numpy(img_bytes)
+            blurred_images = self._blurred_images_filter(data, file_name)
+            sample[self.data_key] = bytes_transform.numpy_to_bytes(blurred_images, file_type)
+        logger.info(f"fileName: {file_name}, method: ImagesBlurredCleaner costs {(time.time() - start):6f} s")
+        return sample
+
+    def _blurred_images_filter(self, image, file_name):
+        # 为方便与其他图片比较可以将图片resize到同一个大小
+        img_resize = cv2.resize(image, (112, 112))
+        # 将图片压缩为单通道的灰度图
+        gray = cv2.cvtColor(img_resize, cv2.COLOR_BGR2GRAY)
+        score = cv2.Laplacian(gray, cv2.CV_64F).var()
+        if score <= self._blurred_threshold:
+            logger.info(f"The image blur is {self._blurred_threshold}, "
+                        f"which exceeds the threshold of {score}. {file_name} is filtered out.")
+            return np.array([])
+        return image

runtime/ops/filter/img_duplicated_images_cleaner/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from deepdata.core.base_op import OPERATORS
+
+OPERATORS.register_module(module_name='ImgDuplicatedImagesCleaner',
+                          module_path="ops.filter.img_duplicated_images_cleaner.process")

runtime/ops/filter/img_duplicated_images_cleaner/metadata.yml

@@ -0,0 +1,16 @@
+name: '重复图片去除'
+name_en: 'Duplicate Image Removal'
+description: '去除重复的图片。'
+description_en: 'Removes duplicate images.'
+language: 'python'
+vendor: 'huawei'
+raw_id: 'ImgDuplicatedImagesCleaner'
+version: '1.0.0'
+types:
+  - 'cleanse'
+modal: 'image'
+effect:
+  before: ''
+  after: ''
+inputs: 'image'
+outputs: 'image'

runtime/ops/filter/img_duplicated_images_cleaner/process.py

@@ -0,0 +1,110 @@
+# -- encoding: utf-8 --
+
+"""
+Description:
+于MD5值计算当前图片与数据集中其它图片是否相同。相同该图片过滤，保留原数据集图片。
+将文件特征数据即MD5值，存到数据库。根据任务uuid获取历史文件特征，遍历特征并进行去重比较
+Create: 2025/1/7
+"""
+
+import json
+import time
+from pathlib import Path
+from typing import Dict, Any
+
+import cv2
+from Crypto.Hash import MD5
+from sqlalchemy import text
+from loguru import logger
+
+from deepdata.sql_manager.sql_manager import SQLManager
+from deepdata.common.utils import get_now_time
+from deepdata.common.utils import bytes_to_numpy, numpy_to_bytes
+from deepdata.core.base_op import Filter
+
+
+class ImgDuplicatedImagesCleaner(Filter):
+    """去除重复图片插件
+    基于MD5值计算当前图片与数据集中其它图片是否相同。相同该图片过滤，保留原数据集图片。
+    """
+
+    def __init__(self, *args, **kwargs):
+        # task_uuid为标识该数据集的唯一标志
+        super().__init__(*args, **kwargs)
+        self.task_uuid = kwargs.get("uuid", "")
+        self.img_resize = 200  # 图片压缩尺寸
+        # 获取数据库sql
+        self.sql_dict = self.load_sql_dict()
+        # 获取数据库连接池
+        self.conn = None  # 数据库连接
+        self.trans = None  # 数据库事务
+
+    @staticmethod
+    def load_sql_dict():
+        """获取sql语句"""
+        sql_config_path = str(Path(__file__).parent / 'sql' / 'sql_config.json')
+        with open(sql_config_path, 'r', encoding='utf-8') as f:
+            return json.load(f)
+
+    def compute_md5(self, img_bytes: bytes) -> str:
+        """将图片统一转化为png无损格式，计算每张图像的md5值"""
+        if not img_bytes:
+            return ""
+        img = bytes_to_numpy(img_bytes)
+        height, width = img.shape[:2]  # 获取原图像的水平方向尺寸和垂直方向尺寸。
+        res = cv2.resize(img, (int(width / height * self.img_resize), self.img_resize), interpolation=cv2.INTER_AREA)
+        img_bytes = numpy_to_bytes(res, ".png")
+        hash_md5 = MD5.new()
+        hash_md5.update(img_bytes)
+        return hash_md5.hexdigest()
+
+    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
+        """重复图片去重算子执行入口"""
+        start = time.time()
+        self.read_file_first(sample)
+        file_name = sample[self.filename_key]
+        self.task_uuid = sample.get("instance_id") if not self.task_uuid else self.task_uuid
+        img_data = self._duplicate_images_filter(file_name, sample[self.data_key])
+        sample[self.data_key] = img_data
+        logger.info(
+            f"fileName: {file_name}, method: DuplicateImagesCleaner costs {(time.time() - start):6f} s")
+        return sample
+
+    def execute_sql(self, md5: str, file_name: str,
+                    img_bytes: bytes) -> bytes:
+        """从数据库中获取文件特征、比较MD5，插入新的文件特征"""
+        timestamp = get_now_time('Asia/Shanghai', '%Y-%m-%d %H:%M:%S', file_name,
+                                 "DuplicateImagesCleaner")
+        query_sql = str(self.sql_dict.get("query_sql"))
+        insert_sql = str(self.sql_dict.get("insert_sql"))
+        create_tables_sql = str(self.sql_dict.get("create_tables_sql"))
+        query_sql_params = {"task_uuid": self.task_uuid, "file_feature": md5}
+        insert_sql_params = {"task_uuid": self.task_uuid, "file_feature": md5, "file_name": file_name.encode("utf-8"),
+                             "timestamp": timestamp}
+
+        db_manager = SQLManager()
+        try:
+            self.conn = db_manager.create_connect()
+        except Exception as e:
+            logger.error(f"fileName: {file_name}, database connection failed: {str(e)}")
+            raise RuntimeError(82000, str(e)) from None
+
+        with self.conn as connection:
+            connection.execute(text(create_tables_sql))
+            # 判断是否有重复文件
+            result = connection.execute(text(query_sql), query_sql_params).fetchall()
+            # 查询记录为空，无重复图片, 插入新文件特征
+            if not result:
+                connection.execute(text(insert_sql), insert_sql_params)
+                return img_bytes
+            logger.info(f"taskId: {self.task_uuid} fileName: {file_name}, method: Duplicate ImagesCleaner. "
+                        f"The image is duplicated and filtered ")
+        return b""
+
+    def _duplicate_images_filter(self, file_name: str, img_bytes: bytes) -> bytes:
+        """重复图片去重算子执行逻辑"""
+        # 如果文件为空，则无需去重，返回原图
+        if not img_bytes:
+            return img_bytes
+        md5 = self.compute_md5(img_bytes)
+        return self.execute_sql(md5, file_name, img_bytes)

runtime/ops/filter/img_duplicated_images_cleaner/sql/sql_config.json

@@ -0,0 +1,5 @@
+{
+  "query_sql": "SELECT * FROM operator_duplicate_img_features WHERE task_uuid = :task_uuid AND file_feature = :file_feature",
+  "insert_sql": "INSERT INTO operator_duplicate_img_features (task_uuid, file_feature, file_name, timestamp) VALUES (:task_uuid, :file_feature, :file_name, :timestamp)",
+  "create_tables_sql": "CREATE TABLE IF NOT EXISTS operator_duplicate_img_features (id SERIAL PRIMARY KEY, task_uuid VARCHAR(255), file_feature TEXT, file_name TEXT, timestamp TIMESTAMP);"
+}

runtime/ops/filter/img_similar_images_cleaner/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from deepdata.core.base_op import OPERATORS
+
+OPERATORS.register_module(module_name='ImgSimilarImagesCleaner',
+                          module_path="ops.filter.img_similar_images_cleaner.process")

runtime/ops/filter/img_similar_images_cleaner/metadata.yml

@@ -0,0 +1,25 @@
+name: '相似图片去除'
+name_en: 'Similar Image Removal'
+description: '去除相似的图片。'
+description_en: 'Removes similar images.'
+language: 'python'
+vendor: 'huawei'
+raw_id: 'ImgSimilarImagesCleaner'
+version: '1.0.0'
+types:
+  - 'cleanse'
+modal: 'image'
+effect:
+  before: ''
+  after: ''
+inputs: 'image'
+outputs: 'image'
+settings:
+  similarThreshold:
+    name: 相似度
+    description: 相似度取值越大，图片相似度越高。
+    type: slider
+    defaultVal: 0.8
+    min: 0
+    max: 1
+    step: 0.01

runtime/ops/filter/img_similar_images_cleaner/process.py

@@ -0,0 +1,240 @@
+# -- encoding: utf-8 --
+
+"""
+Description:
+    1.本算子结合感知哈希算法和ORB两个算法判断图片的相似性
+    2.感知哈希算法则是从图像的整体结构和特征维度来计算图片的相似度。
+    3.ORB算法可以用来对图像中的关键点快速创建特征向量，这些特征向量可以用来识别图像中的对象。通过比较两张图片的特征向量计算相似度。
+    4.感知哈希算法和ORB算法计算相似度高于0.75，则选择二者较大值；若低于0.75，则选择二者最小值作为相似度
+    5.将文件特征数据存到数据库。根据任务uuid获取历史文件特征，遍历特征并进行去重比较
+Create: 2025/1/7
+"""
+import json
+import time
+import zlib
+from pathlib import Path
+from typing import List, Dict, Any
+
+import cv2
+import numpy as np
+from sqlalchemy import text
+from loguru import logger
+
+from deepdata.sql_manager.sql_manager import SQLManager
+from deepdata.common.utils import get_now_time
+from deepdata.common.utils import bytes_to_numpy
+from deepdata.core.base_op import Filter
+
+MAX_RETRIES = 5
+BASE_DELAY = 1
+MAX_DELAY = 30  # 最大延时设置为30秒
+JITTER_FACTOR = 0.25  # 抖动因子为等待时间的25%
+MAX_FEATURES_NUM = 200
+
+
+def get_orb_des(image: np.ndarray) -> np.ndarray:
+    """检测图像中的特征点kp和计算这些特征点的描述符矩阵des_matrix"""
+    if not image.size:
+        return np.array([])
+    orb = cv2.ORB_create()  # 初始化ORB检测器
+    orb.setMaxFeatures(MAX_FEATURES_NUM)  # 设置最大特征点数量为200
+    kp, des_matrix = orb.detectAndCompute(image, None)
+    if des_matrix is None:
+        # 若没有提取出图像特征，描述符矩阵置为空
+        des_matrix = np.array([])
+    return des_matrix
+
+
+class ImgSimilarImagesCleaner(Filter):
+    """去除相似图片的插件"""
+
+    DEFAULT_SIMILAR_THRESHOLD = 0.8  # 默认相似度阈值
+    DEFAULT_ORB_RATIO = 0.8  # 默认特征点距离比率
+    DEFAULT_MIX_SIMILARITY = 0.75  # 默认相似度算法阈值
+    DEFAULT_IMG_RESIZE = 200  # 默认图片压缩尺寸
+    DEFAULT_PAGE_SIZE = 500  # 默认每页数据量
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.similar_threshold = kwargs.get("similarThreshold", self.DEFAULT_SIMILAR_THRESHOLD)  # 默认相似度阈值为0.8
+        # task_uuid为标识该数据集的唯一标志
+        self.task_uuid = kwargs.get("uuid", "")
+        self.orb_ratio = self.DEFAULT_ORB_RATIO  # 特征点距离的比率，该数值为经验值
+        self.mix_similarity = self.DEFAULT_MIX_SIMILARITY  # 选择相似度算法的阈值，该数值为经验值
+        self.img_resize = self.DEFAULT_IMG_RESIZE  # 图片压缩尺寸
+        self.conn = None  # 数据库连接
+        self.trans = None  # 数据库事务
+        self.page_size = self.DEFAULT_PAGE_SIZE  # 每页数据量
+        # 获取数据库sql
+        self.sql_dict = self.load_sql_dict()
+
+    @staticmethod
+    def load_sql_dict():
+        """获取sql语句"""
+        sql_config_path = str(Path(__file__).parent / 'sql' / 'sql_config.json')
+        with open(sql_config_path, 'r', encoding='utf-8') as f:
+            return json.load(f)
+
+    @staticmethod
+    def get_p_hash(image: np.ndarray) -> str:
+        """计算pHash值"""
+        hashed_value = ""
+        if not image.size:
+            return hashed_value
+        gray_image = cv2.cvtColor(cv2.resize(image, (8, 8), interpolation=cv2.INTER_AREA), cv2.COLOR_BGR2GRAY)
+        dct_image = cv2.dct(np.float32(gray_image))
+        hashed_value = ''.join(['1' if x >= 0 else '0' for x in dct_image[:8, :8].flatten()])
+        return hashed_value
+
+    @staticmethod
+    def get_phash_similarity(hash_comparison: str, hash_compared: str) -> float:
+        """通过计算汉明距离，获取图片相似度"""
+        # 若哈希值为空，则相似度为0
+        if not hash_comparison or not hash_compared:
+            return 0.0
+        # 计算汉明距离
+        distance = sum(
+            bit_comparison != bit_compared for bit_comparison, bit_compared in zip(hash_comparison, hash_compared))
+        similarity = 1 - distance / len(hash_comparison)
+        return similarity
+
+    def filter_similar_images(self, img: np.ndarray, file_name: str) -> np.ndarray:
+        """判断数据集中是否存在相似图片"""
+        # 如果文件为空，则无需去重，返回原图
+        if not img.size:
+            return img
+        p_hash = self.get_p_hash(img)
+        height, width = img.shape[:2]  # 获取原图像的水平方向尺寸和垂直方向尺寸。
+        img_resize = cv2.resize(img, (int(width / height * self.img_resize), self.img_resize),
+                                interpolation=cv2.INTER_AREA)
+        des_matrix = get_orb_des(img_resize)
+        return self.execute_sql(p_hash, des_matrix, file_name, img)
+
+    def get_orb_similarity(self, des_matrix: np.ndarray, des_matrix_history: np.ndarray, file_name: str,
+                           file_name_history: str) -> float:
+        """获取图片orb相似度"""
+        # 若描述符矩阵为空，则相似度为0
+        if not des_matrix.size or not des_matrix_history.size:
+            return 0.0
+        # 根据矩阵对角线上元素和的大小，选择描述符矩阵作为训练或查询矩阵
+        train_matrix, query_matrix = des_matrix, des_matrix_history
+        if train_matrix.shape[0] > des_matrix_history.shape[0]:
+            train_matrix, query_matrix = des_matrix_history, des_matrix
+        elif des_matrix.shape[0] == des_matrix_history.shape[0]:
+            if np.trace(des_matrix) > np.trace(des_matrix_history):
+                train_matrix, query_matrix = des_matrix_history, des_matrix
+
+        try:
+            # knn筛选结果
+            matches = (cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=False).
+                       knnMatch(query_matrix, trainDescriptors=train_matrix, k=2))
+            if not matches:
+                return 0.0
+            # 遍历每一对特征点，筛选距离更近的特征点
+            count = 0
+            for (m, n) in matches:
+                if m.distance < self.orb_ratio * n.distance:
+                    count += 1
+            orb_similarity = count / len(matches)
+            return orb_similarity
+        except Exception as e:
+            logger.exception(f"taskId: {self.task_uuid}, failed to compare the similarity between "
+                             f"{file_name} and {file_name_history}: {e}")
+            return 0.0
+
+    def execute_sql(self, p_hash: str, des_matrix: np.ndarray, file_name: str,
+                    img: np.ndarray) -> np.ndarray:
+        des_matrix_binary = zlib.compress(des_matrix.tobytes())  # 使用 zlib 进行压缩数组
+        timestamp = get_now_time('Asia/Shanghai', '%Y-%m-%d %H:%M:%S', file_name,
+                                 "ImgSimilarCleaner")
+        query_task_uuid_sql = str(self.sql_dict.get("query_task_uuid_sql"))
+        insert_sql = str(self.sql_dict.get("insert_sql"))
+        create_tables_sql = str(self.sql_dict.get("create_tables_sql"))
+
+        db_manager = SQLManager()
+        try:
+            self.conn = db_manager.create_connect()
+        except Exception as e:
+            logger.error(f"fileName: {file_name}, database connection failed: {str(e)}")
+            raise RuntimeError(82000, str(e)) from None
+
+        with self.conn as connection:
+            """从数据库中获取文件特征、比较相似度，插入新的文件特征"""
+            connection.execute(text(create_tables_sql))
+            result = connection.execute(text(query_task_uuid_sql), {"task_uuid": self.task_uuid}).fetchall()
+            total_count = len(result)
+            if self.has_similar_images(connection, des_matrix, file_name, p_hash, total_count):
+                    return np.array([])
+
+            insert_data = {
+                "task_uuid": self.task_uuid,
+                "p_hash": p_hash,
+                "des_matrix": des_matrix_binary,
+                "matrix_shape": str(des_matrix.shape),
+                "file_name": file_name.encode("utf-8").hex(),
+                "timestamp": timestamp
+            }
+            connection.execute(text(insert_sql),insert_data)
+        return img
+
+    def has_similar_images(self, connection, des_matrix, file_name, p_hash, total_count):
+        for i in range(0, total_count, self.page_size):
+            query_sql = self.sql_dict.get("query_sql")
+            rows = connection.execute(text(query_sql), {"task_uuid": self.task_uuid, "ge": self.page_size, "le": i}).fetchall()
+            # 对应任务uuid，最后一页没有数据，跳出循环
+            if not rows:
+                break            # 对两张图片进行相似度比较
+            if self.determine_similar_images(rows, p_hash, des_matrix, file_name):
+                return True
+        return False
+
+    def determine_similar_images(self, file_features: List, p_hash: str, des_matrix: np.ndarray,
+                                 file_name: str) -> bool:
+        """根据文件特征，判断两张图片相似度是否超过指定阈值"""
+        for signature in file_features:
+            pash_feature, orb_feature, matrix_shape, file_name_history = signature[2], signature[3], signature[4], \
+                signature[5]
+            if not pash_feature:
+                # 若图片为空，p_hash、des_matrix为空，跳过比对
+                continue
+            # 解压缩数据
+            decompressed_data = zlib.decompress(orb_feature)
+            # 将字节流转换回矩阵
+            des_matrix_history = np.frombuffer(decompressed_data, dtype=np.uint8).reshape(eval(matrix_shape))
+            # 移除转义字符 '\' 并将十六进制字符串转换为字节序列
+            bytes_data = bytes.fromhex(file_name_history)
+            # 解码字节序列为 UTF-8 编码的字符串
+            file_name_decoded = bytes_data.decode('utf-8')
+
+            phash_similarity = self.get_phash_similarity(p_hash, pash_feature)
+            orb_similarity = self.get_orb_similarity(des_matrix, des_matrix_history, file_name, file_name_decoded)
+            max_similarity = max(phash_similarity, orb_similarity)
+            min_similarity = min(phash_similarity, orb_similarity)
+            if max_similarity >= self.mix_similarity:
+                result = max_similarity
+            else:
+                result = min_similarity
+            similarity = round(result, 2)
+            if similarity >= self.similar_threshold:
+                logger.info(
+                    f"fileName: {file_name}, method: ImgSimilarCleaner, dataset: {self.task_uuid}. "
+                    f"This picture is similar to {file_name_decoded}, "
+                    f"and the similarity is {similarity:.4f}. The picture is filtered."
+                )
+                return True
+        return False
+
+    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
+        """去除相似图片算子执行入口"""
+        start = time.time()
+        self.read_file_first(sample)
+        file_name = sample[self.filename_key]
+        img_bytes = sample[self.data_key]
+        self.task_uuid = sample.get("instance_id") if not self.task_uuid else self.task_uuid
+        data = bytes_to_numpy(img_bytes) if img_bytes else np.array([])
+        similar_images = self.filter_similar_images(data, file_name)
+        # 若相似图片，sample[self.data_key]设为空
+        if not similar_images.size:
+            sample[self.data_key] = b""
+        logger.info(f"fileName: {file_name}, method: ImgSimilarCleaner costs {(time.time() - start):6f} s")
+        return sample

runtime/ops/filter/img_similar_images_cleaner/sql/sql_config.json

@@ -0,0 +1,6 @@
+{
+  "query_sql": "SELECT * FROM operator_similar_img_features WHERE task_uuid = :task_uuid ORDER BY timestamp LIMIT :ge OFFSET :le",
+  "insert_sql": "INSERT INTO operator_similar_img_features (task_uuid,p_hash,des_matrix,matrix_shape,file_name,timestamp) VALUES (:task_uuid,:p_hash,:des_matrix,:matrix_shape,:file_name,:timestamp)",
+  "query_task_uuid_sql": "SELECT * FROM operator_similar_img_features WHERE task_uuid = :task_uuid",
+  "create_tables_sql": "CREATE TABLE IF NOT EXISTS operator_similar_img_features (id SERIAL PRIMARY KEY,task_uuid VARCHAR(255),p_hash TEXT,des_matrix BYTEA,matrix_shape TEXT,file_name TEXT,timestamp TIMESTAMP);"
+}

runtime/ops/filter/remove_duplicate_file/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from deepdata.core.base_op import OPERATORS
+
+OPERATORS.register_module(module_name='DuplicateFilesFilter',
+                          module_path="ops.filter.remove_duplicate_file.process")

runtime/ops/filter/remove_duplicate_file/metadata.yml

@@ -0,0 +1,25 @@
+name: '相似文档去除'
+name_en: 'Similar Document Removal'
+description: '相似文档去除。'
+description_en: 'Removes similar documents.'
+language: 'python'
+vendor: 'huawei'
+raw_id: 'DuplicateFilesFilter'
+version: '1.0.0'
+types:
+  - 'cleanse'
+modal: 'text'
+effect:
+  before: '这篇文档跟数据集中的另一篇文档内容几乎一样，执行该算子后，这篇文档会被去除。'
+  after: ''
+inputs: 'text'
+outputs: 'text'
+settings:
+  fileDuplicateThreshold:
+    name: 文档相似度
+    description: 基于MinHash算法和Jaccard相似度，计算当前文档与数据集中其它文档相似性，超过设定值，该文档被去除。
+    type: slider
+    defaultVal: 0.5
+    min: 0
+    max: 1
+    step: 0.1

runtime/ops/filter/remove_duplicate_file/process.py

@@ -0,0 +1,159 @@
+#!/user/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Description: 文档局部内容去重
+Create: 2025/01/07
+"""
+
+import json
+import re
+import time
+from pathlib import Path
+from typing import List, Dict, Any
+
+import numpy as np
+from datasketch import MinHash
+from sqlalchemy import text
+from loguru import logger
+
+from deepdata.sql_manager.sql_manager import SQLManager
+from deepdata.common.utils import get_now_time
+from deepdata.core.base_op import Filter
+
+
+class DuplicateFilesFilter(Filter):
+    """相似文档去除插件
+
+    基于MinHash计算当前文档与数据集中其它文档相似性，相似性高于设定阈值则返回空。
+    """
+
+    def __init__(self, *args, **kwargs):
+        # 标点符号
+        super().__init__(*args, **kwargs)
+        self.punctuation_pattern = "。.？?！!，,；;：:（）()【】{}[]“”""‘’''/\n"
+        # 默认相似度阈值为0.5
+        self.duplicate_th = kwargs.get("fileDuplicateThreshold", 0.5)
+        # task_uuid为标识该数据集的唯一标志
+        self.task_uuid = kwargs.get("uuid", "")
+        # 数据库连接
+        self.conn = None
+        # 数据库事务
+        self.trans = None
+        # 每页数据量
+        self.page_size = 500
+        # 获取数据库sql
+        self.sql_dict = self.load_sql_dict()
+
+    @staticmethod
+    def load_sql_dict():
+        """获取sql语句"""
+        sql_config_path = str(Path(__file__).parent / 'sql' / 'sql_config.json')
+        with open(sql_config_path, 'r', encoding='utf-8') as f:
+            return json.load(f)
+
+    def get_minhash(self, input_text: str) -> MinHash:
+        """获取输入文档的minhash
+
+        Args:
+            input_text: 输入文档内容
+
+        Returns:
+            text_minhash: 输入文档对应的minhash值
+        """
+        text_minhash = MinHash()
+        for word in re.split(f"[{re.escape(self.punctuation_pattern)}]", input_text.strip()):
+            text_minhash.update(word.strip().encode('utf8'))
+        return text_minhash
+
+    def deduplicate_files(self, sample: Dict[str, Any], file_name: str) -> str:
+        """去除相似文件
+
+        Args:
+            content: 待处理的Content对象
+            file_name: 文件名称
+
+        Returns:
+            input_text: 去重后的文件内容，大于相似度值返回空，否则返回原始文本内容。
+        """
+        input_text = sample[self.text_key]
+        if not input_text:
+            return input_text
+        text_minhash = self.get_minhash(input_text)
+        return self.execute_sql(text_minhash, file_name, input_text)
+
+    def execute_sql(self, text_minhash: MinHash, file_name: str,
+                    input_text: str) -> str:
+        """从数据库中获取文件特征、比较相似度，插入新的文件特征"""
+        timestamp = get_now_time('Asia/Shanghai', '%Y-%m-%d %H:%M:%S', file_name,
+                                 "DuplicateFilesFilter")
+        minhash_values = text_minhash.hashvalues
+        # 将 NumPy 数组转换为字符串
+        minhash_values_string = np.array2string(minhash_values)
+        query_task_uuid_sql = self.sql_dict.get("query_task_uuid_sql")
+        insert_sql = self.sql_dict.get("insert_sql")
+        create_tables_sql = self.sql_dict.get("create_tables_sql")
+        db_manager = SQLManager()
+        try:
+            self.conn = db_manager.create_connect()
+        except Exception as e:
+            logger.error(f"fileName: {file_name}, database connection failed: {str(e)}")
+            raise RuntimeError(82000, str(e)) from None
+        with self.conn as connection:
+            connection.execute(text(create_tables_sql))
+            result = connection.execute(text(query_task_uuid_sql), {"task_uuid": self.task_uuid}).fetchall()
+            total_count = len(result)
+            if self.has_similar_text(connection, file_name, text_minhash, total_count):
+                return ""
+            insert_data = {
+                "task_uuid": self.task_uuid,
+                "file_feature": minhash_values_string,
+                "file_name": file_name.encode("utf-8").hex(),
+                "timestamp": timestamp
+            }
+            connection.execute(text(insert_sql), insert_data)
+        return input_text
+
+    def has_similar_text(self, connection, file_name, text_minhash, total_count) -> bool:
+        query_sql = self.sql_dict.get("query_sql")
+        for i in range(0, total_count, self.page_size):
+            rows = connection.execute(
+                text(query_sql), {"task_uuid": self.task_uuid, "file_name": file_name, "ge": self.page_size, "le": i}).fetchall()
+            # 对应任务uuid，最后一页没有数据，跳出循环
+            if not rows:
+                break
+            # 对两个文本进行相似度比较
+            if self.determine_similar_text(rows, text_minhash, file_name):
+                return True
+        return False
+
+    def determine_similar_text(self, file_features: List, text_minhash: MinHash, file_name: str) -> bool:
+        for signature in file_features:
+            # 历史文件特征和历史文件名称
+            file_feature, file_name_history = signature[2], signature[3]
+            if not file_feature:
+                continue
+            minhash_obj = MinHash(num_perm=128)
+            minhash_obj.hashvalues = np.fromstring(file_feature.strip('[]'), dtype=np.uint64, sep=' ')
+            similarity = text_minhash.jaccard(minhash_obj)
+
+            # 移除转义字符 '\' 并将十六进制字符串转换为字节序列
+            bytes_data = bytes.fromhex(file_name_history)
+            # 解码字节序列为 UTF-8 编码的字符串
+            file_name_decoded = bytes_data.decode('utf-8')
+
+            if similarity >= self.duplicate_th:
+                logger.info(f"taskId: {self.task_uuid}, fileName: {file_name} is similar to {file_name_decoded}, "
+                            f"and the similarity is {similarity:4f}")
+                return True
+        return False
+
+    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
+        start = time.time()
+        self.read_file_first(sample)
+        file_name = sample[self.filename_key]
+        self.task_uuid = sample.get("instance_id") if not self.task_uuid else self.task_uuid
+        sample[self.text_key] = self.deduplicate_files(sample, file_name)
+        logger.info(f"taskId: {self.task_uuid} fileName: {file_name}, "
+                    f"method: DuplicateFilesFilter costs {(time.time() - start):6f} s")
+        return sample

runtime/ops/filter/remove_duplicate_file/sql/sql_config.json

@@ -0,0 +1,6 @@
+{
+  "query_sql": "SELECT * FROM operators_similar_text_features WHERE task_uuid = :task_uuid AND file_name <> :file_name ORDER BY timestamp LIMIT :ge OFFSET :le",
+  "create_tables_sql": "CREATE TABLE IF NOT EXISTS operators_similar_text_features (id SERIAL PRIMARY KEY, task_uuid VARCHAR(255), file_feature TEXT, file_name TEXT, timestamp TIMESTAMP);",
+  "insert_sql": "INSERT INTO operators_similar_text_features (task_uuid, file_feature, file_name, timestamp) VALUES (:task_uuid, :file_feature, :file_name, :timestamp)",
+  "query_task_uuid_sql": "SELECT * FROM operators_similar_text_features WHERE task_uuid = :task_uuid"
+}

runtime/ops/filter/remove_file_with_many_sensitive_words/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from deepdata.core.base_op import OPERATORS
+
+OPERATORS.register_module(module_name='FileWithManySensitiveWordsFilter',
+                          module_path="ops.filter.remove_file_with_many_sensitive_words.process")

runtime/ops/filter/remove_file_with_many_sensitive_words/metadata.yml

@@ -0,0 +1,25 @@
+name: '文档敏感词率检查'
+name_en: 'Sensitive Word Rate Check'
+description: '去除敏感词过多的文档。'
+description_en: 'Filters out files that contain excessive sensitive phrases.'
+language: 'python'
+vendor: 'huawei'
+raw_id: 'FileWithManySensitiveWordsFilter'
+version: '1.0.0'
+types:
+  - 'cleanse'
+modal: 'text'
+effect:
+  before: '出售硝酸甘油出售硝酸甘油出售硝酸甘油出售硝酸甘油'
+  after: ''
+inputs: 'text'
+outputs: 'text'
+settings:
+  sensitiveWordsRate:
+    name: 文档敏感词率
+    description: 敏感词的字数/文档总字数 > 设定值，该文档被去除。
+    type: slider
+    defaultVal: 0.01
+    min: 0
+    max: 1
+    step: 0.01

runtime/ops/filter/remove_file_with_many_sensitive_words/process.py

@@ -0,0 +1,117 @@
+#!/user/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Description: 过滤语言概率太低的文档（支持自定义阈值）
+Create: 2023/12/7 15:43
+"""
+import sys
+import time
+from pathlib import Path
+from typing import Dict, Any
+
+from loguru import logger
+
+from deepdata.core.base_op import Filter
+from deepdata.common.utils.aho_corasick import build_trie, add_fail_pointer
+
+sys.setrecursionlimit(5000)
+
+
+class AhoCorasic:
+    """AC自动机算法进行目标字符串搜索"""
+
+    def __init__(self, words):
+        self._root = add_fail_pointer(build_trie(words))
+
+    def search_and_count(self, text: str, special_symbols: set):
+        """
+        匹配敏感词，统计敏感词字数。
+
+        Args:
+            text: 文本
+            special_symbols: 特殊字符（需跳过）
+        Returns:
+            统计敏感词字数
+        """
+        target_count = 0
+        node = self._root
+
+        valid_len = 0  # 当前遍历的有效长度
+        for _, s in enumerate(text):
+            if s in special_symbols:  # 跳过特殊字符
+                continue
+
+            matched = True
+            while s not in node.child:  # 当node.child没有字符s
+                if node == self._root:  # 当node为root（无node.fail），有效长度归0且跳出
+                    valid_len = 0
+                    matched = False
+                    break
+                elif node.fail == self._root:  # node.fail为root场景，有效长度归0，但可继续
+                    valid_len = 0
+                node = node.fail  # 移动到失败指针节点
+            if not matched:
+                continue
+
+            node = node.child.get(s)
+            valid_len += 1
+            if node.word:  # node是单词尾字母
+                target_count += valid_len
+                valid_len = 0
+        return target_count
+
+
+class FileWithManySensitiveWordsFilter(Filter):
+    """外部输入的暴力、色情文本过滤插件"""
+
+    def __init__(self, *args, **kwargs):
+        super(FileWithManySensitiveWordsFilter, self).__init__(*args, **kwargs)
+        root_path = Path(__file__).parent / 'resources'
+        violent_file_path = str(root_path / 'violent.txt')
+        sexual_file_path = str(root_path / 'sexual.txt')
+        political_file_path = str(root_path / 'political.txt')
+        special_symbols_path = str(root_path / 'special_symbols.txt')
+        self._file_sensitive_words_rate = kwargs.get("sensitiveWordsRate", 0.01)  # 参数默认值为0.01
+        self.violent_words = self.load_words_list(violent_file_path)
+        self.sexual_words = self.load_words_list(sexual_file_path)
+        self.political_words = self.load_words_list(political_file_path)
+        self.special_symbols = self.load_words_list(special_symbols_path)
+        self.symbols = self.special_symbols | {"\n", "\t", "\r"}  # 符号，不纳入文本字数统计
+        self.words = self.violent_words | self.sexual_words | self.political_words
+        self.ac_automaton = AhoCorasic(self.words)
+
+    @staticmethod
+    def load_words_list(path):
+        """词表加载"""
+        with open(path, 'r', encoding='utf-8') as f:
+            words = set(f.read().splitlines())
+        return words
+
+    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
+        start = time.time()
+        self.read_file_first(sample)
+        sample[self.text_key] = self._file_with_many_sensitive_words_filter(sample[self.text_key],
+                                                                            sample[self.filename_key])
+        logger.info(f"fileName: {sample[self.filename_key]}, "
+                    f"method: FileWithManySensitiveWordsFilter costs {(time.time() - start):6f} s")
+        return sample
+
+    def _file_with_many_sensitive_words_filter(self, input_data: str, file_name):
+        """过滤敏感词过多的文档"""
+        total_count = 0
+        for s in input_data:
+            if s not in self.symbols:
+                total_count += 1
+        if total_count == 0:
+            return input_data
+
+        # 敏感词率 = 敏感词字数 / 总字数，符号不纳入统计
+        sensitive_rate = self.ac_automaton.search_and_count(input_data, self.special_symbols) / total_count
+        if sensitive_rate >= self._file_sensitive_words_rate:
+            logger.info(f"This document contains too many sensitive words. "
+                        f"The proportion of sensitive words is {sensitive_rate}. "
+                        f"Threshold is {self._file_sensitive_words_rate}. The document {file_name} is filtered.")
+            return ""
+        else:
+            return input_data

runtime/ops/filter/remove_file_with_many_sensitive_words/resources/special_symbols.txt

@@ -0,0 +1,50 @@
+!
+ 
+.
+,
+#
+$
+%
+&
+*
+(
+)
+|
+?
+/
+@
+"
+'
+;
+[
+]
+{
+}
++
+~
+-
+_
+=
+^
+<
+>
+！
+。
+，
+￥
+（
+）
+？
+、
+“
+‘
+；
+【
+】
+——
+…
+……
+《
+》
+:
+：

runtime/ops/filter/remove_file_with_short_or_long_length/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from deepdata.core.base_op import OPERATORS
+
+OPERATORS.register_module(module_name='FileWithShortOrLongLengthFilter',
+                          module_path="ops.filter.remove_file_with_short_or_long_length.process")

runtime/ops/filter/remove_file_with_short_or_long_length/metadata.yml

@@ -0,0 +1,34 @@
+name: '文档字数检查'
+name_en: 'Word Count Check'
+description: '字数不在指定范围会被过滤掉。'
+description_en: 'Filters out documents whose word count is not in the specified range.'
+language: 'python'
+vendor: 'huawei'
+raw_id: 'FileWithShortOrLongLengthFilter'
+version: '1.0.0'
+types:
+  - 'cleanse'
+modal: 'text'
+effect:
+  before: '过短文本'
+  after: ''
+inputs: 'text'
+outputs: 'text'
+settings:
+  fileLength:
+    name: 文档字数
+    description: '过滤字数不在指定范围内的文档，如[10,10000000]。若输入为空，则不对字数上/下限做限制。'
+    type: range
+    properties:
+      - name: fileMinimumLength
+        type: inputNumber
+        defaultVal: 10
+        min: 0
+        max: 10000000000000000
+        step: 1
+      - name: fileMaximumLength
+        type: inputNumber
+        defaultVal: 10000000
+        min: 0
+        max: 10000000000000000
+        step: 1

runtime/ops/filter/remove_file_with_short_or_long_length/process.py

@@ -0,0 +1,55 @@
+#!/user/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Description: 词数目不在指定范围会被过滤掉（支持自定义阈值）
+Create: 2025/01/16
+"""
+
+import re
+import time
+from typing import Dict, Any
+
+from loguru import logger
+
+from deepdata.core.base_op import Filter
+
+
+class FileWithShortOrLongLengthFilter(Filter):
+    """检查文档字数目，词数目不在指定范围会被过滤掉（支持自定义阈值）"""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        file_length_list = kwargs.get("fileLength", [10, 10000000])  # [下限，上限]，默认字数下限为10, 默认字数上限为10000000
+        if len(file_length_list) != 2:  # 要求传入字数目上限和字数目下限
+            logger.error(f"method: FileWithShortOrLongLengthFilter expected 2 arguments, got {len(file_length_list)}")
+            raise RuntimeError(82001, "method: FileWithShortOrLongLengthFilter expected 2 arguments") from None
+        # 用户不输入下限参数时前端传入''，则不对字数目下限控制
+        self._file_minimum_length = 0 if not file_length_list[0] else file_length_list[0]
+        # 用户不输入上限参数时前端传入''，则不对字数目上限控制
+        self._file_maximum_length = float("inf") if not file_length_list[1] else file_length_list[1]
+
+    def execute(self, sample: Dict[str, Any]) -> Dict[str, Any]:
+        start = time.time()
+        self.read_file_first(sample)
+        sample[self.text_key] = self._file_with_short_or_long_length_filter(sample[self.text_key],
+                                                                            sample[self.filename_key])
+        logger.info(f"fileName: {sample[self.filename_key]}, "
+                    f"method: FileWithShortOrLongLengthFilter costs {(time.time() - start):6f} s")
+        return sample
+
+    def _strip_unicode_whitespace(self, text: str):
+        # 常见 Unicode 空格符（涵盖普通空格、全角空格、零宽空格等）
+        pattern = r'[\u0020\u00A0\u1680\u2000-\u200F\u202F\u205F\u3000]+'
+        # 匹配首尾的空格符
+        pattern = fr'^{pattern}|{pattern}$'
+        return re.sub(pattern, '', text)
+
+    def _file_with_short_or_long_length_filter(self, input_data: str, file_name):
+        input_data_tmp = self._strip_unicode_whitespace(input_data)
+        if len(input_data_tmp) < self._file_minimum_length or len(input_data_tmp) > self._file_maximum_length:
+            logger.info(f"The length of input_data is: {len(input_data_tmp)}, "
+                        f"which is not within the threshold range of {self._file_minimum_length} "
+                        f"and {self._file_maximum_length}. {file_name} is filtered.")
+            return ""
+        return input_data