沃巴蒂
ff22ed3a8b
本次同步 09-12 至 09-28 的代码变更，主要内容： 新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务 功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema） 结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本 未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
85 lines
3.2 KiB
Python
85 lines
3.2 KiB
Python
#!/user/bin/python
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
Description: 文本切分算子
|
||
Create: 2023/11/09 10:17
|
||
"""
|
||
|
||
import random
|
||
from typing import List, Dict, Any
|
||
|
||
from loguru import logger
|
||
|
||
from deepdata.common.utils.text_splitter import TextSplitter
|
||
from deepdata.core.base_op import Slicer
|
||
|
||
|
||
class TextSegmentationOperator:
|
||
def __init__(self, max_characters, chunk_size, chunk_overlap):
|
||
try:
|
||
self.text_splitter = TextSplitter(max_characters, chunk_size, chunk_overlap)
|
||
except Exception as err:
|
||
logger.exception(f"init text splitter failed, error is： {err}")
|
||
raise Exception(83001, "init text splitter failed") from None
|
||
|
||
def process(self, input_data: str) -> List[str]:
|
||
if input_data.strip() == "":
|
||
logger.info("input text is empty, return empty chunks.")
|
||
return []
|
||
return self.text_splitter.split_text(input_data)
|
||
|
||
|
||
class Segmentation(Slicer):
|
||
"""切片算法插件"""
|
||
|
||
def __init__(self, *args, **kwargs):
|
||
super(Segmentation, self).__init__(*args, **kwargs)
|
||
self.max_characters = kwargs.get("maxCharacters", -1)
|
||
self.chunk_size = kwargs.get("chunkSize", 800)
|
||
self.chunk_overlap = kwargs.get("chunkOverlap", 100)
|
||
self.slice_num = kwargs.get("sliceNum", -1) # -1 means no limit, export all chunks
|
||
self.text_splitter_op = TextSegmentationOperator(self.max_characters, self.chunk_size, self.chunk_overlap)
|
||
self.last_ops = True
|
||
|
||
def execute(self, sample: Dict[str, Any]) -> List[Dict]:
|
||
|
||
try:
|
||
chunks = self.text_splitter_op.process(sample[self.text_key])
|
||
except Exception as err:
|
||
logger.exception(f"split text failed, error is: {err}")
|
||
raise Exception(83002, "split text failed") from None
|
||
|
||
# Determine how many chunks to save
|
||
num_to_save = len(chunks) if self.slice_num < 0 else min(self.slice_num, len(chunks))
|
||
|
||
sample_list = []
|
||
# If slice_num is set and less than total chunks, randomly sample; otherwise save all in order
|
||
if 0 < self.slice_num < len(chunks):
|
||
selected_indices = sorted(random.sample(range(len(chunks)), num_to_save))
|
||
else:
|
||
selected_indices = list(range(num_to_save))
|
||
|
||
for idx in selected_indices:
|
||
chunk = chunks[idx]
|
||
temp_sample = {
|
||
self.text_key: chunk,
|
||
self.data_key: "",
|
||
self.export_path_key: sample[self.export_path_key],
|
||
self.filename_key: sample[self.filename_key],
|
||
self.fileid_key: sample[self.fileid_key],
|
||
"sequenceId": str(idx),
|
||
"chunkSize": str(len(chunk)),
|
||
}
|
||
self.save_patch_sample(temp_sample, idx + 1, save_format="text")
|
||
sample_list.append(temp_sample)
|
||
|
||
sample["fileNum"] = len(chunks)
|
||
# Keep original text if not all chunks are saved; clear text if all chunks are saved as slices
|
||
if num_to_save < len(chunks):
|
||
sample[self.text_key] = chunks[0] if chunks else ""
|
||
else:
|
||
# All chunks saved as slices, clear text to avoid duplication
|
||
sample[self.text_key] = ""
|
||
|
||
return sample_list
|