68 lines
1.7 KiB
Python
68 lines
1.7 KiB
Python
"""
|
|
文本清理工具
|
|
|
|
提供文本清理和标准化的工具方法。
|
|
"""
|
|
import re
|
|
import logging
|
|
from typing import Optional
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class TextCleaner:
|
|
"""文本清理工具类"""
|
|
|
|
@staticmethod
|
|
def clean(text: Optional[str]) -> str:
|
|
"""清理文本
|
|
|
|
Args:
|
|
text: 原始文本
|
|
|
|
Returns:
|
|
清理后的文本
|
|
"""
|
|
if not text:
|
|
return ""
|
|
|
|
text = TextCleaner._remove_control_characters(text)
|
|
text = TextCleaner._normalize_whitespace(text)
|
|
text = TextCleaner._remove_empty_lines(text)
|
|
|
|
return text.strip()
|
|
|
|
@staticmethod
|
|
def _remove_control_characters(text: str) -> str:
|
|
"""移除控制字符"""
|
|
control_chars = re.compile(r'[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f-\x9f]')
|
|
return control_chars.sub('', text)
|
|
|
|
@staticmethod
|
|
def _normalize_whitespace(text: str) -> str:
|
|
"""标准化空白字符"""
|
|
text = re.sub(r'[ \t]+', ' ', text)
|
|
text = re.sub(r'\n\s*\n\s*\n+', '\n\n', text)
|
|
return text
|
|
|
|
@staticmethod
|
|
def _remove_empty_lines(text: str) -> str:
|
|
"""移除空行"""
|
|
lines = text.split('\n')
|
|
return '\n'.join(line for line in lines if line.strip())
|
|
|
|
@staticmethod
|
|
def has_printable_content(text: str) -> bool:
|
|
"""检查是否包含可打印内容
|
|
|
|
Args:
|
|
text: 文本
|
|
|
|
Returns:
|
|
是否包含可打印内容
|
|
"""
|
|
if not text:
|
|
return False
|
|
return any(c.isprintable() and not c.isspace() for c in text)
|