沃巴蒂
ff22ed3a8b
本次同步 09-12 至 09-28 的代码变更，主要内容： 新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务 功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema） 结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本 未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
122 lines
4.0 KiB
Python
122 lines
4.0 KiB
Python
import logging
|
|
import re
|
|
import sys
|
|
from logging.handlers import RotatingFileHandler
|
|
from pathlib import Path
|
|
|
|
from app.core.config import settings
|
|
|
|
_SIZE_PATTERN = re.compile(r"^\s*(\d+)\s*([KMG]?B?)?\s*$", re.IGNORECASE)
|
|
|
|
|
|
class CenteredLevelNameFormatter(logging.Formatter):
|
|
"""Center the level name in the log output"""
|
|
|
|
def format(self, record):
|
|
# 将 levelname 居中对齐到8个字符
|
|
record.levelname = record.levelname.center(8)
|
|
return super().format(record)
|
|
|
|
|
|
def _parse_size_to_bytes(value: str) -> int:
|
|
match = _SIZE_PATTERN.match(value)
|
|
if not match:
|
|
return 100 * 1024 * 1024
|
|
|
|
amount = int(match.group(1))
|
|
unit = (match.group(2) or "B").upper()
|
|
if unit in {"G", "GB"}:
|
|
return amount * 1024 * 1024 * 1024
|
|
if unit in {"M", "MB"}:
|
|
return amount * 1024 * 1024
|
|
if unit in {"K", "KB"}:
|
|
return amount * 1024
|
|
return amount
|
|
|
|
|
|
def _rotated_log_namer(default_name: str) -> str:
|
|
path = Path(default_name)
|
|
index_suffix = path.suffix
|
|
if not index_suffix.lstrip(".").isdigit():
|
|
return default_name
|
|
|
|
base_path = Path(str(path)[: -len(index_suffix)])
|
|
if not base_path.suffix:
|
|
return default_name
|
|
|
|
return str(base_path.with_name(f"{base_path.stem}{index_suffix}{base_path.suffix}"))
|
|
|
|
|
|
def _remove_managed_handlers(logger: logging.Logger) -> None:
|
|
for handler in list(logger.handlers):
|
|
if getattr(handler, "_DeepData_managed", False):
|
|
logger.removeHandler(handler)
|
|
handler.close()
|
|
|
|
|
|
def _mark_managed(handler: logging.Handler) -> logging.Handler:
|
|
setattr(handler, "_DeepData_managed", True)
|
|
return handler
|
|
|
|
|
|
def setup_logging():
|
|
|
|
log_format = "%(asctime)s [%(levelname)s] - %(name)s - %(message)s"
|
|
date_format = "%Y-%m-%d %H:%M:%S"
|
|
|
|
console_handler = _mark_managed(logging.StreamHandler(sys.stdout))
|
|
console_handler.setLevel(getattr(logging, settings.log_level.upper()))
|
|
|
|
log_dir = Path(settings.log_file_dir)
|
|
log_dir.mkdir(exist_ok=True)
|
|
file_handler = _mark_managed(RotatingFileHandler(
|
|
log_dir / "python-backend.log",
|
|
maxBytes=_parse_size_to_bytes(settings.log_rotation_max_size),
|
|
backupCount=max(1, settings.log_rotation_backup_count),
|
|
encoding="utf-8"
|
|
))
|
|
file_handler.namer = _rotated_log_namer
|
|
file_handler.setLevel(getattr(logging, settings.log_level.upper()))
|
|
|
|
# Style setting - Centered level names
|
|
formatter = CenteredLevelNameFormatter(log_format, date_format)
|
|
console_handler.setFormatter(formatter)
|
|
file_handler.setFormatter(formatter)
|
|
|
|
# Root Logger
|
|
root_logger = logging.getLogger()
|
|
_remove_managed_handlers(root_logger)
|
|
root_logger.setLevel(getattr(logging, settings.log_level.upper()))
|
|
root_logger.addHandler(console_handler)
|
|
root_logger.addHandler(file_handler)
|
|
|
|
# Uvicorn
|
|
uvicorn_logger = logging.getLogger("uvicorn")
|
|
uvicorn_logger.handlers.clear()
|
|
uvicorn_logger.addHandler(console_handler)
|
|
uvicorn_logger.setLevel(logging.INFO)
|
|
|
|
uvicorn_access = logging.getLogger("uvicorn.access")
|
|
uvicorn_access.handlers.clear()
|
|
uvicorn_access.addHandler(console_handler)
|
|
uvicorn_access.setLevel(logging.DEBUG)
|
|
|
|
uvicorn_error = logging.getLogger("uvicorn.error")
|
|
uvicorn_error.handlers.clear()
|
|
uvicorn_error.addHandler(console_handler)
|
|
uvicorn_error.setLevel(logging.ERROR)
|
|
|
|
# SQLAlchemy (ERROR only)
|
|
sqlalchemy_logger = logging.getLogger("sqlalchemy.engine")
|
|
_remove_managed_handlers(sqlalchemy_logger)
|
|
sqlalchemy_logger.setLevel(logging.ERROR)
|
|
sqlalchemy_logger.addHandler(console_handler)
|
|
sqlalchemy_logger.setLevel(logging.ERROR)
|
|
|
|
# Minimize noise from HTTPX and HTTPCore
|
|
logging.getLogger("httpx").setLevel(logging.ERROR)
|
|
logging.getLogger("httpcore").setLevel(logging.ERROR)
|
|
|
|
def get_logger(name: str) -> logging.Logger:
|
|
return logging.getLogger(name)
|