deepdata: core product code (backend, frontend, runtime, deepflow module)

2026-08-30 19:23:38 +08:00
commit bd3db30be9
1405 changed files with 271727 additions and 0 deletions
@@ -0,0 +1,159 @@
+"""
+资源感知并发计算工具
+
+根据系统 CPU 和内存资源动态计算最优并发数
+适配全闪存储等高性能存储场景
+"""
+import os
+import logging
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# 尝试导入 psutil，失败则使用 os.cpu_count
+try:
+    import psutil
+    HAS_PSUTIL = True
+except ImportError:
+    HAS_PSUTIL = False
+    logger.warning("psutil not available, falling back to os.cpu_count() for CPU detection")
+
+
+def get_cpu_count(logical: bool = True) -> int:
+    """获取 CPU 核心数
+    
+    Args:
+        logical: 是否返回逻辑核心数（超线程），False返回物理核心数
+        
+    Returns:
+        CPU 核心数，无法检测时返回默认值 4
+    """
+    if HAS_PSUTIL:
+        count = psutil.cpu_count(logical=logical)
+        if count:
+            return count
+    
+    # 回退到 os.cpu_count()
+    count = os.cpu_count()
+    return count if count else 4
+
+
+def get_available_memory_gb() -> float:
+    """获取可用内存（GB）
+    
+    Returns:
+        可用内存大小（GB），无法检测时返回默认值 8.0
+    """
+    if HAS_PSUTIL:
+        mem = psutil.virtual_memory()
+        return mem.available / (1024 ** 3)
+    
+    # 无法检测，返回保守默认值
+    logger.warning("psutil not available, using default memory estimate")
+    return 8.0
+
+
+def calculate_optimal_concurrent(
+    min_concurrent: int = 8,
+    max_concurrent: int = 128,
+    cpu_factor: float = 4.0,
+    memory_per_task_mb: int = 32,
+    memory_reserve_ratio: float = 0.2,
+) -> int:
+    """根据系统资源计算最优并发数
+    
+    计算逻辑：
+    1. 基于 CPU 核心数：并发数 = CPU核心数 × cpu_factor
+    2. 基于内存限制：可用内存 × (1-reserve_ratio) ÷ memory_per_task
+    3. 最终取两者最小值，并限制在 [min_concurrent, max_concurrent] 范围内
+    
+    Args:
+        min_concurrent: 并发下限
+        max_concurrent: 并发上限
+        cpu_factor: CPU核心系数（全闪存储建议4.0）
+        memory_per_task_mb: 每任务预估内存占用（MB）
+        memory_reserve_ratio: 内存安全保留比例
+        
+    Returns:
+        计算得出的最优并发数
+    """
+    # 获取 CPU 核心数
+    cpu_count = get_cpu_count(logical=True)
+    
+    # 基于 CPU 计算并发数
+    cpu_based_concurrent = int(cpu_count * cpu_factor)
+    
+    # 获取可用内存
+    available_memory_gb = get_available_memory_gb()
+    
+    # 计算可用于并发任务的内存（扣除保留部分）
+    usable_memory_gb = available_memory_gb * (1 - memory_reserve_ratio)
+    usable_memory_mb = usable_memory_gb * 1024
+    
+    # 基于内存计算最大并发数
+    memory_based_concurrent = int(usable_memory_mb / memory_per_task_mb)
+    
+    # 取两者最小值（避免内存耗尽）
+    calculated_concurrent = min(cpu_based_concurrent, memory_based_concurrent)
+    
+    # 限制在范围内
+    final_concurrent = max(min_concurrent, min(max_concurrent, calculated_concurrent))
+    
+    # 记录计算过程
+    logger.info(
+        f"Concurrency calculation: "
+        f"CPU cores={cpu_count}, CPU-based={cpu_based_concurrent}, "
+        f"Available memory={available_memory_gb:.1f}GB, Memory-based={memory_based_concurrent}, "
+        f"Final concurrent={final_concurrent} (range=[{min_concurrent}, {max_concurrent}])"
+    )
+    
+    return final_concurrent
+
+
+def get_concurrent_for_ratio_copy(settings) -> int:
+    """获取配比文件复制的并发数
+    
+    Args:
+        settings: 配置实例
+        
+    Returns:
+        并发数（动态计算或固定值）
+    """
+    if not settings.ratio_copy_dynamic_concurrent:
+        return settings.ratio_copy_fixed_concurrent
+    
+    return calculate_optimal_concurrent(
+        min_concurrent=settings.ratio_copy_min_concurrent,
+        max_concurrent=settings.ratio_copy_max_concurrent,
+        cpu_factor=settings.ratio_copy_cpu_factor,
+        memory_per_task_mb=settings.ratio_copy_memory_per_task_mb,
+        memory_reserve_ratio=settings.ratio_copy_memory_reserve_ratio,
+    )
+
+
+def get_system_resource_info() -> dict:
+    """获取系统资源信息（用于日志和调试）
+    
+    Returns:
+        系统资源信息字典
+    """
+    cpu_logical = get_cpu_count(logical=True)
+    cpu_physical = get_cpu_count(logical=False) if HAS_PSUTIL else cpu_logical
+    
+    if HAS_PSUTIL:
+        mem = psutil.virtual_memory()
+        return {
+            "cpu_logical_cores": cpu_logical,
+            "cpu_physical_cores": cpu_physical,
+            "memory_total_gb": round(mem.total / (1024 ** 3), 2),
+            "memory_available_gb": round(mem.available / (1024 ** 3), 2),
+            "memory_used_percent": round(mem.percent, 1),
+            "psutil_available": True,
+        }
+    
+    return {
+        "cpu_logical_cores": cpu_logical,
+        "cpu_physical_cores": cpu_physical,
+        "memory_available_gb": get_available_memory_gb(),
+        "psutil_available": False,
+    }