deepdata: core product code (backend, frontend, runtime, deepflow module)

runtime/deepdata-python/app/module/shared/util/model_chat.py

@@ -0,0 +1,87 @@
+from openai import OpenAI
+
+
+def call_openai_style_model(base_url, api_key, model_name, prompt, **kwargs):
+    client = OpenAI(
+        base_url=base_url,
+        api_key=api_key
+    )
+
+    response = client.chat.completions.create(
+        model=model_name,
+        messages=[{"role": "user", "content": prompt}],
+        **kwargs
+    )
+    return response.choices[0].message.content
+
+
+def extract_json_substring(raw: str) -> str:
+    """从 LLM 的原始回答中提取最可能的 JSON 字符串片段。
+
+    处理思路：
+    - 原始回答可能是：说明文字 + JSON + 说明文字，甚至带有 Markdown 代码块。
+    - 优先在文本中查找第一个 '{' 或 '[' 作为 JSON 起始；
+    - 再从后向前找最后一个 '}' 或 ']' 作为结束；
+    - 如果找不到合适的边界，就退回原始字符串。
+    - 部分模型可能会在回复中加入 `<think>...</think>` 内部思考内容，应在解析前先去除。
+    - 也有模型会在 JSON 前后增加如 <reasoning>...</reasoning>、<analysis>...</analysis> 等标签，本方法会一并去除。
+    该方法不会保证截取的一定是合法 JSON，但能显著提高 json.loads 的成功率。
+    """
+    if not raw:
+        return raw
+
+    try:
+        import re
+
+        # 1. 先把所有完整的思考标签块整体去掉：<think>...</think> 等
+        thought_tags = [
+            "think",
+            "thinking",
+            "analysis",
+            "reasoning",
+            "reflection",
+            "inner_thoughts",
+        ]
+        for tag in thought_tags:
+            pattern = rf"<{tag}>[\s\S]*?</{tag}>"
+            raw = re.sub(pattern, "", raw, flags=re.IGNORECASE)
+
+        # 2. 再做一次“截取最后一个 </think>（或其它思考标签结束）之后的内容”的兜底
+        #    这样就算标签不成对或嵌套异常，也能保留尾部真正的回答
+        last_pos = -1
+        for tag in thought_tags:
+            # 匹配类似 </think> 或 </THINK>
+            m = list(re.finditer(rf"</{tag}>", raw, flags=re.IGNORECASE))
+            if m:
+                last_pos = max(last_pos, m[-1].end())
+        if last_pos != -1 and last_pos < len(raw):
+            raw = raw[last_pos:]
+
+    except Exception:
+        # 正则异常时不影响后续逻辑，继续使用当前文本
+        pass
+
+    start = None
+    end = None
+
+    # 查找第一个 JSON 起始符号
+    for i, ch in enumerate(raw):
+        if ch in "[{":
+            start = i
+            break
+
+    # 查找最后一个 JSON 结束符号
+    for i in range(len(raw) - 1, -1, -1):
+        if raw[i] in "]}":
+            end = i + 1  # 切片是左闭右开
+            break
+
+    if start is not None and end is not None and start < end:
+        return raw[start:end].strip()
+
+    # 兜底：去掉常见 Markdown 包裹（```json ... ```）
+    stripped = raw.strip()
+    if stripped.startswith("```"):
+        # 去掉首尾 ``` 标记
+        stripped = stripped.strip("`")
+    return stripped

runtime/deepdata-python/app/module/shared/util/resource_utils.py

@@ -0,0 +1,159 @@
+"""
+资源感知并发计算工具
+
+根据系统 CPU 和内存资源动态计算最优并发数
+适配全闪存储等高性能存储场景
+"""
+import os
+import logging
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# 尝试导入 psutil，失败则使用 os.cpu_count
+try:
+    import psutil
+    HAS_PSUTIL = True
+except ImportError:
+    HAS_PSUTIL = False
+    logger.warning("psutil not available, falling back to os.cpu_count() for CPU detection")
+
+
+def get_cpu_count(logical: bool = True) -> int:
+    """获取 CPU 核心数
+    
+    Args:
+        logical: 是否返回逻辑核心数（超线程），False返回物理核心数
+        
+    Returns:
+        CPU 核心数，无法检测时返回默认值 4
+    """
+    if HAS_PSUTIL:
+        count = psutil.cpu_count(logical=logical)
+        if count:
+            return count
+    
+    # 回退到 os.cpu_count()
+    count = os.cpu_count()
+    return count if count else 4
+
+
+def get_available_memory_gb() -> float:
+    """获取可用内存（GB）
+    
+    Returns:
+        可用内存大小（GB），无法检测时返回默认值 8.0
+    """
+    if HAS_PSUTIL:
+        mem = psutil.virtual_memory()
+        return mem.available / (1024 ** 3)
+    
+    # 无法检测，返回保守默认值
+    logger.warning("psutil not available, using default memory estimate")
+    return 8.0
+
+
+def calculate_optimal_concurrent(
+    min_concurrent: int = 8,
+    max_concurrent: int = 128,
+    cpu_factor: float = 4.0,
+    memory_per_task_mb: int = 32,
+    memory_reserve_ratio: float = 0.2,
+) -> int:
+    """根据系统资源计算最优并发数
+    
+    计算逻辑：
+    1. 基于 CPU 核心数：并发数 = CPU核心数 × cpu_factor
+    2. 基于内存限制：可用内存 × (1-reserve_ratio) ÷ memory_per_task
+    3. 最终取两者最小值，并限制在 [min_concurrent, max_concurrent] 范围内
+    
+    Args:
+        min_concurrent: 并发下限
+        max_concurrent: 并发上限
+        cpu_factor: CPU核心系数（全闪存储建议4.0）
+        memory_per_task_mb: 每任务预估内存占用（MB）
+        memory_reserve_ratio: 内存安全保留比例
+        
+    Returns:
+        计算得出的最优并发数
+    """
+    # 获取 CPU 核心数
+    cpu_count = get_cpu_count(logical=True)
+    
+    # 基于 CPU 计算并发数
+    cpu_based_concurrent = int(cpu_count * cpu_factor)
+    
+    # 获取可用内存
+    available_memory_gb = get_available_memory_gb()
+    
+    # 计算可用于并发任务的内存（扣除保留部分）
+    usable_memory_gb = available_memory_gb * (1 - memory_reserve_ratio)
+    usable_memory_mb = usable_memory_gb * 1024
+    
+    # 基于内存计算最大并发数
+    memory_based_concurrent = int(usable_memory_mb / memory_per_task_mb)
+    
+    # 取两者最小值（避免内存耗尽）
+    calculated_concurrent = min(cpu_based_concurrent, memory_based_concurrent)
+    
+    # 限制在范围内
+    final_concurrent = max(min_concurrent, min(max_concurrent, calculated_concurrent))
+    
+    # 记录计算过程
+    logger.info(
+        f"Concurrency calculation: "
+        f"CPU cores={cpu_count}, CPU-based={cpu_based_concurrent}, "
+        f"Available memory={available_memory_gb:.1f}GB, Memory-based={memory_based_concurrent}, "
+        f"Final concurrent={final_concurrent} (range=[{min_concurrent}, {max_concurrent}])"
+    )
+    
+    return final_concurrent
+
+
+def get_concurrent_for_ratio_copy(settings) -> int:
+    """获取配比文件复制的并发数
+    
+    Args:
+        settings: 配置实例
+        
+    Returns:
+        并发数（动态计算或固定值）
+    """
+    if not settings.ratio_copy_dynamic_concurrent:
+        return settings.ratio_copy_fixed_concurrent
+    
+    return calculate_optimal_concurrent(
+        min_concurrent=settings.ratio_copy_min_concurrent,
+        max_concurrent=settings.ratio_copy_max_concurrent,
+        cpu_factor=settings.ratio_copy_cpu_factor,
+        memory_per_task_mb=settings.ratio_copy_memory_per_task_mb,
+        memory_reserve_ratio=settings.ratio_copy_memory_reserve_ratio,
+    )
+
+
+def get_system_resource_info() -> dict:
+    """获取系统资源信息（用于日志和调试）
+    
+    Returns:
+        系统资源信息字典
+    """
+    cpu_logical = get_cpu_count(logical=True)
+    cpu_physical = get_cpu_count(logical=False) if HAS_PSUTIL else cpu_logical
+    
+    if HAS_PSUTIL:
+        mem = psutil.virtual_memory()
+        return {
+            "cpu_logical_cores": cpu_logical,
+            "cpu_physical_cores": cpu_physical,
+            "memory_total_gb": round(mem.total / (1024 ** 3), 2),
+            "memory_available_gb": round(mem.available / (1024 ** 3), 2),
+            "memory_used_percent": round(mem.percent, 1),
+            "psutil_available": True,
+        }
+    
+    return {
+        "cpu_logical_cores": cpu_logical,
+        "cpu_physical_cores": cpu_physical,
+        "memory_available_gb": get_available_memory_gb(),
+        "psutil_available": False,
+    }

runtime/deepdata-python/app/module/shared/util/structured_file.py

@@ -0,0 +1,113 @@
+import json
+
+from enum import Enum
+from jsonschema import validate
+
+class ItemTypes(Enum):
+    QA = "QA"
+    COT = "COT"
+
+
+class StructuredFileItemHandler:
+    def __init__(self):
+        pass
+
+    def get_item_type(self) -> ItemTypes:
+        pass
+
+    def validate_json(self, data):
+        pass
+
+    def get_items_from_file(self, file_path: str) -> list[dict]:
+        file_type = file_path.split(".")[-1].upper()
+        items = []
+        if file_type == "JSON":
+            with open(file_path, "r", encoding="utf-8") as f:
+                data = json.load(f)
+                if not self.validate_json(data):
+                    return items
+                items = data
+        elif file_type == "JSONL":
+            with open(file_path, "r", encoding="utf-8") as f:
+                for line in f:
+                    data = json.loads(line)
+                    if not self.validate_json(data):
+                        continue
+                    items.append(data)
+        return items
+
+class QAItemHandler(StructuredFileItemHandler):
+    def __init__(self):
+        self.schema_alpaca = {
+            "type": "object",
+            "properties": {
+                "instruction": {"type": "string"},
+                "input": {"type": "string"},
+                "output": {"type": "string"}
+            },
+            "required": ["instruction", "output"],
+        }
+        self.schema_alpaca_list = {
+            "type": "array",
+            "items": self.schema_alpaca,
+        }
+        super().__init__()
+
+    def get_item_type(self):
+        return ItemTypes.QA
+
+    def validate_json(self, data):
+        try:
+            validate(instance=data, schema=self.schema_alpaca)
+            return True
+        except Exception as e:
+            try:
+                validate(instance=data, schema=self.schema_alpaca_list)
+                return True
+            except Exception as e:
+                return False
+
+
+class COTItemHandler(StructuredFileItemHandler):
+    def __init__(self):
+        self.schema = {
+            "type": "object",
+            "properties": {
+                "instruction": {"type": "string"},
+                "output": {"type": "string"},
+                "chain_of_thought": {"type": "string"}
+            },
+            "required": ["question", "instruction", "output"],
+        }
+        self.schema_list = {
+            "type": "array",
+            "items": self.schema,
+        }
+        super().__init__()
+
+    def get_item_type(self):
+        return ItemTypes.COT
+
+    def validate_json(self, data):
+        try:
+            validate(instance=data, schema=self.schema)
+            return True
+        except Exception as e:
+            try:
+                validate(instance=data, schema=self.schema_list)
+                return True
+            except Exception as e:
+                return False
+
+
+class StructuredFileHandlerFactory:
+    def __init__(self):
+        self.handlers: list[StructuredFileItemHandler] = []
+        self.handlers.append(QAItemHandler())
+        self.handlers.append(COTItemHandler())
+
+    def get_handler(self, item_type: str) -> StructuredFileItemHandler:
+        for handler in self.handlers:
+            if handler.get_item_type().value == item_type:
+                return handler
+        raise ValueError(f"Unsupported item type: {item_type}")