feat(deepdata): 引入修改密码、文件预览组件与清洗算子增强，deepflow 改名为 deep-flow
本次同步 09-12 至 09-28 的代码变更，主要内容： 新增功能 - 修改密码：后端新增 ChangePasswordRequest DTO 与UserController/UserService 对应接口，前端新增 ChangePasswordDialog 并在 Header 接入入口 - 文件预览组件重构：file-preview 目录拆分出 VideoPreview（支持 blobUrl 直传）、 ImagePreview、AudioPreview，index.tsx 统一调度各类型预览器 - 新增 my_ops 算子模块：markitdown_parser、mineru_pdf_parser、text_quality_evaluator - 新增 tools/minio-file-server MinIO 文件服务 功能增强 - 数据清洗：cleaning_task_service/scheduler/routes 重构， data_juicer_executor 与 wrapper 增强，OperatorTable/LogsTable/TaskDetail 同步调整 - 数据标注：annotation sync.py、project.py 接口与schema 更新 - 数据采集、算子市场、首页与菜单路由相关页面调整 - DataJuicer 算子接入（useOperatorOperations、dj_ops_schema） 结构调整 - deepflow/ 更名为 deep-flow/（419 个文件重命名） - 新增 helm 部署编排、GitHub Actions 之外的自建镜像构建脚本 未纳入本次提交 - scripts-backup/、runtime/python-executor-backup/、components_backup/（备份目录） - *.backup*、__pycache__ 产物 - zh_core_web_sm-3.8.0-py3-none-any.whl（46MB 构建产物） - DataX/（内嵌独立仓库，远程亦未收录其源码）
This commit is contained in:
沃巴蒂
@@ -0,0 +1,25 @@
|
||||
# 基础镜像：python3.11 slim
|
||||
FROM python:3.11-slim
|
||||
|
||||
# 设置工作目录
|
||||
WORKDIR /app
|
||||
|
||||
# 安装系统基础依赖（可选，解决部分包编译/字符编码问题）
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 先复制依赖文件，利用docker缓存层，requirements变更才会重新pip安装
|
||||
COPY requirements.txt .
|
||||
|
||||
# 安装python依赖
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# 复制业务代码
|
||||
COPY main.py .
|
||||
|
||||
# 暴露端口（仅文档提示，不实际开放端口）
|
||||
EXPOSE 8000
|
||||
|
||||
# 启动命令
|
||||
CMD ["python", "main.py", "--port", "8000"]
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 62 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 62 KiB |
@@ -0,0 +1,60 @@
|
||||
# client.py
|
||||
import requests
|
||||
import base64
|
||||
import argparse
|
||||
|
||||
BASE_URL = "http://127.0.0.1:8000"
|
||||
|
||||
def test_upload_form_file(file_path: str):
|
||||
"""表单文件上传（二进制文件）"""
|
||||
url = f"{BASE_URL}/upload/form"
|
||||
with open(file_path, "rb") as f:
|
||||
files = {"file": f}
|
||||
resp = requests.post(url, files=files)
|
||||
print("=== Form上传结果 ===")
|
||||
print("status_code:", resp.status_code)
|
||||
print("resp:", resp.json())
|
||||
if resp.status_code == 200:
|
||||
path = resp.json()
|
||||
md_link = f""
|
||||
dl_link = f"{BASE_URL}{path}?dl=1"
|
||||
print(f"Markdown图片链接：{md_link}")
|
||||
print(f"下载链接：{dl_link}\n")
|
||||
return path
|
||||
return None
|
||||
|
||||
def test_upload_base64(file_path: str, filename: str):
|
||||
"""base64上传，POST JSON"""
|
||||
url = f"{BASE_URL}/upload/base64"
|
||||
with open(file_path, "rb") as f:
|
||||
raw = f.read()
|
||||
b64_str = base64.b64encode(raw).decode("utf-8")
|
||||
payload = {
|
||||
"base64_data": b64_str,
|
||||
"filename": filename
|
||||
}
|
||||
resp = requests.post(url, json=payload)
|
||||
print("=== Base64上传结果 ===")
|
||||
print("status_code:", resp.status_code)
|
||||
print("resp:", resp.json())
|
||||
if resp.status_code == 200:
|
||||
path = resp.json()
|
||||
md_link = f""
|
||||
dl_link = f"{BASE_URL}{path}?dl=1"
|
||||
print(f"Markdown图片链接：{md_link}")
|
||||
print(f"下载链接：{dl_link}\n")
|
||||
return path
|
||||
return None
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='')
|
||||
parser.add_argument("--port", type=int,
|
||||
default=8000, help="缺省: 8000")
|
||||
parser.add_argument("--test_file", type=str,
|
||||
default="test.jpg", help="缺省: test.jpg")
|
||||
args = parser.parse_args()
|
||||
|
||||
test_file = args.test_file
|
||||
BASE_URL = f"http://127.0.0.1:{args.port}"
|
||||
test_upload_form_file(test_file)
|
||||
test_upload_base64(test_file, "风景.jpg")
|
||||
@@ -0,0 +1,15 @@
|
||||
version: "3.8"
|
||||
services:
|
||||
minio-file-server:
|
||||
image: minio-file-server:lastest # python:3.11-slim
|
||||
container_name: minio-file-server
|
||||
restart: always
|
||||
ports:
|
||||
- "18014:8000"
|
||||
volumes:
|
||||
- ./main.py:/app/main.py
|
||||
- ./.env:/app/.env
|
||||
- ./requirements.txt:/app/requirements.txt
|
||||
working_dir: /app
|
||||
command: >
|
||||
sh -c "pip install -r requirements.txt && python main.py --port 8000"
|
||||
@@ -0,0 +1,241 @@
|
||||
import base64
|
||||
import uuid
|
||||
import io
|
||||
import urllib.parse
|
||||
from typing import Optional
|
||||
from fastapi import FastAPI, UploadFile, File, Query, HTTPException, Body
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
from minio import Minio
|
||||
from minio.error import S3Error
|
||||
import argparse
|
||||
|
||||
import base64
|
||||
import uuid
|
||||
import io
|
||||
import urllib.parse
|
||||
import mimetypes
|
||||
import os
|
||||
from typing import Optional
|
||||
from fastapi import FastAPI, UploadFile, File, Query, HTTPException, Body
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
from minio import Minio
|
||||
from minio.error import S3Error
|
||||
|
||||
# 加载环境变量
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env", env_file_encoding="utf-8")
|
||||
minio_endpoint: str
|
||||
minio_access_key: str
|
||||
minio_secret_key: str
|
||||
minio_secure: bool
|
||||
minio_bucket: str
|
||||
public_prefix: str
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
||||
# MinIO Client
|
||||
minio_client = Minio(
|
||||
endpoint=settings.minio_endpoint,
|
||||
access_key=settings.minio_access_key,
|
||||
secret_key=settings.minio_secret_key,
|
||||
secure=settings.minio_secure
|
||||
)
|
||||
|
||||
# 初始化FastAPI
|
||||
app = FastAPI(
|
||||
title="MinIO 文件服务",
|
||||
description="原始文件名存入MinIO对象元数据；上传仅返回MD图片相对路径，访问时自动读取元数据作为下载名称",
|
||||
version="1.0.0"
|
||||
)
|
||||
|
||||
|
||||
def get_ext(filename: str) -> str:
|
||||
if "." in filename:
|
||||
return filename.split(".")[-1]
|
||||
return ""
|
||||
|
||||
|
||||
def build_relative_path(obj_key: str) -> str:
|
||||
"""生成用于Markdown图片的相对路径"""
|
||||
return f"/file/{obj_key}"
|
||||
|
||||
|
||||
@app.post("/upload/form", summary="Form表单上传，仅返回相对路径，原始文件名存入MinIO元数据")
|
||||
async def upload_form_file(
|
||||
file: UploadFile = File(..., description="文件"),
|
||||
filename: Optional[str] = Query(
|
||||
default=None, description="自定义文件名，不传使用原始文件名")
|
||||
):
|
||||
use_filename = filename or file.filename or "file"
|
||||
ext = get_ext(use_filename)
|
||||
obj_key = f"{settings.public_prefix}/{uuid.uuid4()}.{ext}"
|
||||
# ✅ URL编码中文文件名，解决header latin1编码报错
|
||||
metadata = {
|
||||
"X-Amz-Meta-Original-Name": urllib.parse.quote(use_filename)
|
||||
}
|
||||
try:
|
||||
minio_client.put_object(
|
||||
bucket_name=settings.minio_bucket,
|
||||
object_name=obj_key,
|
||||
data=file.file,
|
||||
length=-1,
|
||||
part_size=10*1024*1024,
|
||||
content_type=file.content_type,
|
||||
metadata=metadata
|
||||
)
|
||||
path = build_relative_path(obj_key)
|
||||
return {"path": path}
|
||||
except S3Error as e:
|
||||
raise HTTPException(status_code=500, detail=f"MinIO上传失败: {str(e)}")
|
||||
|
||||
|
||||
@app.post("/upload/base64", summary="Base64上传，仅返回相对路径，原始文件名存入MinIO元数据")
|
||||
async def upload_base64(
|
||||
base64_data: str = Body(),
|
||||
filename: str = Body()
|
||||
):
|
||||
if base64_data.startswith("data:"):
|
||||
base64_data = base64_data.split(",")[-1]
|
||||
try:
|
||||
raw_bytes = base64.b64decode(base64_data)
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="base64数据解码失败")
|
||||
|
||||
ext = get_ext(filename)
|
||||
obj_key = f"{settings.public_prefix}/{uuid.uuid4()}.{ext}"
|
||||
# ✅ URL编码中文文件名
|
||||
metadata = {
|
||||
"X-Amz-Meta-Original-Name": urllib.parse.quote(filename)
|
||||
}
|
||||
# 猜测content-type
|
||||
content_type = "application/octet-stream"
|
||||
if ext.lower() in ["jpg", "jpeg"]:
|
||||
content_type = "image/jpeg"
|
||||
elif ext.lower() == "png":
|
||||
content_type = "image/png"
|
||||
elif ext.lower() == "gif":
|
||||
content_type = "image/gif"
|
||||
elif ext.lower() == "pdf":
|
||||
content_type = "application/pdf"
|
||||
elif ext.lower() == "txt":
|
||||
content_type = "text/plain"
|
||||
|
||||
stream = io.BytesIO(raw_bytes)
|
||||
try:
|
||||
minio_client.put_object(
|
||||
bucket_name=settings.minio_bucket,
|
||||
object_name=obj_key,
|
||||
data=stream,
|
||||
length=len(raw_bytes),
|
||||
content_type=content_type,
|
||||
metadata=metadata
|
||||
)
|
||||
path = build_relative_path(obj_key)
|
||||
return {"path": path}
|
||||
except S3Error as e:
|
||||
raise HTTPException(status_code=500, detail=f"MinIO上传失败: {str(e)}")
|
||||
|
||||
|
||||
@app.get("/file/{object_key:path}", summary="文件访问：默认预览，dl=1强制下载，自动读取MinIO元数据中的原始文件名")
|
||||
def file_access(
|
||||
object_key: str,
|
||||
dl: int = Query(default=0, description="dl=1强制下载；默认0在线预览"),
|
||||
name: str = Query(default="", description="手动指定下载名称，优先级高于元数据")
|
||||
):
|
||||
content_type = "application/octet-stream"
|
||||
original_name = ""
|
||||
stream = None
|
||||
|
||||
# 1. 尝试使用 HEAD 请求获取元数据 (快，不消耗流量)
|
||||
try:
|
||||
stat = minio_client.stat_object(settings.minio_bucket, object_key)
|
||||
content_type = stat.content_type
|
||||
|
||||
# 提取原始文件名 (SDK 通常会去掉 x-amz-meta- 前缀并转小写)
|
||||
original_name = stat.metadata.get("original-name", "")
|
||||
|
||||
stream = minio_client.get_object(settings.minio_bucket, object_key)
|
||||
except S3Error as e:
|
||||
# 如果是文件不存在，直接 404
|
||||
if e.code == "NoSuchKey":
|
||||
raise HTTPException(status_code=404, detail=f"文件不存在: {str(e)}")
|
||||
|
||||
# 2. 降级策略：如果 HEAD 失败 (如 Nginx 代理丢弃鉴权头导致 AccessDenied)，尝试 GET 请求
|
||||
# 这能解决 stat_object 报错但实际有权限读取文件的问题
|
||||
try:
|
||||
response = minio_client.get_object(
|
||||
settings.minio_bucket, object_key)
|
||||
stream = response
|
||||
|
||||
# 从 HTTP Header 中提取信息
|
||||
if response.headers.get("content-type"):
|
||||
content_type = response.headers["content-type"]
|
||||
|
||||
# 提取原始文件名 (Header 中通常包含完整前缀 x-amz-meta-*)
|
||||
# 兼容大小写
|
||||
meta_val = response.headers.get("x-amz-meta-original-name") or \
|
||||
response.headers.get("X-Amz-Meta-Original-Name")
|
||||
if meta_val:
|
||||
original_name = meta_val
|
||||
|
||||
except S3Error as get_err:
|
||||
# 如果 GET 也失败，抛出错误
|
||||
raise HTTPException(status_code=404 if get_err.code == "NoSuchKey" else 500,
|
||||
detail=f"S3 操作失败: {str(get_err)}")
|
||||
except Exception as get_err:
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"读取文件流失败: {str(get_err)}")
|
||||
|
||||
# 3. Content-Type 兜底推断 (处理未设置 Content-Type 的文件)
|
||||
if content_type == "application/octet-stream":
|
||||
guessed_type, _ = mimetypes.guess_type(object_key)
|
||||
if guessed_type:
|
||||
content_type = guessed_type
|
||||
else:
|
||||
# 针对常见后缀手动映射，防止 mimetypes 未覆盖
|
||||
ext = object_key.split(".")[-1].lower()
|
||||
type_map = {
|
||||
'mp4': 'video/mp4', 'webm': 'video/webm', 'mov': 'video/quicktime',
|
||||
'zip': 'application/zip', 'rar': 'application/x-rar-compressed',
|
||||
'7z': 'application/x-7z-compressed', 'tar': 'application/x-tar',
|
||||
'gz': 'application/gzip', 'pdf': 'application/pdf'
|
||||
}
|
||||
content_type = type_map.get(ext, 'application/octet-stream')
|
||||
|
||||
# 4. 设置响应头
|
||||
resp_headers = {}
|
||||
|
||||
# 确定最终下载文件名
|
||||
# 优先级：URL参数 name > 元数据/HTTP Header > 对象 Key 中的文件名
|
||||
final_name = name or urllib.parse.unquote(
|
||||
original_name) or object_key.split("/")[-1]
|
||||
|
||||
if dl == 1:
|
||||
# 强制下载
|
||||
encoded_name = urllib.parse.quote(final_name)
|
||||
resp_headers["Content-Disposition"] = f'attachment; filename="{final_name}"; filename*=utf-8\'\'{encoded_name}'
|
||||
else:
|
||||
# 在线预览 (浏览器默认行为，显式设置 inline 可避免某些浏览器强制下载)
|
||||
pass
|
||||
|
||||
return StreamingResponse(
|
||||
stream,
|
||||
media_type=content_type,
|
||||
headers=resp_headers
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='')
|
||||
parser.add_argument("--port", type=int,
|
||||
default=8000, help="缺省: 8000")
|
||||
args = parser.parse_args()
|
||||
|
||||
import uvicorn
|
||||
uvicorn.run("main:app", host="0.0.0.0", port=args.port)
|
||||
@@ -0,0 +1,5 @@
|
||||
fastapi
|
||||
uvicorn[standard]
|
||||
minio
|
||||
pydantic-settings
|
||||
python-multipart
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 62 KiB |
@@ -0,0 +1,44 @@
|
||||
# test_minio_diag.py
|
||||
import os
|
||||
from minio import Minio
|
||||
from minio.error import S3Error
|
||||
|
||||
print("=== 1. 环境变量检查 ===")
|
||||
print(f"Endpoint: {os.getenv('MINIO_ENDPOINT')}")
|
||||
print(f"Access Key: {os.getenv('MINIO_ACCESS_KEY')}")
|
||||
print(f"Secret Key: {os.getenv('MINIO_SECRET_KEY')[:3]}***") # 隐藏部分
|
||||
print(f"Secure: {os.getenv('MINIO_SECURE')}")
|
||||
print(f"HTTP Proxy: {os.getenv('http_proxy', 'None')}") # 关键：检查是否有代理
|
||||
print(f"HTTPS Proxy: {os.getenv('https_proxy', 'None')}")
|
||||
|
||||
print("\n=== 2. 连接测试 ===")
|
||||
try:
|
||||
client = Minio(
|
||||
os.getenv("MINIO_ENDPOINT"),
|
||||
access_key=os.getenv("MINIO_ACCESS_KEY"),
|
||||
secret_key=os.getenv("MINIO_SECRET_KEY"),
|
||||
secure=os.getenv("MINIO_SECURE").lower() == "true"
|
||||
)
|
||||
|
||||
# 尝试列出 public/ 目录下的文件
|
||||
print(">>> 尝试列出对象 (ListObjects)...")
|
||||
bucket = os.getenv("MINIO_BUCKET")
|
||||
objs = list(client.list_objects(bucket, prefix="public/", recursive=True))
|
||||
if objs:
|
||||
obj_name = objs[0].object_name
|
||||
print(f"✅ 列表成功! 发现文件: {obj_name}")
|
||||
|
||||
# 尝试下载该文件
|
||||
print(f">>> 尝试下载对象 (GetObject): {obj_name}")
|
||||
stream = client.get_object(bucket, obj_name)
|
||||
print(f"✅ 下载成功! 文件大小: {len(stream.read())} bytes")
|
||||
else:
|
||||
print("⚠️ public/ 目录下没有文件")
|
||||
|
||||
except S3Error as e:
|
||||
print(f"❌ S3 错误! Code: {e.code}")
|
||||
print(f" Message: {e.message}")
|
||||
if "AccessDenied" in str(e):
|
||||
print("\n💡 结论: 你的 minioadmin 账号缺乏读取权限。请检查 MinIO 控制台的用户策略。")
|
||||
except Exception as e:
|
||||
print(f"❌ 未知错误: {e}")
|
||||
Reference in New Issue
Block a user