diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81b3cbd3a..244e68573 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,11 +35,15 @@ jobs: # --require-hashes so CI fails on lockfile drift, exactly like the # Docker build does. pip install --require-hashes -r requirements.lock.txt - pip install pytest pytest-asyncio httpx + pip install pytest pytest-asyncio httpx 'moto[s3,server]' mypy - name: Ruff run: pipx run ruff==0.16.6 check . + - name: Mypy ratchet (new type errors are rejected; fix and run + scripts/mypy_ratchet.py --regenerate to tighten the baseline) + run: python scripts/mypy_ratchet.py + - name: Verify lockfile matches requirements.txt # Dependabot bumps requirements.txt but cannot regenerate the hashed # lockfile; without this check a stale lockfile would silently keep diff --git a/Dockerfile b/Dockerfile index eedf04f97..382bb00ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,13 +59,17 @@ COPY --from=frontend-builder /build/fronted-2023/dist ./themes/2023 # 安装系统安全更新 + Python 依赖 # 依赖从带哈希的锁定文件安装（--require-hashes），保证构建可复现、防供应链篡改。 -# 清理 apt 缓存，降低镜像噪音与扫描面 +# gosu 用于入口脚本的数据卷属主修正后降权；清理 apt 缓存，降低镜像噪音与扫描面 RUN apt-get update \ && apt-get upgrade -y --no-install-recommends \ + && apt-get install -y --no-install-recommends gosu \ && rm -rf /var/lib/apt/lists/* \ && pip install --no-cache-dir --require-hashes -r requirements.lock.txt \ && pip cache purge || true +# 非 root 运行用户；数据卷属主由 docker-entrypoint.sh 按需修正（兼容存量 root 卷） +RUN useradd --system --uid 10001 --home-dir /app app + # 环境变量配置 ENV HOST="0.0.0.0" \ PORT=12345 \ @@ -77,6 +81,10 @@ ENV HOST="0.0.0.0" \ EXPOSE 12345 +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + # 生产环境启动命令 # FORWARDED_ALLOW_IPS 默认为空：仅信任直连 IP，避免任意客户端伪造 X-Forwarded-*。 # 若前面有反向代理，请显式设置为代理网段，例如 "10.0.0.0/8,172.16.0.0/12"。 diff --git a/apps/admin/config_service.py b/apps/admin/config_service.py new file mode 100644 index 000000000..60342472f --- /dev/null +++ b/apps/admin/config_service.py @@ -0,0 +1,159 @@ +"""System config write path (ConfigService) + the KeyValue JSON lock. + +keyvalue_write_lock lives here because ConfigService.update_config is its +primary consumer; FileService imports it (one-way, no cycle). +""" +import asyncio + +from fastapi import HTTPException + +from core.settings import ( + ADMIN_SESSION_EXPIRE_MAX, + ADMIN_SESSION_EXPIRE_MIN, + settings, +) +from apps.base.config import refresh_settings +from core.security import ( + INTERNAL_CONFIG_KEYS, + OUTBOUND_ENDPOINT_CONFIG_KEYS, + generate_jwt_secret, + validate_outbound_endpoint, + validate_outbound_hostname, +) +from apps.base.models import KeyValue +from core.utils import hash_password, is_password_hashed, validate_background_url + +# KeyValue 里的 settings/activities/presets 都是整块 JSON 读-改-写； +# 进程内写锁串行化这三个写路径，避免并发管理操作互相覆盖（last-writer-wins）。 +# 多进程部署下锁不跨进程——文档已锁定单 worker 部署。 +keyvalue_write_lock = asyncio.Lock() + + +class ConfigService: + INT_FIELDS = { + "admin_session_expire", + "enable_chunk", + "error_count", + "error_minute", + "login_count", + "login_minute", + "max_save_seconds", + "onedrive_proxy", + "open_upload", + "port", + "s3_proxy", + "server_port", + "server_workers", + "show_admin_addr", + "storage_limit", + "upload_count", + "upload_minute", + "upload_size", + "webdav_proxy", + } + FLOAT_FIELDS = {"opacity"} + + def get_config(self): + config = dict(settings.items()) + config["admin_token"] = "" + for key in INTERNAL_CONFIG_KEYS: + config.pop(key, None) + return config + + async def update_config(self, data: dict): + current_config = dict(settings.items()) + next_config = dict(current_config) + update_data = { + key: value + for key, value in data.items() + if key in settings.default_config and key not in INTERNAL_CONFIG_KEYS + } + + admin_token = update_data.get("admin_token") + admin_password_changed = False + if admin_token is None or admin_token == "": + update_data.pop("admin_token", None) + elif not is_password_hashed(admin_token): + update_data["admin_token"] = hash_password(admin_token) + admin_password_changed = True + else: + admin_password_changed = True + + for key, value in update_data.items(): + if value == "" and key in self.INT_FIELDS | self.FLOAT_FIELDS: + continue + + try: + if key in self.INT_FIELDS: + next_config[key] = int(value) + elif key in self.FLOAT_FIELDS: + next_config[key] = float(value) + else: + next_config[key] = value + except (TypeError, ValueError): + raise HTTPException(status_code=400, detail=f"{key} 配置值格式错误") + + try: + session_expire = int(str(next_config.get("admin_session_expire"))) + except (TypeError, ValueError): + raise HTTPException( + status_code=400, + detail="admin_session_expire 配置值格式错误", + ) + if ( + not ADMIN_SESSION_EXPIRE_MIN <= session_expire <= ADMIN_SESSION_EXPIRE_MAX + or session_expire % ADMIN_SESSION_EXPIRE_MIN != 0 + ): + raise HTTPException( + status_code=400, + detail="admin_session_expire 必须是 1 到 365 个整天", + ) + next_config["admin_session_expire"] = session_expire + + if int(next_config.get("storage_limit", 0)) < 0: + raise HTTPException( + status_code=400, + detail="storage_limit 不能小于 0", + ) + + # 只校验"发生变化"的值：升级前存入的旧格式 background（相对路径、含空格 + # 或括号）在旧版本是合法的，若每次保存都重新校验，存量部署会连无关设置项 + # 都保存不了（一律 400）。渲染侧仍然 html 转义，而任何修改都必须通过校验。 + current_background = str(settings.background or "") + candidate_background = str(next_config.get("background") or "") + if candidate_background != current_background: + try: + validate_background_url(candidate_background) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) + + # 只校验"发生变化"的值：企业内网 minio/webdav 是正当场景，存量部署历史 + # 合法写入的内网 endpoint 若每次保存都重新校验，会连无关设置都保存不了 + # （background 曾有同款回归，上游 #528 修复过——本处沿用同一语义）。 + # s3_hostname 是裸主机名（存储层按 https://{hostname} 拼接），单独分档校验。 + for endpoint_key in OUTBOUND_ENDPOINT_CONFIG_KEYS: + if endpoint_key not in next_config: + continue + candidate = str(next_config[endpoint_key] or "") + current = str(getattr(settings, endpoint_key, "") or "") + if candidate == current: + continue + validator = ( + validate_outbound_hostname + if endpoint_key == "s3_hostname" + else validate_outbound_endpoint + ) + try: + # 写回规范化值（validator 去除首尾空白）：校验通过但入库脏值 + # 会让存储层在连接期才报错，应在校验点归一。 + next_config[endpoint_key] = validator(candidate) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) + + if admin_password_changed: + next_config["jwt_secret"] = generate_jwt_secret() + + async with keyvalue_write_lock: + await KeyValue.update_or_create(key="settings", defaults={"value": next_config}) + await refresh_settings(force=True) + diff --git a/apps/admin/dependencies.py b/apps/admin/dependencies.py index 45dcebedd..a8af11f22 100644 --- a/apps/admin/dependencies.py +++ b/apps/admin/dependencies.py @@ -2,127 +2,40 @@ # @Author : Lan # @File : depends.py # @Software: PyCharm -from fastapi import Header, HTTPException -from fastapi.requests import Request -import base64 -import hmac -import json -import time -from core.settings import ( - ADMIN_SESSION_EXPIRE_DEFAULT, - ADMIN_SESSION_EXPIRE_MAX, - ADMIN_SESSION_EXPIRE_MIN, - settings, -) -from apps.admin.services import FileService, ConfigService, LocalFileService - - -def _get_jwt_secret() -> bytes: - secret = getattr(settings, "jwt_secret", "") - if not secret: - raise RuntimeError("JWT签名密钥未初始化") - return secret.encode() +from fastapi import Header +from fastapi.requests import Request -def get_admin_session_expire_seconds() -> int: - try: - expires_in = int( - getattr(settings, "admin_session_expire", ADMIN_SESSION_EXPIRE_DEFAULT) - ) - except (TypeError, ValueError): - return ADMIN_SESSION_EXPIRE_DEFAULT - if ( - not ADMIN_SESSION_EXPIRE_MIN <= expires_in <= ADMIN_SESSION_EXPIRE_MAX - or expires_in % ADMIN_SESSION_EXPIRE_MIN != 0 - ): - return ADMIN_SESSION_EXPIRE_DEFAULT - return expires_in - +from apps.admin.services import ConfigService, FileService, LocalFileService +from apps.base.auth import ( + _extract_bearer_token, + _get_jwt_secret, + _require_admin_payload, + create_token, + get_admin_session_expire_seconds, + share_required_login, + verify_token, +) -def create_token(data: dict, expires_in: int | None = None) -> str: - """ - 创建JWT token - :param data: 数据负载 - :param expires_in: 过期时间(秒) - """ - token_lifetime = ( - get_admin_session_expire_seconds() if expires_in is None else expires_in - ) - header = base64.urlsafe_b64encode( - json.dumps({"alg": "HS256", "typ": "JWT"}, separators=(",", ":")).encode() - ).decode().rstrip("=") - payload = base64.urlsafe_b64encode( - json.dumps( - {**data, "exp": int(time.time()) + token_lifetime}, - separators=(",", ":"), - ).encode() - ).decode().rstrip("=") - - signature = hmac.new( - _get_jwt_secret(), f"{header}.{payload}".encode(), "sha256" - ).digest() - signature = base64.urlsafe_b64encode(signature).decode().rstrip("=") - - return f"{header}.{payload}.{signature}" - - -def verify_token(token: str) -> dict: - """ - 验证JWT token - :param token: JWT token - :return: 解码后的数据 - """ - try: - header_b64, payload_b64, signature_b64 = token.split(".") - - # 验证签名 - expected_signature = hmac.new( - _get_jwt_secret(), - f"{header_b64}.{payload_b64}".encode(), - "sha256", - ).digest() - expected_signature_b64 = ( - base64.urlsafe_b64encode(expected_signature).decode().rstrip("=") - ) - - if not hmac.compare_digest(signature_b64, expected_signature_b64): - raise ValueError("无效的签名") - - # 解码payload（兼容历史标准 base64 与 urlsafe base64） - padded = payload_b64 + "=" * (-len(payload_b64) % 4) - try: - payload_bytes = base64.urlsafe_b64decode(padded) - except Exception: - payload_bytes = base64.b64decode(padded) - payload = json.loads(payload_bytes) - - # 检查是否过期 - if payload.get("exp", 0) < time.time(): - raise ValueError("token已过期") - - return payload - except Exception as e: - raise ValueError(f"token验证失败: {str(e)}") - - -def _extract_bearer_token(authorization: str) -> str: - if not authorization or not authorization.startswith("Bearer "): - raise HTTPException(status_code=401, detail="未授权或授权校验失败") - token = authorization.split(" ", 1)[1].strip() - if not token: - raise HTTPException(status_code=401, detail="未授权或授权校验失败") - return token - - -def _require_admin_payload(authorization: str) -> dict: - token = _extract_bearer_token(authorization) - try: - payload = verify_token(token) - except ValueError as e: - raise HTTPException(status_code=401, detail=str(e)) - if not payload.get("is_admin", False): - raise HTTPException(status_code=401, detail="未授权或授权校验失败") - return payload +# 认证原语（JWT 创建/校验、Bearer 提取、分享上传门控）已下沉 apps.base.auth +# ——base 分享面与本模块共同消费，留在 base 避免 base→admin 反向依赖。 +# 此处 re-export 保持既有 import 路径兼容（tests/admin.views 等）。 + +__all__ = [ + "_extract_bearer_token", + "_get_jwt_secret", + "_require_admin_payload", + "ADMIN_PUBLIC_ENDPOINTS", + "admin_required", + "create_token", + "get_admin_session", + "get_admin_session_expire_seconds", + "get_config_service", + "get_file_service", + "get_local_file_service", + "share_required_login", + "verify_token", +] def get_admin_session(authorization: str = Header(default=None)) -> dict: @@ -141,7 +54,8 @@ def get_admin_session(authorization: str = Header(default=None)) -> dict: async def admin_required( - authorization: str = Header(default=None), request: Request = None + authorization: str = Header(default=None), + request: Request = None, # type: ignore[assignment] # FastAPI 运行时注入 Request；可选语义由下方 if request 判断 ): """ 验证管理员权限 @@ -151,27 +65,6 @@ async def admin_required( return _require_admin_payload(authorization) -async def share_required_login(authorization: str = Header(default=None)): - """ - 验证分享上传权限 - - 当 settings.open_upload 为False时，要求用户必须登录并具有管理员权限 - 当 settings.open_upload 为True时，允许游客上传 - - :param authorization: 认证头信息 - :param request: 请求对象 - :return: 验证结果 - """ - if not settings.open_upload: - if not authorization or not authorization.startswith("Bearer "): - raise HTTPException( - status_code=403, detail="本站未开启游客上传，如需上传请先登录后台" - ) - _require_admin_payload(authorization) - - return True - - async def get_file_service(): return FileService() diff --git a/apps/admin/local_files.py b/apps/admin/local_files.py new file mode 100644 index 000000000..f349c520d --- /dev/null +++ b/apps/admin/local_files.py @@ -0,0 +1,98 @@ +"""Admin-side local (NAS) file browsing and deletion.""" +from pathlib import Path +from typing import Any + +from fastapi import HTTPException + +from apps.base.local_share import ( + MAX_LIST_ENTRIES, + format_local_ctime, + get_local_root, + normalize_local_relpath, + resolve_under_local, +) + + +class LocalFileService: + async def list_files(self, path: str = ""): + relpath = normalize_local_relpath(path, allow_empty=True) + directory = resolve_under_local(relpath) + if not directory.exists() or not directory.is_dir(): + raise HTTPException(status_code=404, detail="目录不存在") + + root = get_local_root() + items: list[dict[str, Any]] = [] + try: + children = list(directory.iterdir()) + except OSError as exc: + raise HTTPException(status_code=500, detail="无法读取目录") from exc + + children.sort(key=lambda p: (not p.is_dir(), p.name.lower())) + truncated = False + for child in children: + if len(items) >= MAX_LIST_ENTRIES: + truncated = True + break + try: + resolved = child.resolve() + resolved.relative_to(root) + except (OSError, ValueError): + continue + if not resolved.is_file() and not resolved.is_dir(): + continue + child_rel = child.name if not relpath else f"{relpath}/{child.name}" + is_dir = resolved.is_dir() + items.append( + { + "file": child.name, + "name": child.name, + "path": child_rel, + "type": "dir" if is_dir else "file", + "ctime": format_local_ctime(resolved), + "size": None if is_dir else resolved.stat().st_size, + } + ) + + parent = "" + if relpath: + parent_path = Path(relpath).parent.as_posix() + parent = "" if parent_path == "." else parent_path + return { + "path": relpath, + "parent": parent, + "truncated": truncated, + "items": items, + } + + async def delete_file(self, filename: str): + file = LocalFileClass(filename) + if await file.exists(): + await file.delete() + return "删除成功" + raise HTTPException(status_code=404, detail="文件不存在") + + +class LocalFileClass: + def __init__(self, file): + relpath = normalize_local_relpath(file) + self.file = relpath + self.name = Path(relpath).name + self.path = resolve_under_local(relpath) + if self.path.is_file(): + self.ctime = format_local_ctime(self.path) + self.size = self.path.stat().st_size + else: + self.ctime = None + self.size = None + + async def read(self) -> bytes: + with open(self.path, "rb") as fh: + return fh.read() + + async def delete(self): + if not self.path.is_file(): + raise HTTPException(status_code=404, detail="文件不存在") + self.path.unlink() + + async def exists(self): + return self.path.is_file() diff --git a/apps/admin/services.py b/apps/admin/services.py index be9ae4201..744f37e08 100644 --- a/apps/admin/services.py +++ b/apps/admin/services.py @@ -1,4 +1,3 @@ -import asyncio import hashlib from pathlib import Path from datetime import datetime, timedelta @@ -7,32 +6,19 @@ from core.response import APIResponse from core.storage import FileStorageInterface, storages from core.settings import ( - ADMIN_SESSION_EXPIRE_MAX, - ADMIN_SESSION_EXPIRE_MIN, settings, ) -from apps.base.config import refresh_settings from apps.base.services import get_stored_download, response_from_download, stored_file_of -from core.security import INTERNAL_CONFIG_KEYS, generate_jwt_secret from apps.base.models import FileCodes, KeyValue from apps.base.utils import get_expire_info from apps.base.local_share import ( LOCAL_REF_MARKER, - MAX_LIST_ENTRIES, - format_local_ctime, - get_local_root, is_local_ref, - normalize_local_relpath, - resolve_under_local, should_skip_storage_delete, ) from fastapi import HTTPException -from core.utils import get_now, hash_password, is_password_hashed, validate_background_url +from core.utils import get_now -# KeyValue 里的 settings/activities/presets 都是整块 JSON 读-改-写； -# 进程内写锁串行化这三个写路径，避免并发管理操作互相覆盖（last-writer-wins）。 -# 多进程部署下锁不跨进程——文档已锁定单 worker 部署。 -keyvalue_write_lock = asyncio.Lock() class FileService: @@ -1473,196 +1459,7 @@ async def share_local_file(self, item): } -class ConfigService: - INT_FIELDS = { - "admin_session_expire", - "enable_chunk", - "error_count", - "error_minute", - "login_count", - "login_minute", - "max_save_seconds", - "onedrive_proxy", - "open_upload", - "port", - "s3_proxy", - "server_port", - "server_workers", - "show_admin_addr", - "storage_limit", - "upload_count", - "upload_minute", - "upload_size", - "webdav_proxy", - } - FLOAT_FIELDS = {"opacity"} - - def get_config(self): - config = dict(settings.items()) - config["admin_token"] = "" - for key in INTERNAL_CONFIG_KEYS: - config.pop(key, None) - return config - - async def update_config(self, data: dict): - current_config = dict(settings.items()) - next_config = dict(current_config) - update_data = { - key: value - for key, value in data.items() - if key in settings.default_config and key not in INTERNAL_CONFIG_KEYS - } - - admin_token = update_data.get("admin_token") - admin_password_changed = False - if admin_token is None or admin_token == "": - update_data.pop("admin_token", None) - elif not is_password_hashed(admin_token): - update_data["admin_token"] = hash_password(admin_token) - admin_password_changed = True - else: - admin_password_changed = True - - for key, value in update_data.items(): - if value == "" and key in self.INT_FIELDS | self.FLOAT_FIELDS: - continue - - try: - if key in self.INT_FIELDS: - next_config[key] = int(value) - elif key in self.FLOAT_FIELDS: - next_config[key] = float(value) - else: - next_config[key] = value - except (TypeError, ValueError): - raise HTTPException(status_code=400, detail=f"{key} 配置值格式错误") - - try: - session_expire = int(next_config.get("admin_session_expire")) - except (TypeError, ValueError): - raise HTTPException( - status_code=400, - detail="admin_session_expire 配置值格式错误", - ) - if ( - not ADMIN_SESSION_EXPIRE_MIN <= session_expire <= ADMIN_SESSION_EXPIRE_MAX - or session_expire % ADMIN_SESSION_EXPIRE_MIN != 0 - ): - raise HTTPException( - status_code=400, - detail="admin_session_expire 必须是 1 到 365 个整天", - ) - next_config["admin_session_expire"] = session_expire - - if int(next_config.get("storage_limit", 0)) < 0: - raise HTTPException( - status_code=400, - detail="storage_limit 不能小于 0", - ) - - # 只校验"发生变化"的值：升级前存入的旧格式 background（相对路径、含空格 - # 或括号）在旧版本是合法的，若每次保存都重新校验，存量部署会连无关设置项 - # 都保存不了（一律 400）。渲染侧仍然 html 转义，而任何修改都必须通过校验。 - current_background = str(settings.background or "") - candidate_background = str(next_config.get("background") or "") - if candidate_background != current_background: - try: - validate_background_url(candidate_background) - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) - - if admin_password_changed: - next_config["jwt_secret"] = generate_jwt_secret() - - async with keyvalue_write_lock: - await KeyValue.update_or_create(key="settings", defaults={"value": next_config}) - await refresh_settings(force=True) - - -class LocalFileService: - async def list_files(self, path: str = ""): - relpath = normalize_local_relpath(path, allow_empty=True) - directory = resolve_under_local(relpath) - if not directory.exists() or not directory.is_dir(): - raise HTTPException(status_code=404, detail="目录不存在") - - root = get_local_root() - items = [] - try: - children = list(directory.iterdir()) - except OSError as exc: - raise HTTPException(status_code=500, detail="无法读取目录") from exc - - children.sort(key=lambda p: (not p.is_dir(), p.name.lower())) - truncated = False - for child in children: - if len(items) >= MAX_LIST_ENTRIES: - truncated = True - break - try: - resolved = child.resolve() - resolved.relative_to(root) - except (OSError, ValueError): - continue - if not resolved.is_file() and not resolved.is_dir(): - continue - child_rel = child.name if not relpath else f"{relpath}/{child.name}" - is_dir = resolved.is_dir() - items.append( - { - "file": child.name, - "name": child.name, - "path": child_rel, - "type": "dir" if is_dir else "file", - "ctime": format_local_ctime(resolved), - "size": None if is_dir else resolved.stat().st_size, - } - ) - - parent = "" - if relpath: - parent_path = Path(relpath).parent.as_posix() - parent = "" if parent_path == "." else parent_path - return { - "path": relpath, - "parent": parent, - "truncated": truncated, - "items": items, - } - - async def delete_file(self, filename: str): - file = LocalFileClass(filename) - if await file.exists(): - await file.delete() - return "删除成功" - raise HTTPException(status_code=404, detail="文件不存在") - - -class LocalFileClass: - def __init__(self, file): - relpath = normalize_local_relpath(file) - self.file = relpath - self.name = Path(relpath).name - self.path = resolve_under_local(relpath) - if self.path.is_file(): - self.ctime = format_local_ctime(self.path) - self.size = self.path.stat().st_size - else: - self.ctime = None - self.size = None - - async def read(self) -> bytes: - with open(self.path, "rb") as fh: - return fh.read() - - async def write(self, data): - with open(self.path, "wb") as f: - f.write(data) - - async def delete(self): - if not self.path.is_file(): - raise HTTPException(status_code=404, detail="文件不存在") - self.path.unlink() - async def exists(self): - return self.path.is_file() +# —— 门面 re-export：既有 import 路径（apps.admin.services.ConfigService 等）保持兼容 —— +from apps.admin.config_service import ConfigService, keyvalue_write_lock # noqa: E402,F401 +from apps.admin.local_files import LocalFileClass, LocalFileService # noqa: E402,F401 diff --git a/apps/admin/views.py b/apps/admin/views.py index aa665ff74..3ce196160 100644 --- a/apps/admin/views.py +++ b/apps/admin/views.py @@ -3,6 +3,7 @@ # @File : views.py # @Software: PyCharm import datetime +from typing import Any from collections import Counter from typing import Optional @@ -273,7 +274,7 @@ async def batch_update_files( if not data.ids: raise HTTPException(status_code=400, detail="请选择要更新的文件") - update_data = {} + update_data: dict[str, Any] = {} fields_set = data.model_fields_set should_clear_expired_at = bool(data.clear_expired_at) @@ -281,6 +282,7 @@ async def batch_update_files( update_data["expired_at"] = None update_data["expired_count"] = -1 elif "expired_at" in fields_set and data.expired_at != "": + # schema 声明 Union[datetime, str]——透传两种形态，tortoise 均可序列化 update_data["expired_at"] = data.expired_at if ( diff --git a/apps/base/auth.py b/apps/base/auth.py new file mode 100644 index 000000000..506a9b9fc --- /dev/null +++ b/apps/base/auth.py @@ -0,0 +1,150 @@ +"""Cross-cutting auth primitives and the share-upload gate. + +Lives in apps.base because BOTH the share surface (base.views) and the +admin surface (admin.views) consume it; keeping it here avoids the +base → admin reverse dependency. Admin-only session gating stays in +apps.admin.dependencies, which re-exports these primitives for +compatibility. +""" +import base64 +import hmac +import json +import time + +from fastapi import Header, HTTPException + +from core.settings import ( + ADMIN_SESSION_EXPIRE_DEFAULT, + ADMIN_SESSION_EXPIRE_MAX, + ADMIN_SESSION_EXPIRE_MIN, + settings, +) + + +def _get_jwt_secret() -> bytes: + secret = getattr(settings, "jwt_secret", "") + if not secret: + raise RuntimeError("JWT签名密钥未初始化") + return secret.encode() + + +def get_admin_session_expire_seconds() -> int: + try: + expires_in = int( + getattr(settings, "admin_session_expire", ADMIN_SESSION_EXPIRE_DEFAULT) + ) + except (TypeError, ValueError): + return ADMIN_SESSION_EXPIRE_DEFAULT + if ( + not ADMIN_SESSION_EXPIRE_MIN <= expires_in <= ADMIN_SESSION_EXPIRE_MAX + or expires_in % ADMIN_SESSION_EXPIRE_MIN != 0 + ): + return ADMIN_SESSION_EXPIRE_DEFAULT + return expires_in + + +def create_token(data: dict, expires_in: int | None = None) -> str: + """ + 创建JWT token + :param data: 数据负载 + :param expires_in: 过期时间(秒) + """ + token_lifetime = ( + get_admin_session_expire_seconds() if expires_in is None else expires_in + ) + header = base64.urlsafe_b64encode( + json.dumps({"alg": "HS256", "typ": "JWT"}, separators=(",", ":")).encode() + ).decode().rstrip("=") + payload = base64.urlsafe_b64encode( + json.dumps( + {**data, "exp": int(time.time()) + token_lifetime}, + separators=(",", ":"), + ).encode() + ).decode().rstrip("=") + + signature_digest = hmac.new( + _get_jwt_secret(), f"{header}.{payload}".encode(), "sha256" + ).digest() + signature = base64.urlsafe_b64encode(signature_digest).decode().rstrip("=") + + return f"{header}.{payload}.{signature}" + + +def verify_token(token: str) -> dict: + """ + 验证JWT token + :param token: JWT token + :return: 解码后的数据 + """ + try: + header_b64, payload_b64, signature_b64 = token.split(".") + + # 验证签名 + expected_signature = hmac.new( + _get_jwt_secret(), + f"{header_b64}.{payload_b64}".encode(), + "sha256", + ).digest() + expected_signature_b64 = ( + base64.urlsafe_b64encode(expected_signature).decode().rstrip("=") + ) + + if not hmac.compare_digest(signature_b64, expected_signature_b64): + raise ValueError("无效的签名") + + # 解码payload（兼容历史标准 base64 与 urlsafe base64） + padded = payload_b64 + "=" * (-len(payload_b64) % 4) + try: + payload_bytes = base64.urlsafe_b64decode(padded) + except Exception: + payload_bytes = base64.b64decode(padded) + payload = json.loads(payload_bytes) + + # 检查是否过期 + if payload.get("exp", 0) < time.time(): + raise ValueError("token已过期") + + return payload + except Exception as e: + raise ValueError(f"token验证失败: {str(e)}") + + +def _extract_bearer_token(authorization: str) -> str: + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException(status_code=401, detail="未授权或授权校验失败") + token = authorization.split(" ", 1)[1].strip() + if not token: + raise HTTPException(status_code=401, detail="未授权或授权校验失败") + return token + + +def _require_admin_payload(authorization: str) -> dict: + token = _extract_bearer_token(authorization) + try: + payload = verify_token(token) + except ValueError as e: + raise HTTPException(status_code=401, detail=str(e)) + if not payload.get("is_admin", False): + raise HTTPException(status_code=401, detail="未授权或授权校验失败") + return payload + + +async def share_required_login(authorization: str = Header(default=None)): + """ + 验证分享上传权限 + + 当 settings.open_upload 为False时，要求用户必须登录并具有管理员权限 + 当 settings.open_upload 为True时，允许游客上传 + + :param authorization: 认证头信息 + :param request: 请求对象 + :return: 验证结果 + """ + if not settings.open_upload: + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException( + status_code=403, detail="本站未开启游客上传，如需上传请先登录后台" + ) + _require_admin_payload(authorization) + + return True diff --git a/apps/base/dependencies.py b/apps/base/dependencies.py index 0be60a439..c7748547c 100644 --- a/apps/base/dependencies.py +++ b/apps/base/dependencies.py @@ -1,5 +1,5 @@ from ipaddress import ip_address, ip_network -from typing import Dict, Iterable, Union +from typing import Dict, Iterable, TypedDict from datetime import datetime, timedelta from fastapi import HTTPException, Request @@ -65,9 +65,14 @@ def get_client_ip(request: Request) -> str: return client_host +class _IPRecord(TypedDict): + count: int + time: datetime + + class IPRateLimit: def __init__(self, count: int, minutes: int): - self.ips: Dict[str, Dict[str, Union[int, datetime]]] = {} + self.ips: Dict[str, _IPRecord] = {} self.count = count self.minutes = minutes @@ -81,7 +86,9 @@ def check_ip(self, ip: str) -> bool: return True def add_ip(self, ip: str) -> int: - ip_info = self.ips.get(ip, {"count": 0, "time": datetime.now()}) + ip_info = self.ips.setdefault( + ip, {"count": 0, "time": datetime.now()} + ) ip_info["count"] += 1 ip_info["time"] = datetime.now() self.ips[ip] = ip_info diff --git a/apps/base/models.py b/apps/base/models.py index 3820dee1b..d9d58e112 100644 --- a/apps/base/models.py +++ b/apps/base/models.py @@ -2,13 +2,13 @@ # @Author : Lan # @File : models.py # @Software: PyCharm -from typing import Optional + +from typing import Any from tortoise.models import Model from tortoise.contrib.pydantic import pydantic_model_creator from tortoise import fields, models -from datetime import datetime from core.utils import get_now @@ -57,14 +57,13 @@ class UploadChunk(models.Model): class KeyValue(Model): - id: Optional[int] = fields.IntField(pk=True) - key: Optional[str] = fields.CharField( - max_length=255, description="键", index=True, unique=True - ) - value: Optional[str] = fields.JSONField(description="值", null=True) - created_at: Optional[datetime] = fields.DatetimeField( - auto_now_add=True, description="创建时间" - ) + # 与 FileCodes 保持同一风格：类属性不写值注解（等号右侧的 Field 描述符 + # 才是类级真身；实例级由 tortoise 运行时给出对应 Python 值） + id = fields.IntField(pk=True) + key = fields.CharField(max_length=255, description="键", index=True, unique=True) + # JSONField 实际承载 dict/list（如 settings 配置） + value: Any = fields.JSONField(description="值", null=True) + created_at = fields.DatetimeField(auto_now_add=True, description="创建时间") class PresignUploadSession(models.Model): diff --git a/apps/base/pages.py b/apps/base/pages.py index 5bd174324..09179b414 100644 --- a/apps/base/pages.py +++ b/apps/base/pages.py @@ -3,7 +3,7 @@ import html from fastapi import APIRouter, HTTPException, Request -from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse +from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse from apps.base.config import initialize_system, is_runtime_initialized from apps.base.setup_wizard import ( @@ -88,6 +88,24 @@ async def theme_asset(asset_path: str): return FileResponse(resolve_theme_file("assets", asset_path)) +async def not_found_handler(request, exc=None): + """区分"浏览器导航到未知路径"与"API 处理器抛出的 404"。 + + Starlette 的 status-code handler 优先于 HTTPException 类 handler， + 直接注册 index 会让全应用所有 HTTPException(404) 变成 200 HTML 页 + （负路径测试抓到的存量 bug）。浏览器（Accept 含 text/html）仍拿到 + 主题首页做 SPA 兜底；API 客户端拿到 JSON 404。 + """ + if request is not None and request.method in {"GET", "HEAD"} and "text/html" in ( + request.headers.get("accept", "") + ): + return await index(request, exc) + return JSONResponse( + status_code=404, + content={"code": 404, "message": "Not Found", "detail": "资源不存在"}, + ) + + @router.get("/") async def index(request=None, exc=None): # Site config is admin input (and during the setup window anyone can claim it); diff --git a/apps/base/services.py b/apps/base/services.py index 39115e3dd..29026d169 100644 --- a/apps/base/services.py +++ b/apps/base/services.py @@ -76,9 +76,12 @@ async def validate_file_size(file: UploadFile, max_size: int) -> int: """Return the upload's size, rejecting anything above max_size.""" size = file.size if size is None: - await file.seek(0, 2) # type: ignore[arg-type] + # 无长度声明的上传（如 chunked 传输）：必须走底层文件对象拿末尾偏移。 + # 注意两层都不能用：UploadFile.seek 只接受单参数（TypeError）； + # 底层 SpooledTemporaryFile.seek 是同步方法（不能 await）。 + file.file.seek(0, 2) size = file.file.tell() - await file.seek(0) + file.file.seek(0) if size > max_size: max_size_mb = max_size / (1024 * 1024) raise HTTPException( @@ -194,7 +197,9 @@ async def create_file_share( raise finally: await release_storage(token) - return {"code": code, "name": file.filename} + # UploadFile.filename 类型为 str | None；路径生成已用 "" 兜底， + # 响应处保持 str 以匹配 dict[str, str] 契约 + return {"code": code, "name": file.filename or ""} @staticmethod async def complete_chunked_upload( @@ -399,6 +404,9 @@ def response_from_download(download: StoredDownload): return Response( download.content, media_type=download.media_type, headers=download.headers ) + if download.stream_factory is None: + # 数据契约要求三者至少其一；防御性 500 语义 + raise ValueError("StoredDownload 缺少可用的下载载荷") return StreamingResponse( download.stream_factory(), media_type=download.media_type, diff --git a/apps/base/setup_wizard.py b/apps/base/setup_wizard.py index 0a9bbb0b9..a8e454969 100644 --- a/apps/base/setup_wizard.py +++ b/apps/base/setup_wizard.py @@ -5,6 +5,8 @@ import html from urllib.parse import parse_qs +from typing import Any + from fastapi import Request from fastapi.responses import HTMLResponse @@ -73,14 +75,14 @@ def build_public_meta() -> dict: -def get_form_value(data: dict, key: str, default: str = "") -> str: +def get_form_value(data: dict[str, Any], key: str, default: str = "") -> str: value = data.get(key, default) if isinstance(value, list): value = value[-1] if value else default return str(value if value is not None else default) -def get_form_list(data: dict, key: str) -> list[str]: +def get_form_list(data: dict[str, Any], key: str) -> list[str]: value = data.get(key, []) if isinstance(value, list): return [str(item) for item in value if str(item)] @@ -96,7 +98,7 @@ def normalize_bool_field(data: dict, key: str, default: bool) -> bool: def parse_int_field( - data: dict, + data: dict[str, Any], key: str, default: int, label: str, diff --git a/apps/base/views.py b/apps/base/views.py index 85746bf4a..f52784d99 100644 --- a/apps/base/views.py +++ b/apps/base/views.py @@ -1,4 +1,5 @@ import hashlib +import hmac import os import uuid from datetime import timedelta @@ -12,7 +13,7 @@ from starlette.responses import Response from tortoise.expressions import Case, F, Q, When -from apps.admin.dependencies import share_required_login +from apps.base.auth import share_required_login from apps.base.models import FileCodes, UploadChunk, PresignUploadSession from apps.base.quota import release_storage, reserve_storage from apps.base.services import ( @@ -243,12 +244,16 @@ async def select_file(data: SelectFileModel, ip: str = Depends(ip_limit["error"] async def download_file(key: str, code: str, ip: str = Depends(ip_limit["error"])): file_storage: FileStorageInterface = storages[settings.file_storage]() normalized_code = normalize_share_code(code) - # 同时接受当前窗口与上一窗口 token，避免时间窗边界竞态导致偶发 403 - valid_keys = { + # 同时接受当前窗口与上一窗口 token，避免时间窗边界竞态导致偶发 403。 + # 逐个常量时间比较（hmac.compare_digest），避免 set 成员判断的时序侧信道。 + valid_keys = [ await get_select_token(normalized_code, offset=0), await get_select_token(normalized_code, offset=1), - } - if key not in valid_keys: + ] + if not any( + hmac.compare_digest(key.encode(), candidate.encode()) + for candidate in valid_keys + ): ip_limit["error"].add_ip(ip) raise HTTPException(status_code=403, detail="下载鉴权失败") has, file_code = await get_code_file_by_code(normalized_code) diff --git a/core/database.py b/core/database.py index 1b041193a..ff86f8d34 100644 --- a/core/database.py +++ b/core/database.py @@ -47,7 +47,7 @@ def _lock_file(file_obj: IO[str]) -> None: if os.fstat(file_obj.fileno()).st_size == 0: file_obj.write("0") file_obj.flush() - msvcrt.locking(file_obj.fileno(), msvcrt.LK_LOCK, 1) + msvcrt.locking(file_obj.fileno(), msvcrt.LK_LOCK, 1) # type: ignore[attr-defined] else: import fcntl @@ -58,7 +58,7 @@ def _unlock_file(file_obj: IO[str]) -> None: if os.name == "nt": import msvcrt - msvcrt.locking(file_obj.fileno(), msvcrt.LK_UNLCK, 1) + msvcrt.locking(file_obj.fileno(), msvcrt.LK_UNLCK, 1) # type: ignore[attr-defined] else: import fcntl diff --git a/core/security.py b/core/security.py index 1e8055632..16b134eaf 100644 --- a/core/security.py +++ b/core/security.py @@ -1,7 +1,11 @@ import copy +import ipaddress +import os import secrets +import socket from dataclasses import dataclass from typing import Any +from urllib.parse import urlsplit from core.utils import hash_password, is_password_hashed, verify_password @@ -67,3 +71,102 @@ def prepare_security_config(config: dict[str, Any]) -> SecurityConfigResult: result.changed = True return result + + +# 出站端点白名单/黑名单（SSRF 防护）。 +# 这些配置项会被服务器直接当作上游地址访问：被劫持的管理员会话可以把实例 +# 变成内网代理跳板，或把数据发往任意地址。校验只在写入入口（update_config）， +# 已存值不受影响；APP_ENV 非 production（本地开发，如 minio/webdav）时放行。 +OUTBOUND_ENDPOINT_CONFIG_KEYS = ("s3_endpoint_url", "s3_hostname", "webdav_url") + +_ENDPOINT_SCHEME_WHITELIST = {"http", "https"} +_ENDPOINT_HOST_DENY_SUFFIXES = ( + ".local", + ".internal", + ".lan", + ".home.arpa", + ".corp", + ".localhost", +) + + +def _endpoint_ip_is_denied(ip_str: str) -> bool: + try: + ip = ipaddress.ip_address(ip_str) + except ValueError: + return False + return ( + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_reserved + or ip.is_multicast + or ip.is_unspecified + ) + + +def _endpoint_host_is_denied(host: str) -> bool: + host = host.strip("[]").lower() + if host in {"localhost"} or host.endswith(_ENDPOINT_HOST_DENY_SUFFIXES): + return True + # 字面 IP 直接判；非字面形态（速记 "127.1"、十进制 2130706433、十六进制、 + # *.localhost、nip.io 类 DNS 映射）交给解析后逐 IP 复判——静态字符串 + # 黑名单对它们全部失效，这是探测中实测确认的绕过面。 + if _endpoint_ip_is_denied(host): + return True + try: + infos = socket.getaddrinfo(host, None) + except (socket.gaierror, UnicodeError, OSError): + return False + for info in infos: + # getaddrinfo 的 sockaddr 首元素在 IPv4/IPv6 下均为 str（int 仅出现在 + # AF_UNIX 等异形族）——显式 str() 让类型检查器与运行时一致。 + if _endpoint_ip_is_denied(str(info[4][0])): + return True + return False + + +def validate_outbound_endpoint(value: Any) -> str: + """校验完整 URL 形态的出站端点（s3_endpoint_url / webdav_url）。""" + endpoint = str(value or "").strip() + if not endpoint: + return "" + + if os.environ.get("APP_ENV", "development") != "production": + return endpoint + + parts = urlsplit(endpoint) + if parts.scheme.lower() not in _ENDPOINT_SCHEME_WHITELIST: + raise ValueError(f"端点协议必须是 http(s)：{endpoint}") + host = parts.hostname or "" + if not host: + raise ValueError(f"端点缺少主机名：{endpoint}") + if _endpoint_host_is_denied(host): + raise ValueError(f"端点不允许指向内网/保留地址：{endpoint}") + return endpoint + + +def validate_outbound_hostname(value: Any) -> str: + """校验裸主机名形态的出站端点（s3_hostname，存储层拼接 https://{hostname}）。""" + hostname = str(value or "").strip() + if not hostname: + return "" + + if os.environ.get("APP_ENV", "development") != "production": + return hostname + + if "://" in hostname or "/" in hostname: + raise ValueError(f"s3_hostname 应为裸主机名，不含协议或路径：{hostname}") + # 取主机部分：[v6] 形态取括号内；host:port 取冒号前；裸 IPv6（多个冒号）整体 + if "[" in hostname: + host = hostname.split("[", 1)[1].split("]", 1)[0] + rest = hostname.split("]", 1)[1] if "]" in hostname else "" + if rest and not rest.startswith(":"): + raise ValueError(f"s3_hostname 的 IPv6 括号后只允许端口：{hostname}") + elif hostname.count(":") == 1: + host = hostname.split(":")[0] + else: + host = hostname + if not host or _endpoint_host_is_denied(host): + raise ValueError(f"端点不允许指向内网/保留地址：{hostname}") + return hostname diff --git a/core/settings.py b/core/settings.py index 19f58b13e..f66905ede 100644 --- a/core/settings.py +++ b/core/settings.py @@ -1,3 +1,4 @@ +from typing import Any # @Time : 2023/8/15 09:51 # @Author : Lan # @File : settings.py @@ -14,7 +15,7 @@ if not data_root.exists(): data_root.mkdir(parents=True, exist_ok=True) -DEFAULT_CONFIG = { +DEFAULT_CONFIG: dict[str, Any] = { "file_storage": "local", "storage_path": "", "storage_limit": 0, diff --git a/core/storage.py b/core/storage.py deleted file mode 100644 index c5ab29066..000000000 --- a/core/storage.py +++ /dev/null @@ -1,1428 +0,0 @@ -# @Time : 2023/8/11 20:06 -# @Author : Lan -# @File : storage.py -# @Software: PyCharm -import base64 -import hashlib -import os -import tempfile -from core.logger import logger -import shutil -from typing import BinaryIO, Optional -from urllib.parse import quote, unquote - -import aiofiles -import aiohttp -import asyncio -from pathlib import Path -import datetime -from dataclasses import dataclass -import re -import aioboto3 -from botocore.config import Config -from core.errors import StorageError -from core.settings import data_root, settings -from core.utils import get_file_url, sanitize_filename -from starlette.background import BackgroundTask - - -@dataclass -class StoredDownload: - """Framework-free description of a file download. - - Backends return this; the view layer builds the starlette Response: - - ``path`` set -> FileResponse (local files, Range support for free) - - ``content`` set -> small full-read Response (legacy OpenDAL fallback) - - ``stream_factory`` set -> StreamingResponse(stream_factory(), ...) - ``background`` is an optional response-sent cleanup hook. - """ - - filename: str - headers: dict - media_type: str = "application/octet-stream" - path: object = None - content: object = None - stream_factory: object = None - background: object = None - - -@dataclass -class StoredFile: - """Plain, framework- and ORM-free description of a stored file. - - Storage backends accept this instead of ORM models so core/ never imports - apps/. Callers (views/tasks/services) build it from their own records. - """ - - file_path: str - uuid_file_name: str - code: str = "" - prefix: str = "" - suffix: str = "" - text: str = "" - - def get_file_path(self) -> str: - return f"{self.file_path}/{self.uuid_file_name}" - - -class FileStorageInterface: - - @staticmethod - def _get_chunk_record(chunk_records: dict, index: int): - """Look up the caller-provided record for chunk `index`. - - Records are plain objects exposing ``chunk_hash``; fetching them from - the DB is the caller's job (keeps storage ORM-free). - """ - chunk_record = chunk_records.get(index) - if not chunk_record: - raise ValueError(f"分片{index}记录不存在") - return chunk_record - - def _verify_and_hash_chunk( - self, - index: int, - chunk_record, - chunk_data: bytes, - file_sha256, - ) -> None: - """Verify a chunk against its recorded hash, then stripe it into the - whole-file digest. Raises the shared ValueError wording on mismatch.""" - current_hash = hashlib.sha256(chunk_data).hexdigest() - if current_hash != chunk_record.chunk_hash: - raise ValueError( - f"分片{index}哈希不匹配: 期望 {chunk_record.chunk_hash}, 实际 {current_hash}" - ) - file_sha256.update(chunk_data) - - async def save_file( - self, stream: BinaryIO, save_path: str, content_type: Optional[str] = None - ): - """Save a binary stream (caller owns closing the stream).""" - raise NotImplementedError - - async def delete_file(self, file_code: StoredFile): - """ - 删除文件 - """ - raise NotImplementedError - - async def get_file_url(self, file_code: StoredFile): - """ - 获取文件分享的url - - 如果服务不支持直接访问文件，可以通过服务器中转下载。 - 此时，此方法可以调用 utils.py 中的 `get_file_url` 方法，获取服务器中转下载的url - """ - raise NotImplementedError - - async def get_file_response(self, file_code: StoredFile): - """ - 获取文件响应 - - 如果服务不支持直接访问文件，则需要实现该方法，返回文件响应 - 其余情况，可以不实现该方法 - """ - raise NotImplementedError - - async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str): - """ - 保存分片文件 - :param upload_id: 上传会话ID - :param chunk_index: 分片索引 - :param chunk_data: 分片数据 - :param chunk_hash: 分片哈希值 - :param save_path: 文件保存路径 - """ - raise NotImplementedError - - async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]: - """ - 合并分片文件并返回文件路径和完整哈希值 - :param upload_id: 上传会话ID - :param chunk_info: 分片信息 - :param save_path: 文件保存路径 - :return: (文件路径, 文件哈希值) - """ - raise NotImplementedError - - async def generate_presigned_upload_url(self, save_path: str, expires_in: int = 900) -> Optional[str]: - """ - 生成预签名上传URL - :param save_path: 文件保存路径 - :param expires_in: URL过期时间（秒），默认15分钟 - :return: 预签名URL，如果不支持直传则返回None - """ - return None # 默认不支持直传，使用代理模式 - - async def file_exists(self, save_path: str) -> bool: - """ - 检查文件是否存在 - :param save_path: 文件路径 - :return: 文件是否存在 - """ - raise NotImplementedError - - async def clean_chunks(self, upload_id: str, save_path: str): - """ - 清理临时分片文件 - :param upload_id: 上传会话ID - :param save_path: 文件保存路径 - """ - raise NotImplementedError - - -class SystemFileStorage(FileStorageInterface): - def __init__(self): - self.chunk_size = 256 * 1024 - self.root_path = data_root - - def _resolve_safe_path(self, relative_path: str) -> Path: - """将相对路径解析到数据根目录内，阻止路径穿越。""" - root = self.root_path.resolve() - raw = str(relative_path or "").replace("\\", "/").lstrip("/") - if any(part == ".." for part in raw.split("/")): - raise ValueError("非法文件路径") - candidate = (root / raw).resolve() - try: - candidate.relative_to(root) - except ValueError as exc: - raise ValueError("非法文件路径") from exc - return candidate - - def _save(self, file, save_path): - with open(save_path, "wb") as f: - chunk = file.read(self.chunk_size) - while chunk: - f.write(chunk) - chunk = file.read(self.chunk_size) - - async def save_file( - self, stream: BinaryIO, save_path: str, content_type: Optional[str] = None - ): - path_obj = Path(str(save_path).replace("\\", "/")) - directory = str(path_obj.parent).replace("\\", "/").lstrip("/") - # 提取原始文件名并进行清理 - filename = await sanitize_filename(path_obj.name) - # 构建安全的完整保存路径 - safe_save_path = self._resolve_safe_path(f"{directory}/{filename}" if directory not in {"", "."} else filename) - # 确保目录存在 - if not safe_save_path.parent.exists(): - safe_save_path.parent.mkdir(parents=True) - await asyncio.to_thread(self._save, stream, safe_save_path) - - async def delete_file(self, file_code: StoredFile): - save_path = self._resolve_safe_path(file_code.get_file_path()) - if save_path.exists(): - save_path.unlink() - - async def get_file_url(self, file_code: StoredFile): - return await get_file_url(file_code.code) - - async def get_file_response(self, file_code: StoredFile): - file_path = self._resolve_safe_path(file_code.get_file_path()) - if not file_path.exists(): - raise StorageError(status_code=404, detail="文件已过期删除") - filename = f"{file_code.prefix}{file_code.suffix}" - encoded_filename = quote(filename, safe='') - content_disposition = f"attachment; filename*=UTF-8''{encoded_filename}" - - # 尝试获取文件系统大小，如果成功则设置 Content-Length - headers = {"Content-Disposition": content_disposition} - try: - content_length = file_path.stat().st_size - headers["Content-Length"] = str(content_length) - except Exception: - # 如果获取文件大小失败，则不提供 Content-Length - pass - - return StoredDownload( - filename=filename, - headers=headers, - path=file_path, - ) - - async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str): - """ - 保存分片文件到本地文件系统 - :param upload_id: 上传会话ID - :param chunk_index: 分片索引 - :param chunk_data: 分片数据 - :param chunk_hash: 分片哈希值 - :param save_path: 文件保存路径 - """ - # 先校验目标文件路径合法，再将分片落到同级 chunks 目录。 - self._resolve_safe_path(save_path) - chunk_path = self._resolve_safe_path( - str(Path(save_path).parent / "chunks" / upload_id / f"{chunk_index}.part") - ) - if not chunk_path.parent.exists(): - chunk_path.parent.mkdir(parents=True, exist_ok=True) - # 使用临时文件写入，确保原子性 - temp_path = chunk_path.with_suffix('.tmp') - try: - async with aiofiles.open(temp_path, "wb") as f: - await f.write(chunk_data) - # 原子重命名 - temp_path.rename(chunk_path) - except Exception as e: - if temp_path.exists(): - temp_path.unlink() - raise e - - async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]: - """ - 合并本地文件系统的分片文件并返回文件路径和完整哈希值 - :param upload_id: 上传会话ID - :param chunk_info: 分片信息 - :param save_path: 文件保存路径 - :return: (文件路径, 文件哈希值) - """ - output_path = self._resolve_safe_path(save_path) - output_path.parent.mkdir(parents=True, exist_ok=True) - chunk_base_dir = self._resolve_safe_path( - str(Path(save_path).parent / "chunks" / upload_id) - ) - file_sha256 = hashlib.sha256() - - # 使用临时文件写入，确保原子性 - temp_output = output_path.with_suffix('.merging') - try: - async with aiofiles.open(temp_output, "wb") as out_file: - for i in range(total_chunks): - # 获取分片记录 - chunk_record = self._get_chunk_record(chunk_records, i) - chunk_path = chunk_base_dir / f"{i}.part" - if not chunk_path.exists(): - raise ValueError(f"分片{i}文件不存在") - async with aiofiles.open(chunk_path, "rb") as in_file: - chunk_data = await in_file.read() - self._verify_and_hash_chunk(i, chunk_record, chunk_data, file_sha256) - await out_file.write(chunk_data) - # 原子重命名 - temp_output.rename(output_path) - except Exception as e: - if temp_output.exists(): - temp_output.unlink() - raise e - return str(output_path), file_sha256.hexdigest() - - async def clean_chunks(self, upload_id: str, save_path: str): - """ - 清理本地文件系统的临时分片文件 - :param upload_id: 上传会话ID - :param save_path: 文件保存路径 - """ - chunk_dir = self._resolve_safe_path( - str(Path(save_path).parent / "chunks" / upload_id) - ) - if chunk_dir.exists(): - try: - shutil.rmtree(chunk_dir) - except Exception as e: - logger.warning(f"清理本地分片目录失败: {e}") - # 清理父级 chunks 目录（如果为空） - chunks_parent = chunk_dir.parent - if chunks_parent.exists() and not any(chunks_parent.iterdir()): - try: - chunks_parent.rmdir() - except Exception as e: - logger.warning(f"清理 chunks 父目录失败: {e}") - - async def file_exists(self, save_path: str) -> bool: - """ - 检查文件是否存在于本地文件系统 - :param save_path: 文件路径 - :return: 文件是否存在 - """ - try: - file_path = self._resolve_safe_path(save_path) - except ValueError: - return False - return file_path.exists() - - -class S3FileStorage(FileStorageInterface): - def __init__(self): - self.access_key_id = settings.s3_access_key_id - self.secret_access_key = settings.s3_secret_access_key - self.bucket_name = settings.s3_bucket_name - self.s3_hostname = settings.s3_hostname - self.region_name = settings.s3_region_name - self.signature_version = settings.s3_signature_version - self.endpoint_url = settings.s3_endpoint_url or f"https://{self.s3_hostname}" - self.aws_session_token = settings.aws_session_token - self.addressing_style = str(settings.s3_addressing_style or "auto").lower() - self.proxy = settings.s3_proxy - self.session = aioboto3.Session( - aws_access_key_id=self.access_key_id, - aws_secret_access_key=self.secret_access_key, - ) - if not settings.s3_endpoint_url: - self.endpoint_url = f"https://{self.s3_hostname}" - else: - # 如果提供了 s3_endpoint_url，则优先使用它 - self.endpoint_url = settings.s3_endpoint_url - - def _client_config(self) -> Config: - config = {"signature_version": self.signature_version} - s3_config = {} - if self.addressing_style in {"path", "virtual", "auto"}: - s3_config["addressing_style"] = self.addressing_style - if s3_config: - config["s3"] = s3_config - return Config(**config) - - def _client(self): - return self.session.client( - "s3", - endpoint_url=self.endpoint_url, - aws_session_token=self.aws_session_token, - region_name=self.region_name, - config=self._client_config(), - ) - - async def save_file( - self, stream: BinaryIO, save_path: str, content_type: Optional[str] = None - ): - async with self._client() as s3: - # 使用 upload_fileobj 流式上传，避免将整个文件加载到内存 - await s3.upload_fileobj( - stream, - self.bucket_name, - save_path, - ExtraArgs={"ContentType": content_type or "application/octet-stream"}, - ) - - async def delete_file(self, file_code: StoredFile): - async with self._client() as s3: - await s3.delete_object( - Bucket=self.bucket_name, Key=file_code.get_file_path() - ) - - async def get_file_response(self, file_code: StoredFile): - try: - filename = file_code.prefix + file_code.suffix - content_length = None # 初始化为 None，表示未知大小 - - async with self._client() as s3: - # 尝试获取文件大小（HEAD请求） - try: - head_response = await s3.head_object( - Bucket=self.bucket_name, - Key=file_code.get_file_path() - ) - # 从HEAD响应中获取Content-Length - if 'ContentLength' in head_response: - content_length = head_response['ContentLength'] - elif 'Content-Length' in head_response['ResponseMetadata']['HTTPHeaders']: - content_length = int(head_response['ResponseMetadata']['HTTPHeaders']['Content-Length']) - except Exception: - # 如果HEAD请求失败，则不提供 Content-Length - pass - - link = await s3.generate_presigned_url( - "get_object", - Params={ - "Bucket": self.bucket_name, - "Key": file_code.get_file_path(), - }, - ExpiresIn=3600, - ) - - # 创建ClientSession并传递给生成器复用 - session = aiohttp.ClientSession() - - async def stream_generator(): - try: - async with session.get(link) as resp: - if resp.status != 200: - raise StorageError( - status_code=resp.status, - detail=f"从S3获取文件失败: {resp.status}" - ) - # 设置块大小（例如64KB） - chunk_size = 65536 - while True: - chunk = await resp.content.read(chunk_size) - if not chunk: - break - yield chunk - finally: - await session.close() - - encoded_filename = quote(filename, safe='') - headers = { - "Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}" - } - if content_length is not None: - headers["Content-Length"] = str(content_length) - return StoredDownload( - filename=filename, - headers=headers, - stream_factory=stream_generator, - # 兜底关闭会话：客户端中断时与 generator finally 双保险 - background=BackgroundTask(session.close), - ) - except StorageError: - raise - except Exception: - raise StorageError(status_code=503, detail="服务代理下载异常，请稍后再试") - - async def get_file_url(self, file_code: StoredFile): - if file_code.prefix == "文本分享": - return file_code.text - if self.proxy: - return await get_file_url(file_code.code) - else: - async with self._client() as s3: - result = await s3.generate_presigned_url( - "get_object", - Params={ - "Bucket": self.bucket_name, - "Key": file_code.get_file_path(), - }, - ExpiresIn=3600, - ) - return result - - async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str): - """ - 保存分片到 S3（使用独立对象存储每个分片） - 注意：这里不使用 S3 原生的 multipart upload，而是将每个分片作为独立对象存储 - """ - chunk_key = str(Path(save_path).parent / "chunks" / upload_id / f"{chunk_index}.part") - async with self._client() as s3: - # 将分片作为独立对象上传 - await s3.put_object( - Bucket=self.bucket_name, - Key=chunk_key, - Body=chunk_data, - Metadata={ - 'chunk-hash': chunk_hash, - 'chunk-index': str(chunk_index) - } - ) - - async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]: - """ - 合并 S3 上的分片文件 - 使用 S3 的 multipart upload API 实现流式合并，避免内存问题 - """ - file_sha256 = hashlib.sha256() - chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) - - async with self._client() as s3: - # 创建 multipart upload - mpu = await s3.create_multipart_upload( - Bucket=self.bucket_name, - Key=save_path, - ContentType='application/octet-stream' - ) - mpu_id = mpu['UploadId'] - parts = [] - - try: - # 按顺序读取、验证并上传每个分片 - for i in range(total_chunks): - chunk_key = f"{chunk_dir}/{i}.part" - chunk_record = self._get_chunk_record(chunk_records, i) - - try: - response = await s3.get_object( - Bucket=self.bucket_name, - Key=chunk_key - ) - chunk_data = await response['Body'].read() - except Exception as e: - raise ValueError(f"分片{i}文件不存在: {e}") - - self._verify_and_hash_chunk(i, chunk_record, chunk_data, file_sha256) - - # 上传分片到 multipart upload - part_response = await s3.upload_part( - Bucket=self.bucket_name, - Key=save_path, - UploadId=mpu_id, - PartNumber=i + 1, # S3 part numbers start at 1 - Body=chunk_data - ) - parts.append({ - 'PartNumber': i + 1, - 'ETag': part_response['ETag'] - }) - - # 释放内存 - del chunk_data - - # 完成 multipart upload - await s3.complete_multipart_upload( - Bucket=self.bucket_name, - Key=save_path, - UploadId=mpu_id, - MultipartUpload={'Parts': parts} - ) - except Exception as e: - # 出错时取消 multipart upload - await s3.abort_multipart_upload( - Bucket=self.bucket_name, - Key=save_path, - UploadId=mpu_id - ) - raise e - - return save_path, file_sha256.hexdigest() - - async def clean_chunks(self, upload_id: str, save_path: str): - """ - 清理 S3 上的临时分片文件 - :param upload_id: 上传会话ID - :param save_path: 文件保存路径 - """ - chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) - async with self._client() as s3: - try: - # 列出并删除所有分片对象 - paginator = s3.get_paginator('list_objects_v2') - async for page in paginator.paginate(Bucket=self.bucket_name, Prefix=chunk_dir): - objects = page.get('Contents', []) - if objects: - delete_objects = [{'Key': obj['Key']} for obj in objects] - await s3.delete_objects( - Bucket=self.bucket_name, - Delete={'Objects': delete_objects} - ) - except Exception as e: - logger.warning(f"清理 S3 分片数据时出错: {e}") - - async def generate_presigned_upload_url(self, save_path: str, expires_in: int = 900) -> Optional[str]: - """ - 生成S3预签名上传URL - :param save_path: 文件保存路径 - :param expires_in: URL过期时间（秒），默认15分钟 - :return: 预签名PUT URL - """ - async with self._client() as s3: - return await s3.generate_presigned_url( - "put_object", - Params={ - "Bucket": self.bucket_name, - "Key": save_path, - }, - ExpiresIn=expires_in, - ) - - async def file_exists(self, save_path: str) -> bool: - """ - 检查文件是否存在于S3 - :param save_path: 文件路径 - :return: 文件是否存在 - """ - async with self._client() as s3: - last_error = None - for attempt in range(3): - try: - await s3.head_object(Bucket=self.bucket_name, Key=save_path) - return True - except Exception as e: - last_error = e - if attempt < 2: - await asyncio.sleep(0.2 * (attempt + 1)) - - try: - result = await s3.list_objects_v2( - Bucket=self.bucket_name, - Prefix=save_path, - MaxKeys=1, - ) - for item in result.get("Contents", []): - if item.get("Key") == save_path: - return True - except Exception as e: - last_error = e - - logger.warning(f"S3文件确认失败 key={save_path}: {last_error}") - return False - - -class OneDriveFileStorage(FileStorageInterface): - def __init__(self): - try: - import msal - from office365.graph_client import GraphClient - from office365.runtime.client_request_exception import ( - ClientRequestException, - ) - except ImportError: - raise ImportError("请先安装`msal`和`Office365-REST-Python-Client`") - self.msal = msal - self.domain = settings.onedrive_domain - self.client_id = settings.onedrive_client_id - self.username = settings.onedrive_username - self.password = settings.onedrive_password - self.proxy = settings.onedrive_proxy - self._ClientRequestException = ClientRequestException - - try: - client = GraphClient(self.acquire_token_pwd) - self.root_path = ( - client.me.drive.root.get_by_path(settings.onedrive_root_path) - .get() - .execute_query() - ) - except ClientRequestException as e: - if e.code == "itemNotFound": - client.me.drive.root.create_folder(settings.onedrive_root_path) - self.root_path = ( - client.me.drive.root.get_by_path( - settings.onedrive_root_path) - .get() - .execute_query() - ) - else: - raise e - except Exception as e: - raise Exception("OneDrive验证失败，请检查配置是否正确\n" + str(e)) - - def acquire_token_pwd(self): - authority_url = f"https://login.microsoftonline.com/{self.domain}" - app = self.msal.PublicClientApplication( - authority=authority_url, client_id=self.client_id - ) - result = app.acquire_token_by_username_password( - username=self.username, - password=self.password, - scopes=["https://graph.microsoft.com/.default"], - ) - return result - - def _get_path_str(self, path): - if isinstance(path, str): - path = path.replace("\\", "/").replace("//", "/").split("/") - elif isinstance(path, Path): - path = str(path).replace("\\", "/").replace("//", "/").split("/") - else: - raise TypeError("path must be str or Path") - path[-1] = path[-1].split(".")[0] - return "/".join(path) - - async def save_file( - self, stream: BinaryIO, save_path: str, content_type: Optional[str] = None - ): - """保存文件（自动创建目录；修复旧实现把 save_path 字符串当函数调用的崩溃）""" - content = await asyncio.to_thread(stream.read) - normalized = str(save_path).replace("\\", "/") - name = await sanitize_filename(Path(normalized).name) - dir_path = "/".join(normalized.split("/")[:-1]) - - current_folder = self.root_path - for part in dir_path.split("/"): - if not part: - continue - try: - current_folder = current_folder.get_by_path(part).get().execute_query() - except self._ClientRequestException as e: - if e.code == "itemNotFound": - current_folder = current_folder.create_folder(part).execute_query() - else: - raise e - - await asyncio.to_thread( - lambda: current_folder.get_by_path(name) - .upload(name, content) - .execute_query() - ) - - def _delete(self, save_path): - path = self._get_path_str(save_path) - try: - self.root_path.get_by_path(path).delete_object().execute_query() - except self._ClientRequestException as e: - if e.code == "itemNotFound": - pass - else: - raise e - - async def delete_file(self, file_code: StoredFile): - await asyncio.to_thread(self._delete, file_code.get_file_path()) - - def _convert_link_to_download_link(self, link): - p1 = re.search(r"https://(.+)\.sharepoint\.com", link).group(1) - p2 = re.search(r"personal/(.+)/", link).group(1) - p3 = re.search(rf"{p2}/(.+)", link).group(1) - return f"https://{p1}.sharepoint.com/personal/{p2}/_layouts/52/download.aspx?share={p3}" - - def _get_file_url(self, save_path, name): - path = self._get_path_str(save_path) - remote_file = self.root_path.get_by_path(path + "/" + name) - expiration_datetime = datetime.datetime.now( - tz=datetime.timezone.utc - ) + datetime.timedelta(hours=1) - expiration_datetime = expiration_datetime.strftime( - "%Y-%m-%dT%H:%M:%SZ") - permission = remote_file.create_link( - "view", "anonymous", expiration_datetime=expiration_datetime - ).execute_query() - return self._convert_link_to_download_link(permission.link.webUrl) - - async def get_file_response(self, file_code: StoredFile): - try: - filename = file_code.prefix + file_code.suffix - link = await asyncio.to_thread( - self._get_file_url, file_code.get_file_path(), filename - ) - - content_length = None # 初始化为 None，表示未知大小 - - # 创建ClientSession并复用 - session = aiohttp.ClientSession() - - # 尝试发送HEAD请求获取Content-Length - try: - async with session.head(link) as resp: - if resp.status == 200 and 'Content-Length' in resp.headers: - content_length = int(resp.headers['Content-Length']) - except Exception: - # 如果HEAD请求失败，则不提供 Content-Length - pass - - async def stream_generator(): - try: - async with session.get(link) as resp: - if resp.status != 200: - raise StorageError( - status_code=resp.status, - detail=f"从OneDrive获取文件失败: {resp.status}" - ) - chunk_size = 65536 - while True: - chunk = await resp.content.read(chunk_size) - if not chunk: - break - yield chunk - finally: - await session.close() - - encoded_filename = quote(filename, safe='') - headers = { - "Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}" - } - if content_length is not None: - headers["Content-Length"] = str(content_length) - return StoredDownload( - filename=filename, - headers=headers, - stream_factory=stream_generator, - # 兜底关闭会话：客户端中断时与 generator finally 双保险 - background=BackgroundTask(session.close), - ) - except StorageError: - raise - except Exception: - raise StorageError(status_code=503, detail="服务代理下载异常，请稍后再试") - - async def get_file_url(self, file_code: StoredFile): - if self.proxy: - return await get_file_url(file_code.code) - else: - return await asyncio.to_thread( - self._get_file_url, - file_code.get_file_path(), - f"{file_code.prefix}{file_code.suffix}", - ) - - def _save_chunk(self, chunk_path: str, chunk_data: bytes): - """同步保存分片到 OneDrive""" - path_parts = chunk_path.replace("\\", "/").split("/") - filename = path_parts[-1] - dir_path = "/".join(path_parts[:-1]) - - # 确保目录存在 - current_folder = self.root_path - for part in dir_path.split("/"): - if part: - try: - current_folder = current_folder.get_by_path(part).get().execute_query() - except self._ClientRequestException as e: - if e.code == "itemNotFound": - current_folder = current_folder.create_folder(part).execute_query() - else: - raise e - - # 上传分片 - current_folder.upload(filename, chunk_data).execute_query() - - async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str): - """保存分片到 OneDrive""" - chunk_path = str(Path(save_path).parent / "chunks" / upload_id / f"{chunk_index}.part") - await asyncio.to_thread(self._save_chunk, chunk_path, chunk_data) - - def _read_chunk(self, chunk_path: str) -> bytes: - """同步读取分片""" - path = self._get_path_str(chunk_path) - file_obj = self.root_path.get_by_path(path).get().execute_query() - return file_obj.get_content().execute_query().value - - def _upload_merged(self, save_path: str, data: bytes): - """同步上传合并后的文件""" - path_parts = save_path.replace("\\", "/").split("/") - filename = path_parts[-1] - dir_path = "/".join(path_parts[:-1]) - - # 确保目录存在 - current_folder = self.root_path - for part in dir_path.split("/"): - if part: - try: - current_folder = current_folder.get_by_path(part).get().execute_query() - except self._ClientRequestException as e: - if e.code == "itemNotFound": - current_folder = current_folder.create_folder(part).execute_query() - else: - raise e - - current_folder.upload(filename, data).execute_query() - - async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]: - """合并 OneDrive 上的分片文件，使用临时文件避免内存问题""" - file_sha256 = hashlib.sha256() - chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) - - # 使用临时文件存储合并数据 - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - temp_path = temp_file.name - - try: - async with aiofiles.open(temp_path, 'wb') as out_file: - for i in range(total_chunks): - chunk_path = f"{chunk_dir}/{i}.part" - chunk_record = self._get_chunk_record(chunk_records, i) - - try: - chunk_data = await asyncio.to_thread(self._read_chunk, chunk_path) - except Exception as e: - raise ValueError(f"分片{i}文件不存在: {e}") - - self._verify_and_hash_chunk(i, chunk_record, chunk_data, file_sha256) - await out_file.write(chunk_data) - del chunk_data # 释放内存 - - # 读取临时文件并上传 - async with aiofiles.open(temp_path, 'rb') as f: - merged_content = await f.read() - await asyncio.to_thread(self._upload_merged, save_path, merged_content) - finally: - # 清理临时文件 - if os.path.exists(temp_path): - os.unlink(temp_path) - - return save_path, file_sha256.hexdigest() - - def _delete_chunk_dir(self, chunk_dir: str): - """同步删除分片目录""" - try: - path = self._get_path_str(chunk_dir) - self.root_path.get_by_path(path).delete_object().execute_query() - except self._ClientRequestException as e: - if e.code != "itemNotFound": - raise e - - async def clean_chunks(self, upload_id: str, save_path: str): - """清理 OneDrive 上的临时分片文件""" - chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) - try: - await asyncio.to_thread(self._delete_chunk_dir, chunk_dir) - except Exception as e: - logger.warning(f"清理 OneDrive 分片时出错: {e}") - - def _file_exists(self, save_path: str) -> bool: - """同步检查文件是否存在""" - try: - path = self._get_path_str(save_path) - self.root_path.get_by_path(path).get().execute_query() - return True - except self._ClientRequestException as e: - if e.code == "itemNotFound": - return False - raise e - - async def file_exists(self, save_path: str) -> bool: - """ - 检查文件是否存在于OneDrive - :param save_path: 文件路径 - :return: 文件是否存在 - """ - return await asyncio.to_thread(self._file_exists, save_path) - - -class OpenDALFileStorage(FileStorageInterface): - def __init__(self): - try: - import opendal - except ImportError: - raise ImportError('请先安装 `opendal`, 例如: "pip install opendal"') - self.service = settings.opendal_scheme - service_settings = {} - for key, value in settings.items(): - if key.startswith("opendal_" + self.service): - setting_name = key.split("_", 2)[2] - service_settings[setting_name] = value - self.operator = opendal.AsyncOperator( - settings.opendal_scheme, **service_settings - ) - - async def save_file( - self, stream: BinaryIO, save_path: str, content_type: Optional[str] = None - ): - # 使用 asyncio.to_thread 避免阻塞事件循环 - content = await asyncio.to_thread(stream.read) - await self.operator.write(save_path, content) - - async def delete_file(self, file_code: StoredFile): - await self.operator.delete(file_code.get_file_path()) - - async def get_file_url(self, file_code: StoredFile): - return await get_file_url(file_code.code) - - async def get_file_response(self, file_code: StoredFile): - try: - filename = file_code.prefix + file_code.suffix - content_length = None # 初始化为 None，表示未知大小 - - # 尝试获取文件大小 - try: - stat_result = await self.operator.stat(file_code.get_file_path()) - if hasattr(stat_result, 'content_length') and stat_result.content_length: - content_length = stat_result.content_length - elif hasattr(stat_result, 'size') and stat_result.size: - content_length = stat_result.size - except Exception: - # 如果获取大小失败，则不提供 Content-Length - pass - - # 尝试使用流式读取器 - try: - # OpenDAL 可能提供 reader 方法返回一个异步读取器 - reader = await self.operator.reader(file_code.get_file_path()) - except AttributeError: - # 如果 reader 方法不存在，回退到全量读取（兼容旧版本） - content = await self.operator.read(file_code.get_file_path()) - encoded_filename = quote(filename, safe='') - headers = { - "Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}" - } - if content_length is not None: - headers["Content-Length"] = str(content_length) - return StoredDownload( - filename=filename, - headers=headers, - content=content, - ) - - async def stream_generator(): - chunk_size = 65536 - while True: - chunk = await reader.read(chunk_size) - if not chunk: - break - yield chunk - - encoded_filename = quote(filename, safe='') - headers = { - "Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}" - } - if content_length is not None: - headers["Content-Length"] = str(content_length) - return StoredDownload( - filename=filename, - headers=headers, - stream_factory=stream_generator, - ) - except Exception as e: - logger.info(e) - raise StorageError(status_code=404, detail="文件已过期删除") - - async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str): - """保存分片到 OpenDAL 存储""" - chunk_path = str(Path(save_path).parent / "chunks" / upload_id / f"{chunk_index}.part") - await self.operator.write(chunk_path, chunk_data) - - async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]: - """合并 OpenDAL 存储上的分片文件，使用临时文件避免内存问题""" - file_sha256 = hashlib.sha256() - chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) - - # 使用临时文件存储合并数据 - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - temp_path = temp_file.name - - try: - async with aiofiles.open(temp_path, 'wb') as out_file: - for i in range(total_chunks): - chunk_path = f"{chunk_dir}/{i}.part" - chunk_record = self._get_chunk_record(chunk_records, i) - - try: - chunk_data = await self.operator.read(chunk_path) - except Exception as e: - raise ValueError(f"分片{i}文件不存在: {e}") - - self._verify_and_hash_chunk(i, chunk_record, chunk_data, file_sha256) - await out_file.write(chunk_data) - del chunk_data # 释放内存 - - # 读取临时文件并写入存储 - async with aiofiles.open(temp_path, 'rb') as f: - merged_content = await f.read() - await self.operator.write(save_path, merged_content) - finally: - # 清理临时文件 - if os.path.exists(temp_path): - os.unlink(temp_path) - - return save_path, file_sha256.hexdigest() - - async def clean_chunks(self, upload_id: str, save_path: str): - """清理 OpenDAL 存储上的临时分片文件""" - chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) - try: - # OpenDAL 支持递归删除 - await self.operator.remove_all(chunk_dir) - except Exception as e: - logger.warning(f"清理 OpenDAL 分片时出错: {e}") - - async def file_exists(self, save_path: str) -> bool: - """ - 检查文件是否存在于OpenDAL存储 - :param save_path: 文件路径 - :return: 文件是否存在 - """ - try: - await self.operator.stat(save_path) - return True - except Exception: - return False - - -class WebDAVFileStorage(FileStorageInterface): - _instance: Optional["WebDAVFileStorage"] = None - - def __init__(self): - if not hasattr(self, "_initialized"): - self.base_url = settings.webdav_url.rstrip("/") + "/" - self.auth = aiohttp.BasicAuth( - login=settings.webdav_username, password=settings.webdav_password - ) - self._initialized = True - - def _build_url(self, path: str) -> str: - encoded_path = quote(str(path.replace("\\", "/").lstrip("/")).lstrip("/")) - return f"{self.base_url}{encoded_path}" - - async def _mkdir_p(self, directory_path: str): - """递归创建目录（类似mkdir -p）""" - path_obj = Path(unquote(directory_path)) - current_path = "" - - async with aiohttp.ClientSession(auth=self.auth) as session: - # 逐级检查目录是否存在 - for part in path_obj.parts: - current_path = str(Path(current_path) / part) - url = self._build_url(current_path) - - # 检查目录是否存在 - async with session.head(url) as resp: - if resp.status == 404: - # 创建目录 - async with session.request("MKCOL", url) as mkcol_resp: - if mkcol_resp.status not in (200, 201, 409): - content = await mkcol_resp.text() - raise StorageError( - status_code=mkcol_resp.status, - detail=f"目录创建失败: {content[:200]}", - ) - - async def _is_dir_empty(self, dir_path: str) -> bool: - """检查目录是否为空""" - url = self._build_url(dir_path) - - async with aiohttp.ClientSession(auth=self.auth) as session: - async with session.request("PROPFIND", url, headers={"Depth": "1"}) as resp: - if resp.status != 207: # 207 是 Multi-Status 响应 - return False - content = await resp.text() - # 如果只有一个 response（当前目录），说明目录为空 - return content.count("") <= 1 - - async def _delete_empty_dirs(self, file_path: str, session: aiohttp.ClientSession): - """递归删除空目录""" - path_obj = Path(file_path) - current_path = path_obj.parent - - while str(current_path) != ".": - if not await self._is_dir_empty(str(current_path)): - break - - url = self._build_url(str(current_path)) - async with session.delete(url) as resp: - if resp.status not in (200, 204, 404): - break - - current_path = current_path.parent - - async def save_file( - self, stream: BinaryIO, save_path: str, content_type: Optional[str] = None - ): - """保存文件（自动创建目录，流式上传）""" - path_obj = Path(save_path) - directory_path = str(path_obj.parent) - # 提取原始文件名并进行清理 - filename = await sanitize_filename(path_obj.name) - # 构建安全的保存路径 - safe_save_path = str(Path(directory_path) / filename) - - try: - # 先创建目录结构 - await self._mkdir_p(directory_path) - # 上传文件（流式） - url = self._build_url(safe_save_path) - - async def file_sender(): - """流式读取文件内容""" - chunk_size = 256 * 1024 # 256KB chunks - while True: - chunk = await asyncio.to_thread(stream.read, chunk_size) - if not chunk: - break - yield chunk - - async with aiohttp.ClientSession(auth=self.auth) as session: - async with session.put( - url, - data=file_sender(), - headers={"Content-Type": content_type or "application/octet-stream"} - ) as resp: - if resp.status not in (200, 201, 204): - content = await resp.text() - raise StorageError( - status_code=resp.status, - detail=f"文件上传失败: {content[:200]}", - ) - except aiohttp.ClientError as e: - raise StorageError( - status_code=503, detail=f"WebDAV连接异常: {str(e)}") - - async def delete_file(self, file_code: StoredFile): - """删除WebDAV文件及空目录""" - file_path = file_code.get_file_path() - url = self._build_url(file_path) - try: - async with aiohttp.ClientSession(auth=self.auth) as session: - # 删除文件 - async with session.delete(url) as resp: - if resp.status not in (200, 204, 404): - content = await resp.text() - raise StorageError( - status_code=resp.status, - detail=f"WebDAV删除失败: {content[:200]}", - ) - - # 使用同一个 session 删除空目录 - await self._delete_empty_dirs(file_path, session) - - except aiohttp.ClientError as e: - raise StorageError( - status_code=503, detail=f"WebDAV连接异常: {str(e)}") - - async def get_file_url(self, file_code: StoredFile): - return await get_file_url(file_code.code) - - async def get_file_response(self, file_code: StoredFile): - """获取文件响应（代理模式）""" - try: - filename = file_code.prefix + file_code.suffix - url = self._build_url(file_code.get_file_path()) - content_length = None # 初始化为 None，表示未知大小 - - # 创建ClientSession并复用（包含认证头） - session = aiohttp.ClientSession(headers={ - "Authorization": f"Basic {base64.b64encode(f'{settings.webdav_username}:{settings.webdav_password}'.encode()).decode()}" - }) - - # 尝试发送HEAD请求获取Content-Length - try: - async with session.head(url) as resp: - if resp.status == 200 and 'Content-Length' in resp.headers: - content_length = int(resp.headers['Content-Length']) - except Exception: - # 如果HEAD请求失败，则不提供 Content-Length - pass - - async def stream_generator(): - try: - async with session.get(url) as resp: - if resp.status != 200: - raise StorageError( - status_code=resp.status, - detail=f"文件获取失败{resp.status}: {await resp.text()}", - ) - chunk_size = 65536 - while True: - chunk = await resp.content.read(chunk_size) - if not chunk: - break - yield chunk - finally: - await session.close() - - encoded_filename = quote(filename, safe='') - headers = { - "Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}" - } - if content_length is not None: - headers["Content-Length"] = str(content_length) - return StoredDownload( - filename=filename, - headers=headers, - stream_factory=stream_generator, - # 兜底关闭会话：客户端中断时与 generator finally 双保险 - background=BackgroundTask(session.close), - ) - except aiohttp.ClientError as e: - raise StorageError( - status_code=503, detail=f"WebDAV连接异常: {str(e)}") - - async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str): - """保存分片到 WebDAV""" - chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) - chunk_path = f"{chunk_dir}/{chunk_index}.part" - - # 先创建目录结构 - await self._mkdir_p(chunk_dir) - - chunk_url = self._build_url(chunk_path) - async with aiohttp.ClientSession(auth=self.auth) as session: - async with session.put(chunk_url, data=chunk_data) as resp: - if resp.status not in (200, 201, 204): - content = await resp.text() - raise StorageError( - status_code=resp.status, - detail=f"分片上传失败: {content[:200]}" - ) - - async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]: - """ - 合并 WebDAV 上的分片文件 - 使用临时文件避免内存问题 - """ - file_sha256 = hashlib.sha256() - chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) - - # 使用临时文件存储合并数据，避免内存问题 - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - temp_path = temp_file.name - - try: - async with aiohttp.ClientSession(auth=self.auth) as session: - # 按顺序读取并验证每个分片，写入临时文件 - async with aiofiles.open(temp_path, 'wb') as out_file: - for i in range(total_chunks): - chunk_path = f"{chunk_dir}/{i}.part" - chunk_url = self._build_url(chunk_path) - - # 获取分片记录 - chunk_record = self._get_chunk_record(chunk_records, i) - - # 下载分片数据 - async with session.get(chunk_url) as resp: - if resp.status != 200: - raise ValueError(f"分片{i}文件不存在或无法访问") - chunk_data = await resp.read() - - # 验证哈希 - self._verify_and_hash_chunk(i, chunk_record, chunk_data, file_sha256) - await out_file.write(chunk_data) - del chunk_data # 释放内存 - - # 确保目标目录存在 - output_dir = str(Path(save_path).parent) - await self._mkdir_p(output_dir) - - # 流式上传合并后的文件 - output_url = self._build_url(save_path) - - async def file_sender(): - async with aiofiles.open(temp_path, 'rb') as f: - while True: - chunk = await f.read(256 * 1024) - if not chunk: - break - yield chunk - - async with session.put(output_url, data=file_sender()) as resp: - if resp.status not in (200, 201, 204): - content = await resp.text() - raise StorageError( - status_code=resp.status, - detail=f"合并文件上传失败: {content[:200]}" - ) - finally: - # 清理临时文件 - if os.path.exists(temp_path): - os.unlink(temp_path) - - return save_path, file_sha256.hexdigest() - - async def clean_chunks(self, upload_id: str, save_path: str): - """ - 清理 WebDAV 上的临时分片文件 - :param upload_id: 上传会话ID - :param save_path: 文件保存路径 - """ - chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) - chunk_dir_url = self._build_url(chunk_dir) - async with aiohttp.ClientSession(auth=self.auth) as session: - try: - # 检查分片目录是否存在 - async with session.request("PROPFIND", chunk_dir_url, headers={"Depth": "1"}) as resp: - if resp.status == 207: # 207 表示 Multi-Status - # 获取目录下的所有分片文件 - xml_data = await resp.text() - file_paths = re.findall( - r'(.*?)', xml_data) - for file_path in file_paths: - if file_path.endswith(".part"): - # 删除分片文件 - file_url = self._build_url(file_path) - async with session.delete(file_url) as delete_resp: - if delete_resp.status not in (200, 204, 404): - logger.warning(f"删除分片文件失败: {file_path}") - - # 删除分片目录 - async with session.delete(chunk_dir_url) as delete_resp: - if delete_resp.status not in (200, 204, 404): - logger.warning(f"删除分片目录失败: {chunk_dir_url}") - else: - logger.info(f"分片目录不存在: {chunk_dir_url}") - except Exception as e: - logger.warning(f"清理 WebDAV 分片时出错: {e}") - - async def file_exists(self, save_path: str) -> bool: - """ - 检查文件是否存在于WebDAV - :param save_path: 文件路径 - :return: 文件是否存在 - """ - url = self._build_url(save_path) - async with aiohttp.ClientSession(auth=self.auth) as session: - async with session.head(url) as resp: - return resp.status == 200 - - -storages = { - "local": SystemFileStorage, - "s3": S3FileStorage, - "onedrive": OneDriveFileStorage, - "opendal": OpenDALFileStorage, - "webdav": WebDAVFileStorage, -} diff --git a/core/storage/__init__.py b/core/storage/__init__.py new file mode 100644 index 000000000..757e31532 --- /dev/null +++ b/core/storage/__init__.py @@ -0,0 +1,25 @@ +"""Storage backend package (split from the former 1400+ line storage.py). + +Public surface is re-exported here — `from core.storage import X` keeps +working for every historical name. +""" +from core.storage._base import ( # noqa: F401 + FileStorageInterface, + S3_MIN_MULTIPART_PART_SIZE, + StoredDownload, + StoredFile, + build_attachment_headers, +) +from core.storage.local import SystemFileStorage # noqa: F401 +from core.storage.s3 import S3FileStorage # noqa: F401 +from core.storage.onedrive import OneDriveFileStorage # noqa: F401 +from core.storage.opendal import OpenDALFileStorage # noqa: F401 +from core.storage.webdav import WebDAVFileStorage # noqa: F401 + +storages = { + "local": SystemFileStorage, + "s3": S3FileStorage, + "onedrive": OneDriveFileStorage, + "opendal": OpenDALFileStorage, + "webdav": WebDAVFileStorage, +} diff --git a/core/storage/_base.py b/core/storage/_base.py new file mode 100644 index 000000000..47ff99db8 --- /dev/null +++ b/core/storage/_base.py @@ -0,0 +1,181 @@ +# @Time : 2023/8/11 20:06 +# @Author : Lan +# @File : storage.py +# @Software: PyCharm +import hashlib +from typing import BinaryIO, Optional +from urllib.parse import quote + +from collections.abc import Callable +from typing import Any +from dataclasses import dataclass +from pathlib import Path + + +@dataclass + + +class StoredDownload: + """Framework-free description of a file download. + + Backends return this; the view layer builds the starlette Response: + - ``path`` set -> FileResponse (local files, Range support for free) + - ``content`` set -> small full-read Response (legacy OpenDAL fallback) + - ``stream_factory`` set -> StreamingResponse(stream_factory(), ...) + ``background`` is an optional response-sent cleanup hook. + """ + + filename: str + headers: dict + media_type: str = "application/octet-stream" + path: Path | None = None + content: bytes | None = None + stream_factory: Callable[[], Any] | None = None + background: Any = None # starlette BackgroundTask（core 层不 import starlette） + + +@dataclass +class StoredFile: + """Plain, framework- and ORM-free description of a stored file. + + Storage backends accept this instead of ORM models so core/ never imports + apps/. Callers (views/tasks/services) build it from their own records. + """ + + file_path: str + uuid_file_name: str + code: str = "" + prefix: str = "" + suffix: str = "" + text: str = "" + + def get_file_path(self) -> str: + return f"{self.file_path}/{self.uuid_file_name}" + + + + +# S3 multipart 除最后一片外每部分最小 5MB（服务端强制，小于即 EntityTooSmall） +S3_MIN_MULTIPART_PART_SIZE = 5 * 1024 * 1024 + + +def build_attachment_headers(filename: str, content_length=None) -> dict: + """所有存储后端统一的下载响应头。 + + Content-Disposition: attachment 是防御存储型 XSS 的关键——同源下载路径 + （/share/download）因此永不内联渲染 HTML/SVG。此函数是唯一构造点， + 新增后端必须复用（tests/test_attachment_guard.py 有源码级 tripwire）。 + """ + encoded_filename = quote(filename, safe="") + headers = {"Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}"} + if content_length is not None: + headers["Content-Length"] = str(content_length) + return headers + +class FileStorageInterface: + + @staticmethod + def _get_chunk_record(chunk_records: dict, index: int): + """Look up the caller-provided record for chunk `index`. + + Records are plain objects exposing ``chunk_hash``; fetching them from + the DB is the caller's job (keeps storage ORM-free). + """ + chunk_record = chunk_records.get(index) + if not chunk_record: + raise ValueError(f"分片{index}记录不存在") + return chunk_record + + def _verify_and_hash_chunk( + self, + index: int, + chunk_record, + chunk_data: bytes, + file_sha256, + ) -> None: + """Verify a chunk against its recorded hash, then stripe it into the + whole-file digest. Raises the shared ValueError wording on mismatch.""" + current_hash = hashlib.sha256(chunk_data).hexdigest() + if current_hash != chunk_record.chunk_hash: + raise ValueError( + f"分片{index}哈希不匹配: 期望 {chunk_record.chunk_hash}, 实际 {current_hash}" + ) + file_sha256.update(chunk_data) + + async def save_file( + self, stream: BinaryIO, save_path: str, content_type: Optional[str] = None + ): + """Save a binary stream (caller owns closing the stream).""" + raise NotImplementedError + + async def delete_file(self, file_code: StoredFile): + """ + 删除文件 + """ + raise NotImplementedError + + async def get_file_url(self, file_code: StoredFile): + """ + 获取文件分享的url + + 如果服务不支持直接访问文件，可以通过服务器中转下载。 + 此时，此方法可以调用 utils.py 中的 `get_file_url` 方法，获取服务器中转下载的url + """ + raise NotImplementedError + + async def get_file_response(self, file_code: StoredFile): + """ + 获取文件响应 + + 如果服务不支持直接访问文件，则需要实现该方法，返回文件响应 + 其余情况，可以不实现该方法 + """ + raise NotImplementedError + + async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str): + """ + 保存分片文件 + :param upload_id: 上传会话ID + :param chunk_index: 分片索引 + :param chunk_data: 分片数据 + :param chunk_hash: 分片哈希值 + :param save_path: 文件保存路径 + """ + raise NotImplementedError + + async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]: + """ + 合并分片文件并返回文件路径和完整哈希值 + :param upload_id: 上传会话ID + :param chunk_info: 分片信息 + :param save_path: 文件保存路径 + :return: (文件路径, 文件哈希值) + """ + raise NotImplementedError + + async def generate_presigned_upload_url(self, save_path: str, expires_in: int = 900) -> Optional[str]: + """ + 生成预签名上传URL + :param save_path: 文件保存路径 + :param expires_in: URL过期时间（秒），默认15分钟 + :return: 预签名URL，如果不支持直传则返回None + """ + return None # 默认不支持直传，使用代理模式 + + async def file_exists(self, save_path: str) -> bool: + """ + 检查文件是否存在 + :param save_path: 文件路径 + :return: 文件是否存在 + """ + raise NotImplementedError + + async def clean_chunks(self, upload_id: str, save_path: str): + """ + 清理临时分片文件 + :param upload_id: 上传会话ID + :param save_path: 文件保存路径 + """ + raise NotImplementedError + + diff --git a/core/storage/local.py b/core/storage/local.py new file mode 100644 index 000000000..75942f5f3 --- /dev/null +++ b/core/storage/local.py @@ -0,0 +1,184 @@ +import hashlib +from core.logger import logger +import shutil +from typing import BinaryIO, Optional + +import aiofiles +import asyncio +from pathlib import Path +from core.errors import StorageError +from core.settings import data_root +from core.utils import get_file_url, sanitize_filename + +from core.storage._base import ( + FileStorageInterface, + StoredDownload, + StoredFile, + build_attachment_headers, +) + + +class SystemFileStorage(FileStorageInterface): + def __init__(self): + self.chunk_size = 256 * 1024 + self.root_path = data_root + + def _resolve_safe_path(self, relative_path: str) -> Path: + """将相对路径解析到数据根目录内，阻止路径穿越。""" + root = self.root_path.resolve() + raw = str(relative_path or "").replace("\\", "/").lstrip("/") + if any(part == ".." for part in raw.split("/")): + raise ValueError("非法文件路径") + candidate = (root / raw).resolve() + try: + candidate.relative_to(root) + except ValueError as exc: + raise ValueError("非法文件路径") from exc + return candidate + + def _save(self, file, save_path): + with open(save_path, "wb") as f: + chunk = file.read(self.chunk_size) + while chunk: + f.write(chunk) + chunk = file.read(self.chunk_size) + + async def save_file( + self, stream: BinaryIO, save_path: str, content_type: Optional[str] = None + ): + path_obj = Path(str(save_path).replace("\\", "/")) + directory = str(path_obj.parent).replace("\\", "/").lstrip("/") + # 提取原始文件名并进行清理 + filename = await sanitize_filename(path_obj.name) + # 构建安全的完整保存路径 + safe_save_path = self._resolve_safe_path(f"{directory}/{filename}" if directory not in {"", "."} else filename) + # 确保目录存在 + if not safe_save_path.parent.exists(): + safe_save_path.parent.mkdir(parents=True) + await asyncio.to_thread(self._save, stream, safe_save_path) + + async def delete_file(self, file_code: StoredFile): + save_path = self._resolve_safe_path(file_code.get_file_path()) + if save_path.exists(): + save_path.unlink() + + async def get_file_url(self, file_code: StoredFile): + return await get_file_url(file_code.code) + + async def get_file_response(self, file_code: StoredFile): + file_path = self._resolve_safe_path(file_code.get_file_path()) + if not file_path.exists(): + raise StorageError(status_code=404, detail="文件已过期删除") + filename = f"{file_code.prefix}{file_code.suffix}" + try: + headers = build_attachment_headers(filename, file_path.stat().st_size) + except OSError: + # 文件大小不可得时省略 Content-Length + headers = build_attachment_headers(filename) + + return StoredDownload( + filename=filename, + headers=headers, + path=file_path, + ) + + async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str): + """ + 保存分片文件到本地文件系统 + :param upload_id: 上传会话ID + :param chunk_index: 分片索引 + :param chunk_data: 分片数据 + :param chunk_hash: 分片哈希值 + :param save_path: 文件保存路径 + """ + # 先校验目标文件路径合法，再将分片落到同级 chunks 目录。 + self._resolve_safe_path(save_path) + chunk_path = self._resolve_safe_path( + str(Path(save_path).parent / "chunks" / upload_id / f"{chunk_index}.part") + ) + if not chunk_path.parent.exists(): + chunk_path.parent.mkdir(parents=True, exist_ok=True) + # 使用临时文件写入，确保原子性 + temp_path = chunk_path.with_suffix('.tmp') + try: + async with aiofiles.open(temp_path, "wb") as f: + await f.write(chunk_data) + # 原子重命名 + temp_path.rename(chunk_path) + except Exception as e: + if temp_path.exists(): + temp_path.unlink() + raise e + + async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]: + """ + 合并本地文件系统的分片文件并返回文件路径和完整哈希值 + :param upload_id: 上传会话ID + :param chunk_info: 分片信息 + :param save_path: 文件保存路径 + :return: (文件路径, 文件哈希值) + """ + output_path = self._resolve_safe_path(save_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + chunk_base_dir = self._resolve_safe_path( + str(Path(save_path).parent / "chunks" / upload_id) + ) + file_sha256 = hashlib.sha256() + + # 使用临时文件写入，确保原子性 + temp_output = output_path.with_suffix('.merging') + try: + async with aiofiles.open(temp_output, "wb") as out_file: + for i in range(total_chunks): + # 获取分片记录 + chunk_record = self._get_chunk_record(chunk_records, i) + chunk_path = chunk_base_dir / f"{i}.part" + if not chunk_path.exists(): + raise ValueError(f"分片{i}文件不存在") + async with aiofiles.open(chunk_path, "rb") as in_file: + chunk_data = await in_file.read() + self._verify_and_hash_chunk(i, chunk_record, chunk_data, file_sha256) + await out_file.write(chunk_data) + # 原子重命名 + temp_output.rename(output_path) + except Exception as e: + if temp_output.exists(): + temp_output.unlink() + raise e + return str(output_path), file_sha256.hexdigest() + + async def clean_chunks(self, upload_id: str, save_path: str): + """ + 清理本地文件系统的临时分片文件 + :param upload_id: 上传会话ID + :param save_path: 文件保存路径 + """ + chunk_dir = self._resolve_safe_path( + str(Path(save_path).parent / "chunks" / upload_id) + ) + if chunk_dir.exists(): + try: + shutil.rmtree(chunk_dir) + except Exception as e: + logger.warning(f"清理本地分片目录失败: {e}") + # 清理父级 chunks 目录（如果为空） + chunks_parent = chunk_dir.parent + if chunks_parent.exists() and not any(chunks_parent.iterdir()): + try: + chunks_parent.rmdir() + except Exception as e: + logger.warning(f"清理 chunks 父目录失败: {e}") + + async def file_exists(self, save_path: str) -> bool: + """ + 检查文件是否存在于本地文件系统 + :param save_path: 文件路径 + :return: 文件是否存在 + """ + try: + file_path = self._resolve_safe_path(save_path) + except ValueError: + return False + return file_path.exists() + + diff --git a/core/storage/onedrive.py b/core/storage/onedrive.py new file mode 100644 index 000000000..ba46000f1 --- /dev/null +++ b/core/storage/onedrive.py @@ -0,0 +1,339 @@ +import hashlib +import os +import tempfile +from core.logger import logger +from typing import BinaryIO, Optional + +import aiofiles +import aiohttp +import asyncio +from pathlib import Path +import datetime +import re +from core.errors import StorageError +from core.settings import settings +from core.utils import get_file_url, sanitize_filename +from starlette.background import BackgroundTask + +from core.storage._base import ( + FileStorageInterface, + StoredDownload, + StoredFile, + build_attachment_headers, +) + + +class OneDriveFileStorage(FileStorageInterface): + def __init__(self): + try: + import msal + from office365.graph_client import GraphClient + from office365.runtime.client_request_exception import ( + ClientRequestException, + ) + except ImportError: + raise ImportError("请先安装`msal`和`Office365-REST-Python-Client`") + self.msal = msal + self.domain = settings.onedrive_domain + self.client_id = settings.onedrive_client_id + self.username = settings.onedrive_username + self.password = settings.onedrive_password + self.proxy = settings.onedrive_proxy + self._ClientRequestException = ClientRequestException + + try: + client = GraphClient(self.acquire_token_pwd) + self.root_path = ( + client.me.drive.root.get_by_path(settings.onedrive_root_path) + .get() + .execute_query() + ) + except ClientRequestException as e: + if e.code == "itemNotFound": + client.me.drive.root.create_folder(settings.onedrive_root_path) + self.root_path = ( + client.me.drive.root.get_by_path( + settings.onedrive_root_path) + .get() + .execute_query() + ) + else: + raise e + except Exception as e: + raise Exception("OneDrive验证失败，请检查配置是否正确\n" + str(e)) + + def acquire_token_pwd(self): + authority_url = f"https://login.microsoftonline.com/{self.domain}" + app = self.msal.PublicClientApplication( + authority=authority_url, client_id=self.client_id + ) + result = app.acquire_token_by_username_password( + username=self.username, + password=self.password, + scopes=["https://graph.microsoft.com/.default"], + ) + return result + + def _get_path_str(self, path): + if isinstance(path, str): + path = path.replace("\\", "/").replace("//", "/").split("/") + elif isinstance(path, Path): + path = str(path).replace("\\", "/").replace("//", "/").split("/") + else: + raise TypeError("path must be str or Path") + path[-1] = path[-1].split(".")[0] + return "/".join(path) + + async def save_file( + self, stream: BinaryIO, save_path: str, content_type: Optional[str] = None + ): + """保存文件（自动创建目录；修复旧实现把 save_path 字符串当函数调用的崩溃）""" + content = await asyncio.to_thread(stream.read) + normalized = str(save_path).replace("\\", "/") + name = await sanitize_filename(Path(normalized).name) + dir_path = "/".join(normalized.split("/")[:-1]) + + current_folder = self.root_path + for part in dir_path.split("/"): + if not part: + continue + try: + current_folder = current_folder.get_by_path(part).get().execute_query() + except self._ClientRequestException as e: + if e.code == "itemNotFound": + current_folder = current_folder.create_folder(part).execute_query() + else: + raise e + + await asyncio.to_thread( + lambda: current_folder.get_by_path(name) + .upload(name, content) + .execute_query() + ) + + def _delete(self, save_path): + path = self._get_path_str(save_path) + try: + self.root_path.get_by_path(path).delete_object().execute_query() + except self._ClientRequestException as e: + if e.code == "itemNotFound": + pass + else: + raise e + + async def delete_file(self, file_code: StoredFile): + await asyncio.to_thread(self._delete, file_code.get_file_path()) + + def _convert_link_to_download_link(self, link): + p1 = re.search(r"https://(.+)\.sharepoint\.com", link).group(1) + p2 = re.search(r"personal/(.+)/", link).group(1) + p3 = re.search(rf"{p2}/(.+)", link).group(1) + return f"https://{p1}.sharepoint.com/personal/{p2}/_layouts/52/download.aspx?share={p3}" + + def _get_file_url(self, save_path, name): + path = self._get_path_str(save_path) + remote_file = self.root_path.get_by_path(path + "/" + name) + expiration_datetime = datetime.datetime.now( + tz=datetime.timezone.utc + ) + datetime.timedelta(hours=1) + expiration_datetime = expiration_datetime.strftime( + "%Y-%m-%dT%H:%M:%SZ") + permission = remote_file.create_link( + "view", "anonymous", expiration_datetime=expiration_datetime + ).execute_query() + return self._convert_link_to_download_link(permission.link.webUrl) + + async def get_file_response(self, file_code: StoredFile): + try: + filename = file_code.prefix + file_code.suffix + try: + link = await asyncio.to_thread( + self._get_file_url, file_code.get_file_path(), filename + ) + except self._ClientRequestException as e: + # 对象不存在时前置 404（与 local/S3/WebDAV 语义对齐）， + # 不再落入外层兜底的 503 + if str(getattr(e, "code", "")).lower() in {"itemnotfound", "404"}: + raise StorageError( + status_code=404, detail="文件已过期删除" + ) from e + raise + + content_length = None # 初始化为 None，表示未知大小 + + # 创建ClientSession并复用 + session = aiohttp.ClientSession() + + # 尝试发送HEAD请求获取Content-Length + try: + async with session.head(link) as resp: + if resp.status == 200 and 'Content-Length' in resp.headers: + content_length = int(resp.headers['Content-Length']) + except Exception: + # 如果HEAD请求失败，则不提供 Content-Length + pass + + async def stream_generator(): + try: + async with session.get(link) as resp: + if resp.status != 200: + raise StorageError( + status_code=resp.status, + detail=f"从OneDrive获取文件失败: {resp.status}" + ) + chunk_size = 65536 + while True: + chunk = await resp.content.read(chunk_size) + if not chunk: + break + yield chunk + finally: + await session.close() + + headers = build_attachment_headers(filename, content_length) + return StoredDownload( + filename=filename, + headers=headers, + stream_factory=stream_generator, + # 兜底关闭会话：客户端中断时与 generator finally 双保险 + background=BackgroundTask(session.close), + ) + except StorageError: + raise + except Exception: + raise StorageError(status_code=503, detail="服务代理下载异常，请稍后再试") + + async def get_file_url(self, file_code: StoredFile): + if self.proxy: + return await get_file_url(file_code.code) + else: + return await asyncio.to_thread( + self._get_file_url, + file_code.get_file_path(), + f"{file_code.prefix}{file_code.suffix}", + ) + + def _save_chunk(self, chunk_path: str, chunk_data: bytes): + """同步保存分片到 OneDrive""" + path_parts = chunk_path.replace("\\", "/").split("/") + filename = path_parts[-1] + dir_path = "/".join(path_parts[:-1]) + + # 确保目录存在 + current_folder = self.root_path + for part in dir_path.split("/"): + if part: + try: + current_folder = current_folder.get_by_path(part).get().execute_query() + except self._ClientRequestException as e: + if e.code == "itemNotFound": + current_folder = current_folder.create_folder(part).execute_query() + else: + raise e + + # 上传分片 + current_folder.upload(filename, chunk_data).execute_query() + + async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str): + """保存分片到 OneDrive""" + chunk_path = str(Path(save_path).parent / "chunks" / upload_id / f"{chunk_index}.part") + await asyncio.to_thread(self._save_chunk, chunk_path, chunk_data) + + def _read_chunk(self, chunk_path: str) -> bytes: + """同步读取分片""" + path = self._get_path_str(chunk_path) + file_obj = self.root_path.get_by_path(path).get().execute_query() + return file_obj.get_content().execute_query().value + + def _upload_merged(self, save_path: str, data: bytes): + """同步上传合并后的文件""" + path_parts = save_path.replace("\\", "/").split("/") + filename = path_parts[-1] + dir_path = "/".join(path_parts[:-1]) + + # 确保目录存在 + current_folder = self.root_path + for part in dir_path.split("/"): + if part: + try: + current_folder = current_folder.get_by_path(part).get().execute_query() + except self._ClientRequestException as e: + if e.code == "itemNotFound": + current_folder = current_folder.create_folder(part).execute_query() + else: + raise e + + current_folder.upload(filename, data).execute_query() + + async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]: + """合并 OneDrive 上的分片文件，使用临时文件避免内存问题""" + file_sha256 = hashlib.sha256() + chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) + + # 使用临时文件存储合并数据 + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + temp_path = temp_file.name + + try: + async with aiofiles.open(temp_path, 'wb') as out_file: + for i in range(total_chunks): + chunk_path = f"{chunk_dir}/{i}.part" + chunk_record = self._get_chunk_record(chunk_records, i) + + try: + chunk_data = await asyncio.to_thread(self._read_chunk, chunk_path) + except Exception as e: + raise ValueError(f"分片{i}文件不存在: {e}") + + self._verify_and_hash_chunk(i, chunk_record, chunk_data, file_sha256) + await out_file.write(chunk_data) + del chunk_data # 释放内存 + + # 读取临时文件并上传 + async with aiofiles.open(temp_path, 'rb') as f: + merged_content = await f.read() + await asyncio.to_thread(self._upload_merged, save_path, merged_content) + finally: + # 清理临时文件 + if os.path.exists(temp_path): + os.unlink(temp_path) + + return save_path, file_sha256.hexdigest() + + def _delete_chunk_dir(self, chunk_dir: str): + """同步删除分片目录""" + try: + path = self._get_path_str(chunk_dir) + self.root_path.get_by_path(path).delete_object().execute_query() + except self._ClientRequestException as e: + if e.code != "itemNotFound": + raise e + + async def clean_chunks(self, upload_id: str, save_path: str): + """清理 OneDrive 上的临时分片文件""" + chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) + try: + await asyncio.to_thread(self._delete_chunk_dir, chunk_dir) + except Exception as e: + logger.warning(f"清理 OneDrive 分片时出错: {e}") + + def _file_exists(self, save_path: str) -> bool: + """同步检查文件是否存在""" + try: + path = self._get_path_str(save_path) + self.root_path.get_by_path(path).get().execute_query() + return True + except self._ClientRequestException as e: + if e.code == "itemNotFound": + return False + raise e + + async def file_exists(self, save_path: str) -> bool: + """ + 检查文件是否存在于OneDrive + :param save_path: 文件路径 + :return: 文件是否存在 + """ + return await asyncio.to_thread(self._file_exists, save_path) + + diff --git a/core/storage/opendal.py b/core/storage/opendal.py new file mode 100644 index 000000000..bdcb34d46 --- /dev/null +++ b/core/storage/opendal.py @@ -0,0 +1,160 @@ +import hashlib +import os +import tempfile +from core.logger import logger +from typing import BinaryIO, Optional + +import aiofiles +import asyncio +from pathlib import Path +from core.errors import StorageError +from core.settings import settings +from core.utils import get_file_url + +from core.storage._base import ( + FileStorageInterface, + StoredDownload, + StoredFile, + build_attachment_headers, +) + + +class OpenDALFileStorage(FileStorageInterface): + def __init__(self): + try: + import opendal + except ImportError: + raise ImportError('请先安装 `opendal`, 例如: "pip install opendal"') + self.service = settings.opendal_scheme + service_settings = {} + for key, value in settings.items(): + if key.startswith("opendal_" + self.service): + setting_name = key.split("_", 2)[2] + service_settings[setting_name] = value + self.operator = opendal.AsyncOperator( + settings.opendal_scheme, **service_settings + ) + + async def save_file( + self, stream: BinaryIO, save_path: str, content_type: Optional[str] = None + ): + # 使用 asyncio.to_thread 避免阻塞事件循环 + content = await asyncio.to_thread(stream.read) + await self.operator.write(save_path, content) + + async def delete_file(self, file_code: StoredFile): + await self.operator.delete(file_code.get_file_path()) + + async def get_file_url(self, file_code: StoredFile): + return await get_file_url(file_code.code) + + async def get_file_response(self, file_code: StoredFile): + try: + filename = file_code.prefix + file_code.suffix + content_length = None # 初始化为 None，表示未知大小 + + # 尝试获取文件大小 + try: + stat_result = await self.operator.stat(file_code.get_file_path()) + if hasattr(stat_result, 'content_length') and stat_result.content_length: + content_length = stat_result.content_length + elif hasattr(stat_result, 'size') and stat_result.size: + content_length = stat_result.size + except Exception: + # 如果获取大小失败，则不提供 Content-Length + pass + + # 尝试使用流式读取器 + try: + # OpenDAL 可能提供 reader 方法返回一个异步读取器 + reader = await self.operator.reader(file_code.get_file_path()) + except AttributeError: + # 如果 reader 方法不存在，回退到全量读取（兼容旧版本） + content = await self.operator.read(file_code.get_file_path()) + headers = build_attachment_headers(filename, content_length) + return StoredDownload( + filename=filename, + headers=headers, + content=content, + ) + + async def stream_generator(): + chunk_size = 65536 + while True: + chunk = await reader.read(chunk_size) + if not chunk: + break + yield chunk + + headers = build_attachment_headers(filename, content_length) + return StoredDownload( + filename=filename, + headers=headers, + stream_factory=stream_generator, + ) + except Exception as e: + logger.info(e) + raise StorageError(status_code=404, detail="文件已过期删除") + + async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str): + """保存分片到 OpenDAL 存储""" + chunk_path = str(Path(save_path).parent / "chunks" / upload_id / f"{chunk_index}.part") + await self.operator.write(chunk_path, chunk_data) + + async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]: + """合并 OpenDAL 存储上的分片文件，使用临时文件避免内存问题""" + file_sha256 = hashlib.sha256() + chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) + + # 使用临时文件存储合并数据 + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + temp_path = temp_file.name + + try: + async with aiofiles.open(temp_path, 'wb') as out_file: + for i in range(total_chunks): + chunk_path = f"{chunk_dir}/{i}.part" + chunk_record = self._get_chunk_record(chunk_records, i) + + try: + chunk_data = await self.operator.read(chunk_path) + except Exception as e: + raise ValueError(f"分片{i}文件不存在: {e}") + + self._verify_and_hash_chunk(i, chunk_record, chunk_data, file_sha256) + await out_file.write(chunk_data) + del chunk_data # 释放内存 + + # 读取临时文件并写入存储 + async with aiofiles.open(temp_path, 'rb') as f: + merged_content = await f.read() + await self.operator.write(save_path, merged_content) + finally: + # 清理临时文件 + if os.path.exists(temp_path): + os.unlink(temp_path) + + return save_path, file_sha256.hexdigest() + + async def clean_chunks(self, upload_id: str, save_path: str): + """清理 OpenDAL 存储上的临时分片文件""" + chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) + try: + # OpenDAL 支持递归删除 + await self.operator.remove_all(chunk_dir) + except Exception as e: + logger.warning(f"清理 OpenDAL 分片时出错: {e}") + + async def file_exists(self, save_path: str) -> bool: + """ + 检查文件是否存在于OpenDAL存储 + :param save_path: 文件路径 + :return: 文件是否存在 + """ + try: + await self.operator.stat(save_path) + return True + except Exception: + return False + + diff --git a/core/storage/s3.py b/core/storage/s3.py new file mode 100644 index 000000000..2dd25eacc --- /dev/null +++ b/core/storage/s3.py @@ -0,0 +1,345 @@ +from botocore.exceptions import ClientError +import hashlib +from core.logger import logger +from typing import BinaryIO, Optional + +import aiohttp +import asyncio +from pathlib import Path +import aioboto3 +from botocore.config import Config +from core.errors import StorageError +from core.settings import settings +from core.utils import get_file_url +from starlette.background import BackgroundTask + +from core.storage._base import ( + FileStorageInterface, + S3_MIN_MULTIPART_PART_SIZE, + StoredDownload, + StoredFile, + build_attachment_headers, +) + + +class S3FileStorage(FileStorageInterface): + def __init__(self): + self.access_key_id = settings.s3_access_key_id + self.secret_access_key = settings.s3_secret_access_key + self.bucket_name = settings.s3_bucket_name + self.s3_hostname = settings.s3_hostname + self.region_name = settings.s3_region_name + self.signature_version = settings.s3_signature_version + self.endpoint_url = settings.s3_endpoint_url or f"https://{self.s3_hostname}" + self.aws_session_token = settings.aws_session_token + self.addressing_style = str(settings.s3_addressing_style or "auto").lower() + self.proxy = settings.s3_proxy + self.session = aioboto3.Session( + aws_access_key_id=self.access_key_id, + aws_secret_access_key=self.secret_access_key, + ) + if not settings.s3_endpoint_url: + self.endpoint_url = f"https://{self.s3_hostname}" + else: + # 如果提供了 s3_endpoint_url，则优先使用它 + self.endpoint_url = settings.s3_endpoint_url + + def _client_config(self) -> Config: + config = {"signature_version": self.signature_version} + s3_config = {} + if self.addressing_style in {"path", "virtual", "auto"}: + s3_config["addressing_style"] = self.addressing_style + if s3_config: + config["s3"] = s3_config + return Config(**config) + + def _client(self): + return self.session.client( + "s3", + endpoint_url=self.endpoint_url, + aws_session_token=self.aws_session_token, + region_name=self.region_name, + config=self._client_config(), + ) + + async def save_file( + self, stream: BinaryIO, save_path: str, content_type: Optional[str] = None + ): + async with self._client() as s3: + # 使用 upload_fileobj 流式上传，避免将整个文件加载到内存 + await s3.upload_fileobj( + stream, + self.bucket_name, + save_path, + ExtraArgs={"ContentType": content_type or "application/octet-stream"}, + ) + + async def delete_file(self, file_code: StoredFile): + async with self._client() as s3: + await s3.delete_object( + Bucket=self.bucket_name, Key=file_code.get_file_path() + ) + + async def get_file_response(self, file_code: StoredFile): + try: + filename = file_code.prefix + file_code.suffix + content_length = None # 初始化为 None，表示未知大小 + + async with self._client() as s3: + # 尝试获取文件大小（HEAD请求）；对象不存在时前置 404—— + # 与 local 后端语义一致（M2 行为统一），不能签出 200 的坏流 + try: + head_response = await s3.head_object( + Bucket=self.bucket_name, + Key=file_code.get_file_path() + ) + # 从HEAD响应中获取Content-Length + if 'ContentLength' in head_response: + content_length = head_response['ContentLength'] + elif 'Content-Length' in head_response['ResponseMetadata']['HTTPHeaders']: + content_length = int(head_response['ResponseMetadata']['HTTPHeaders']['Content-Length']) + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code", "") + if error_code in {"404", "NoSuchKey", "NotFound"}: + raise StorageError( + status_code=404, detail="文件已过期删除" + ) from e + # 其他 HEAD 错误不阻断：流式下载阶段会给出真实状态 + except Exception: + # 如果HEAD请求失败，则不提供 Content-Length + pass + + link = await s3.generate_presigned_url( + "get_object", + Params={ + "Bucket": self.bucket_name, + "Key": file_code.get_file_path(), + }, + ExpiresIn=3600, + ) + + # 创建ClientSession并传递给生成器复用 + session = aiohttp.ClientSession() + + async def stream_generator(): + try: + async with session.get(link) as resp: + if resp.status != 200: + raise StorageError( + status_code=resp.status, + detail=f"从S3获取文件失败: {resp.status}" + ) + # 设置块大小（例如64KB） + chunk_size = 65536 + while True: + chunk = await resp.content.read(chunk_size) + if not chunk: + break + yield chunk + finally: + await session.close() + + headers = build_attachment_headers(filename, content_length) + return StoredDownload( + filename=filename, + headers=headers, + stream_factory=stream_generator, + # 兜底关闭会话：客户端中断时与 generator finally 双保险 + background=BackgroundTask(session.close), + ) + except StorageError: + raise + except Exception: + raise StorageError(status_code=503, detail="服务代理下载异常，请稍后再试") + + async def get_file_url(self, file_code: StoredFile): + if file_code.prefix == "文本分享": + return file_code.text + if self.proxy: + return await get_file_url(file_code.code) + else: + async with self._client() as s3: + result = await s3.generate_presigned_url( + "get_object", + Params={ + "Bucket": self.bucket_name, + "Key": file_code.get_file_path(), + }, + ExpiresIn=3600, + ) + return result + + async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str): + """ + 保存分片到 S3（使用独立对象存储每个分片） + 注意：这里不使用 S3 原生的 multipart upload，而是将每个分片作为独立对象存储 + """ + chunk_key = str(Path(save_path).parent / "chunks" / upload_id / f"{chunk_index}.part") + async with self._client() as s3: + # 将分片作为独立对象上传 + await s3.put_object( + Bucket=self.bucket_name, + Key=chunk_key, + Body=chunk_data, + Metadata={ + 'chunk-hash': chunk_hash, + 'chunk-index': str(chunk_index) + } + ) + + async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]: + """ + 合并 S3 上的分片文件 + 使用 S3 的 multipart upload API 实现流式合并，避免内存问题 + """ + file_sha256 = hashlib.sha256() + chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) + + async with self._client() as s3: + # 创建 multipart upload + mpu = await s3.create_multipart_upload( + Bucket=self.bucket_name, + Key=save_path, + ContentType='application/octet-stream' + ) + mpu_id = mpu['UploadId'] + parts = [] + + try: + # 按顺序读取、验证每个分片；S3 multipart 规范要求除最后一片外 + # 每个部分 ≥5MB（EntityTooSmall），而分片大小由客户端决定（常见 + # 2-4MB）——因此缓冲到 S3_MIN_MULTIPART_PART_SIZE 再上传 part， + # 内存上界 = 5MB + 单个分片，不破坏流式合并的初衷。 + part_buffer = bytearray() + part_number = 0 + + async def _flush_part(): + nonlocal part_number + if not part_buffer: + return + part_number += 1 + part_response = await s3.upload_part( + Bucket=self.bucket_name, + Key=save_path, + UploadId=mpu_id, + PartNumber=part_number, + Body=bytes(part_buffer), + ) + parts.append({ + 'PartNumber': part_number, + 'ETag': part_response['ETag'] + }) + part_buffer.clear() + + for i in range(total_chunks): + chunk_key = f"{chunk_dir}/{i}.part" + chunk_record = self._get_chunk_record(chunk_records, i) + + try: + response = await s3.get_object( + Bucket=self.bucket_name, + Key=chunk_key + ) + chunk_data = await response['Body'].read() + except Exception as e: + raise ValueError(f"分片{i}文件不存在: {e}") + + self._verify_and_hash_chunk(i, chunk_record, chunk_data, file_sha256) + part_buffer.extend(chunk_data) + if len(part_buffer) >= S3_MIN_MULTIPART_PART_SIZE: + await _flush_part() + + # 收尾：剩余缓冲作为最后一个 part（S3 允许最后一片小于 5MB； + # 恰好整除时缓冲为空，跳过） + await _flush_part() + + # 完成 multipart upload + await s3.complete_multipart_upload( + Bucket=self.bucket_name, + Key=save_path, + UploadId=mpu_id, + MultipartUpload={'Parts': parts} + ) + except Exception as e: + # 出错时取消 multipart upload + await s3.abort_multipart_upload( + Bucket=self.bucket_name, + Key=save_path, + UploadId=mpu_id + ) + raise e + + return save_path, file_sha256.hexdigest() + + async def clean_chunks(self, upload_id: str, save_path: str): + """ + 清理 S3 上的临时分片文件 + :param upload_id: 上传会话ID + :param save_path: 文件保存路径 + """ + chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) + async with self._client() as s3: + try: + # 列出并删除所有分片对象 + paginator = s3.get_paginator('list_objects_v2') + async for page in paginator.paginate(Bucket=self.bucket_name, Prefix=chunk_dir): + objects = page.get('Contents', []) + if objects: + delete_objects = [{'Key': obj['Key']} for obj in objects] + await s3.delete_objects( + Bucket=self.bucket_name, + Delete={'Objects': delete_objects} + ) + except Exception as e: + logger.warning(f"清理 S3 分片数据时出错: {e}") + + async def generate_presigned_upload_url(self, save_path: str, expires_in: int = 900) -> Optional[str]: + """ + 生成S3预签名上传URL + :param save_path: 文件保存路径 + :param expires_in: URL过期时间（秒），默认15分钟 + :return: 预签名PUT URL + """ + async with self._client() as s3: + return await s3.generate_presigned_url( + "put_object", + Params={ + "Bucket": self.bucket_name, + "Key": save_path, + }, + ExpiresIn=expires_in, + ) + + async def file_exists(self, save_path: str) -> bool: + """ + 检查文件是否存在于S3 + :param save_path: 文件路径 + :return: 文件是否存在 + """ + async with self._client() as s3: + last_error = None + for attempt in range(3): + try: + await s3.head_object(Bucket=self.bucket_name, Key=save_path) + return True + except Exception as e: + last_error = e + if attempt < 2: + await asyncio.sleep(0.2 * (attempt + 1)) + + try: + result = await s3.list_objects_v2( + Bucket=self.bucket_name, + Prefix=save_path, + MaxKeys=1, + ) + for item in result.get("Contents", []): + if item.get("Key") == save_path: + return True + except Exception as e: + last_error = e + + logger.warning(f"S3文件确认失败 key={save_path}: {last_error}") + return False + + diff --git a/core/storage/webdav.py b/core/storage/webdav.py new file mode 100644 index 000000000..710f23ca0 --- /dev/null +++ b/core/storage/webdav.py @@ -0,0 +1,354 @@ +import base64 +import hashlib +import os +import tempfile +from core.logger import logger +from typing import BinaryIO, Optional +from urllib.parse import quote, unquote + +import aiofiles +import aiohttp +import asyncio +from pathlib import Path +import re +from core.errors import StorageError +from core.settings import settings +from core.utils import get_file_url, sanitize_filename +from starlette.background import BackgroundTask + +from core.storage._base import ( + FileStorageInterface, + StoredDownload, + StoredFile, + build_attachment_headers, +) + + +class WebDAVFileStorage(FileStorageInterface): + def __init__(self): + if not hasattr(self, "_initialized"): + self.base_url = settings.webdav_url.rstrip("/") + "/" + # aiohttp 4.0 移除 BasicAuth(auth=...) 参数——改用编码后的 Authorization 头 + self.auth_headers = { + "Authorization": aiohttp.encode_basic_auth( + settings.webdav_username, settings.webdav_password + ) + } + self._initialized = True + + def _build_url(self, path: str) -> str: + encoded_path = quote(str(path.replace("\\", "/").lstrip("/")).lstrip("/")) + return f"{self.base_url}{encoded_path}" + + async def _mkdir_p(self, directory_path: str): + """递归创建目录（类似mkdir -p）""" + path_obj = Path(unquote(directory_path)) + current_path = "" + + async with aiohttp.ClientSession(headers=self.auth_headers) as session: + # 逐级检查目录是否存在 + for part in path_obj.parts: + current_path = str(Path(current_path) / part) + url = self._build_url(current_path) + + # 检查目录是否存在 + async with session.head(url) as resp: + if resp.status == 404: + # 创建目录 + async with session.request("MKCOL", url) as mkcol_resp: + if mkcol_resp.status not in (200, 201, 409): + content = await mkcol_resp.text() + raise StorageError( + status_code=mkcol_resp.status, + detail=f"目录创建失败: {content[:200]}", + ) + + async def _is_dir_empty(self, dir_path: str) -> bool: + """检查目录是否为空""" + url = self._build_url(dir_path) + + async with aiohttp.ClientSession(headers=self.auth_headers) as session: + async with session.request("PROPFIND", url, headers={"Depth": "1"}) as resp: + if resp.status != 207: # 207 是 Multi-Status 响应 + return False + content = await resp.text() + # 如果只有一个 response（当前目录），说明目录为空 + return content.count("") <= 1 + + async def _delete_empty_dirs(self, file_path: str, session: aiohttp.ClientSession): + """递归删除空目录""" + path_obj = Path(file_path) + current_path = path_obj.parent + + while str(current_path) != ".": + if not await self._is_dir_empty(str(current_path)): + break + + url = self._build_url(str(current_path)) + async with session.delete(url) as resp: + if resp.status not in (200, 204, 404): + break + + current_path = current_path.parent + + async def save_file( + self, stream: BinaryIO, save_path: str, content_type: Optional[str] = None + ): + """保存文件（自动创建目录，流式上传）""" + path_obj = Path(save_path) + directory_path = str(path_obj.parent) + # 提取原始文件名并进行清理 + filename = await sanitize_filename(path_obj.name) + # 构建安全的保存路径 + safe_save_path = str(Path(directory_path) / filename) + + try: + # 先创建目录结构 + await self._mkdir_p(directory_path) + # 上传文件（流式） + url = self._build_url(safe_save_path) + + async def file_sender(): + """流式读取文件内容""" + chunk_size = 256 * 1024 # 256KB chunks + while True: + chunk = await asyncio.to_thread(stream.read, chunk_size) + if not chunk: + break + yield chunk + + async with aiohttp.ClientSession(headers=self.auth_headers) as session: + async with session.put( + url, + data=file_sender(), + headers={"Content-Type": content_type or "application/octet-stream"} + ) as resp: + if resp.status not in (200, 201, 204): + content = await resp.text() + raise StorageError( + status_code=resp.status, + detail=f"文件上传失败: {content[:200]}", + ) + except aiohttp.ClientError as e: + raise StorageError( + status_code=503, detail=f"WebDAV连接异常: {str(e)}") + + async def delete_file(self, file_code: StoredFile): + """删除WebDAV文件及空目录""" + file_path = file_code.get_file_path() + url = self._build_url(file_path) + try: + async with aiohttp.ClientSession(headers=self.auth_headers) as session: + # 删除文件 + async with session.delete(url) as resp: + if resp.status not in (200, 204, 404): + content = await resp.text() + raise StorageError( + status_code=resp.status, + detail=f"WebDAV删除失败: {content[:200]}", + ) + + # 使用同一个 session 删除空目录 + await self._delete_empty_dirs(file_path, session) + + except aiohttp.ClientError as e: + raise StorageError( + status_code=503, detail=f"WebDAV连接异常: {str(e)}") + + async def get_file_url(self, file_code: StoredFile): + return await get_file_url(file_code.code) + + async def get_file_response(self, file_code: StoredFile): + """获取文件响应（代理模式）""" + try: + filename = file_code.prefix + file_code.suffix + url = self._build_url(file_code.get_file_path()) + content_length = None # 初始化为 None，表示未知大小 + + # 创建ClientSession并复用（包含认证头） + session = aiohttp.ClientSession(headers={ + "Authorization": f"Basic {base64.b64encode(f'{settings.webdav_username}:{settings.webdav_password}'.encode()).decode()}" + }) + + # 尝试发送HEAD请求获取Content-Length；对象不存在时前置 404 + # （与 local/S3 语义对齐），连接层错误映射 503——两者都不能 + # 静默吞掉后签出 200 坏流。异常路径回收 session 防泄漏。 + try: + try: + async with session.head(url) as resp: + if resp.status == 404: + raise StorageError( + status_code=404, detail="文件已过期删除" + ) + if resp.status == 200 and 'Content-Length' in resp.headers: + content_length = int(resp.headers['Content-Length']) + except StorageError: + raise + except aiohttp.ClientError as e: + raise StorageError( + status_code=503, detail=f"WebDAV连接异常: {str(e)}" + ) from e + except Exception: + # 其他 HEAD 异常不阻断：流式下载阶段会给出真实状态 + pass + + except BaseException: + await session.close() + raise + + async def stream_generator(): + try: + async with session.get(url) as resp: + if resp.status != 200: + raise StorageError( + status_code=resp.status, + detail=f"文件获取失败{resp.status}: {await resp.text()}", + ) + chunk_size = 65536 + while True: + chunk = await resp.content.read(chunk_size) + if not chunk: + break + yield chunk + finally: + await session.close() + + headers = build_attachment_headers(filename, content_length) + return StoredDownload( + filename=filename, + headers=headers, + stream_factory=stream_generator, + # 兜底关闭会话：客户端中断时与 generator finally 双保险 + background=BackgroundTask(session.close), + ) + except aiohttp.ClientError as e: + raise StorageError( + status_code=503, detail=f"WebDAV连接异常: {str(e)}") + + async def save_chunk(self, upload_id: str, chunk_index: int, chunk_data: bytes, chunk_hash: str, save_path: str): + """保存分片到 WebDAV""" + chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) + chunk_path = f"{chunk_dir}/{chunk_index}.part" + + # 先创建目录结构 + await self._mkdir_p(chunk_dir) + + chunk_url = self._build_url(chunk_path) + async with aiohttp.ClientSession(headers=self.auth_headers) as session: + async with session.put(chunk_url, data=chunk_data) as resp: + if resp.status not in (200, 201, 204): + content = await resp.text() + raise StorageError( + status_code=resp.status, + detail=f"分片上传失败: {content[:200]}" + ) + + async def merge_chunks(self, upload_id: str, total_chunks: int, chunk_size: int, save_path: str, chunk_records: dict) -> tuple[str, str]: + """ + 合并 WebDAV 上的分片文件 + 使用临时文件避免内存问题 + """ + file_sha256 = hashlib.sha256() + chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) + + # 使用临时文件存储合并数据，避免内存问题 + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + temp_path = temp_file.name + + try: + async with aiohttp.ClientSession(headers=self.auth_headers) as session: + # 按顺序读取并验证每个分片，写入临时文件 + async with aiofiles.open(temp_path, 'wb') as out_file: + for i in range(total_chunks): + chunk_path = f"{chunk_dir}/{i}.part" + chunk_url = self._build_url(chunk_path) + + # 获取分片记录 + chunk_record = self._get_chunk_record(chunk_records, i) + + # 下载分片数据 + async with session.get(chunk_url) as resp: + if resp.status != 200: + raise ValueError(f"分片{i}文件不存在或无法访问") + chunk_data = await resp.read() + + # 验证哈希 + self._verify_and_hash_chunk(i, chunk_record, chunk_data, file_sha256) + await out_file.write(chunk_data) + del chunk_data # 释放内存 + + # 确保目标目录存在 + output_dir = str(Path(save_path).parent) + await self._mkdir_p(output_dir) + + # 流式上传合并后的文件 + output_url = self._build_url(save_path) + + async def file_sender(): + async with aiofiles.open(temp_path, 'rb') as f: + while True: + chunk = await f.read(256 * 1024) + if not chunk: + break + yield chunk + + async with session.put(output_url, data=file_sender()) as resp: + if resp.status not in (200, 201, 204): + content = await resp.text() + raise StorageError( + status_code=resp.status, + detail=f"合并文件上传失败: {content[:200]}" + ) + finally: + # 清理临时文件 + if os.path.exists(temp_path): + os.unlink(temp_path) + + return save_path, file_sha256.hexdigest() + + async def clean_chunks(self, upload_id: str, save_path: str): + """ + 清理 WebDAV 上的临时分片文件 + :param upload_id: 上传会话ID + :param save_path: 文件保存路径 + """ + chunk_dir = str(Path(save_path).parent / "chunks" / upload_id) + chunk_dir_url = self._build_url(chunk_dir) + async with aiohttp.ClientSession(headers=self.auth_headers) as session: + try: + # 检查分片目录是否存在 + async with session.request("PROPFIND", chunk_dir_url, headers={"Depth": "1"}) as resp: + if resp.status == 207: # 207 表示 Multi-Status + # 获取目录下的所有分片文件 + xml_data = await resp.text() + file_paths = re.findall( + r'(.*?)', xml_data) + for file_path in file_paths: + if file_path.endswith(".part"): + # 删除分片文件 + file_url = self._build_url(file_path) + async with session.delete(file_url) as delete_resp: + if delete_resp.status not in (200, 204, 404): + logger.warning(f"删除分片文件失败: {file_path}") + + # 删除分片目录 + async with session.delete(chunk_dir_url) as delete_resp: + if delete_resp.status not in (200, 204, 404): + logger.warning(f"删除分片目录失败: {chunk_dir_url}") + else: + logger.info(f"分片目录不存在: {chunk_dir_url}") + except Exception as e: + logger.warning(f"清理 WebDAV 分片时出错: {e}") + + async def file_exists(self, save_path: str) -> bool: + """ + 检查文件是否存在于WebDAV + :param save_path: 文件路径 + :return: 文件是否存在 + """ + url = self._build_url(save_path) + async with aiohttp.ClientSession(headers=self.auth_headers) as session: + async with session.head(url) as resp: + return resp.status == 200 + + diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 000000000..3c1cd2873 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# FileCodeBox 容器入口：兼容式非 root 降权。 +# +# 默认以 root 进入（兼容存量 root 属主的数据卷），按需修正 data 目录属主后 +# 用 gosu 降权到 app 用户运行；若部署方显式指定了 user（非 root），则直接运行 +# 不做任何 chown，保证两种形态都可预期。 +set -e + +DATA_DIR="${DATA_DIR:-/app/data}" + +if [ "$(id -u)" = "0" ]; then + mkdir -p "$DATA_DIR" + # 仅当存在非 app 属主的文件时才 chown，避免大卷启动变慢 + if [ -n "$(find "$DATA_DIR" ! -uid "$(id -u app)" -print -quit 2>/dev/null)" ]; then + chown -R app:app "$DATA_DIR" + fi + exec gosu app "$@" +fi + +exec "$@" diff --git a/main.py b/main.py index 0127b270b..98e5c06b6 100644 --- a/main.py +++ b/main.py @@ -20,7 +20,7 @@ refresh_settings, ) from apps.base.models import KeyValue -from apps.base.pages import index, router as pages_router +from apps.base.pages import not_found_handler, router as pages_router from apps.base.setup_wizard import build_setup_page, is_setup_path, setup_response, wants_html_response from apps.base.tasks import ( clean_expired_presign_sessions, @@ -141,7 +141,7 @@ async def refresh_settings_middleware(request, call_next): app.include_router(pages_router) # 404 时返回主题首页（index 兼任 exception handler 与 GET / 路由） -app.add_exception_handler(404, index) +app.add_exception_handler(404, not_found_handler) if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml index e1e48de65..e29ff52e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,8 @@ dev = [ "httpx", "ruff", "pre-commit", + "moto[s3,server]", + "mypy", ] [tool.ruff] @@ -30,3 +32,9 @@ target-version = "py312" # (unused imports, undefined names). Style families (pyupgrade, broad-except, # import sorting) are deferred to keep the initial diff minimal. select = ["E4", "E7", "E9", "F"] + +[tool.mypy] +python_version = "3.12" +# 渐进引入：基线棘轮模式（scripts/mypy-ratchet.py）——只许修复减少、不许新增。 +# 修掉基线里的错误后跑 `--regenerate` 收紧基线。 +ignore_missing_imports = true diff --git a/requirements.lock.txt b/requirements.lock.txt index 6901baabd..3f2e9b439 100644 --- a/requirements.lock.txt +++ b/requirements.lock.txt @@ -1,9 +1,9 @@ # This file was autogenerated by uv via the following command: -# uv pip compile requirements.lock.tmp --generate-hashes --universal -o requirements.lock +# uv pip compile requirements.txt --generate-hashes --universal -o requirements.lock.txt aioboto3==15.5.0 \ --hash=sha256:cc880c4d6a8481dd7e05da89f41c384dbd841454fc1998ae25ca9c39201437a6 \ --hash=sha256:ea8d8787d315594842fbfcf2c4dce3bac2ad61be275bc8584b2ce9a3402a6979 - # via -r requirements.lock.tmp + # via -r requirements.txt aiobotocore==2.25.1 \ --hash=sha256:ea9be739bfd7ece8864f072ec99bb9ed5c7e78ebb2b0b15f29781fbe02daedbc \ --hash=sha256:eb6daebe3cbef5b39a0bb2a97cffbe9c7cb46b2fcc399ad141f369f3c2134b1f @@ -12,134 +12,134 @@ aiofiles==25.1.0 \ --hash=sha256:a8d728f0a29de45dc521f18f07297428d56992a742f0cd2701ba86e44d23d5b2 \ --hash=sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695 # via - # -r requirements.lock.tmp + # -r requirements.txt # aioboto3 aiohappyeyeballs==2.7.1 \ --hash=sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d \ --hash=sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 # via aiohttp -aiohttp==3.14.2 \ - --hash=sha256:03330676d8caa28bb33fa7104b0d542d9aac93350abcd91bf68e64abd531c320 \ - --hash=sha256:052478c7d01035d805302db50c2ef626b1c1ba0fe2f6d4a22ae6eaeb43bf2316 \ - --hash=sha256:09d1b0deec698d1198eb0b8f910dd9432d856985abbfea3f06be8b296a6619b4 \ - --hash=sha256:0baed2a2367a28456b612f4c3fd28bb86b00fadfb6454e706d8f65c21636bfd7 \ - --hash=sha256:0bfea68a48c8071d49aabdf5cd9a6939dcb246db65730e8dc76295fe02f7c73c \ - --hash=sha256:0e56babe35076f69ec9327833b71439eeccd10f51fe56c1a533da8f24923f014 \ - --hash=sha256:0eb1c9fd51f231ac8dc9d5824d5c2efc45337d429db0123fa9d4c20f570fdfc3 \ - --hash=sha256:0fb26fcc5ebf765095fe0c6ab7501574d3108c57fca9a0d462be15a65c9deb8d \ - --hash=sha256:114299c08cce8ad4ebb21fafe766378864109e88ad8cf63cf6acb384ff844a57 \ - --hash=sha256:135570f5b470c72c4988a58986f1f847ad336721f77fcc18fda8472bd3bbe3db \ - --hash=sha256:15292b08ce7dd45e268fce542228894b4735102e8ee77163bd665b35fc2b5598 \ - --hash=sha256:165b0dcc65960ffc9c99aa4ba1c3c76dbc7a34845c3c23a0bd3fbf33b3d12569 \ - --hash=sha256:17eecd6ee9bfc8e31b6003137d74f349f0ac3797111a2df87e23acb4a7a912ea \ - --hash=sha256:18fcc3a5cc7dde1d8f7903e309055294c28894c9434588645817e374f3b83d03 \ - --hash=sha256:1aa4f3b44563a88da4407cef8a13438e9e386967720a826a10a633493f69208f \ - --hash=sha256:1b9251f43d78ff675c0ddfcd53ba61abecc1f74eedc6287bb6657f6c6a033fe7 \ - --hash=sha256:1c05afdd28ecacce5a1f63275a2e3dce09efddd3a63d143ee9799fda83989c8d \ - --hash=sha256:1fc31339824ec922cb7424d624b5b6c11d8942d077b2585e5bd602ca1a1e27ed \ - --hash=sha256:205181d896f73436ac60cf6644e545544c759ab1c3ec8c34cc1e044689611361 \ - --hash=sha256:2280d165ab38355144d9984cdce77ce506cee019a07390bab7fd13682248ce91 \ - --hash=sha256:2a382aa6bb85347515ead043257445baeec0885d42bfedb962093b134c3b4816 \ - --hash=sha256:2d2eedae227cd5cbd0bccc5e759f71e1af2cd77b7f74ce413bb9a2b87f94a272 \ - --hash=sha256:2f1b9540d2d0f2f95590528a1effd0ba5370f6ec189ac925e70b5eecae02dc77 \ - --hash=sha256:2f7ca81d936d820ae479971a6b6214b1b867420b5b58e54a1e7157716a943754 \ - --hash=sha256:30a5ed81f752f182961237414a3cd0af209c0f74f06d66f66f9fcb8964f4978d \ - --hash=sha256:30e41662123806e4590a0440585122ac33c89a2465a8be81cc1b50656ca0e432 \ - --hash=sha256:312d414c294a1e26aa12888e8fd37cd2e1131e9c48ddcf2a4c6b590290d52a49 \ - --hash=sha256:3523ec0cc524a413699f25ec8340f3da368484bc9d5f2a1bf87f233ac20599bf \ - --hash=sha256:386ce4e709b4cc40f9ef9a132ad8e672d2d164a65451305672df656e7794c68e \ - --hash=sha256:3d4238e50a378f5ac69a1e0162715c676bd082dede2e5c4f67ca7fd0014cb09d \ - --hash=sha256:3ec4b6501a076b2f73844256da17d6b7acb15bb74ee0e908a67feb9412371166 \ - --hash=sha256:3f3381f81bc1c6cbe160b2a3708d39d05014329118e6b648b95edc841eeeebd4 \ - --hash=sha256:40bedff39ea83185f3f98a41155dd9da28b365c432e5bd90e7be140bcef0b7f3 \ - --hash=sha256:4181d72e0e6d1735c1fae56381193c6ae211d584d06413980c00775b9b2a176a \ - --hash=sha256:41b5b66b1ac2c48b61e420691eb9741d17d9068f2bc23b5ee3e750faa564bc8f \ - --hash=sha256:42372e1f1a8dca0dcd5daf922849004ec1120042d0e24f14c926f97d2275ca79 \ - --hash=sha256:43387429e4f2ec4047aaf9f935db003d4aa1268ea9021164877fd6b012b6396a \ - --hash=sha256:4610638d3135afaefadf179bffd1bbf3434d3dc7a5d0a4c4219b99fa976e944d \ - --hash=sha256:46b8887aa303075c1e5b24123f314a1a7bbfa03d0213dff8bb70503b2148c853 \ - --hash=sha256:476cf7fac10619ad6d08e1df0225d07b5a8d57c04963a171ad845d5a349d47ef \ - --hash=sha256:483b6f964bbbdaa99a0cd7def631208c44e39d243b95cff23ebc812db8a80e03 \ - --hash=sha256:4ca802547f1128008addfc21b24959f5cbf30a8952d365e7daa078a0d884b242 \ - --hash=sha256:56432ee8f7abe47c97717cfbf5c32430463ea8a7138e12a87b7891fa6084c8ff \ - --hash=sha256:5e94a8c4445bfdaa30773c81f2be7f129673e0f528945e542b8bd024b2979134 \ - --hash=sha256:5fe25c4c44ea5b56fd4512e2065e09384987fc8cc98e41bc8749efe12f653abb \ - --hash=sha256:63b840c03979732ec92e570f0bd6beb6311e2b5d19cacbfcd8cc7f6dd2693900 \ - --hash=sha256:65cd3bb118f42fceceb9e8a615c735a01453d019c673f35c57b420601cc1a83a \ - --hash=sha256:66de80888db2176655f8df0b705b817f5ae3834e6566cc2caa89360871d90195 \ - --hash=sha256:673217cbc9370ebf8cd048b0889d7cbe922b7bb48f4e4c02d31cfefa140bd946 \ - --hash=sha256:68a6f7cd8d2c70869a2a5fe97a16e86a4e13a6ed6f0d9e6029aef7573e344cd6 \ - --hash=sha256:6b63709e259e3b3d7922b235606564e91ed4c224e777cc0ca4cae04f5f559206 \ - --hash=sha256:6bea8451e26cd67645d9b2ee18232e438ddfc36cea35feecb4537f2359fc7030 \ - --hash=sha256:6c244f7a65cbec04c830a301aae443c529d4dbca5fddfd4b19e5a179d896adfd \ - --hash=sha256:6cde463b9dd9ce4343785c5a39127b40fce059ae6fbd320f5a045a38c3d25cd0 \ - --hash=sha256:6e30743bd3ab6ad98e9abbad6ccb39c52bcf6f11f9e3d4b6df97afffe8df53f3 \ - --hash=sha256:70570f50bda5037b416db8fcba595cf808ecf0fdce12d64e850b5ae1db7f64d4 \ - --hash=sha256:71501bc03ede681401269c569e6f9306c761c1c7d4296675e8e78dd07147070f \ - --hash=sha256:7719cef2a9dc5e10cd5f476ec1744b25c5ac4da733a9a687d91c42de7d4afe30 \ - --hash=sha256:7871c94f3400358530ac4906dd7a526c5a24099cd5c48f53ffc4b1cb5037d7d7 \ - --hash=sha256:7ae767b7dffd316cc2d0abf3e1f90132b4c1a2819a32d8bcb1ba749800ea6273 \ - --hash=sha256:7e254b0d636957174a03ca210289e867a62bb9502081e1b44a8c2bb1f6266ecd \ - --hash=sha256:7e328d02fb46b9a8dbfa070d98967e8b7eaa1d9ee10ae03fb664bdf30d58ccf0 \ - --hash=sha256:8241ee6c7fff3ebb1e6b237bccc1d90b46d07c06cf978e9f2ecad43e29dac67a \ - --hash=sha256:82d14d66d6147441b6571833405c828980efc17bda98075a248104ffdd330c30 \ - --hash=sha256:86861a430657bc71e0f89b195de5f8fa495c0b9b5864cf2f89bd5ec1dbb6b77a \ - --hash=sha256:87c9b03be0c18c3b3587be979149830381e37ac4a6ca8557dbe72e44fcad66c3 \ - --hash=sha256:89120e926c68c4e60c78514d76e16fc15689d8df35843b2a6bf6c4cc0d64b11a \ - --hash=sha256:8c2cdb684c153f377157e856257ee8535c75d8478343e4bb1e83ca73bdfa3d31 \ - --hash=sha256:8d1f3802887f0e0dc07387a081dca3ad0b5758e32bdf5fb619b12ac22b8e9b56 \ - --hash=sha256:8f7b19e27b78a3a927b1932af93af7645806153e8f541cee8fe856426142503f \ - --hash=sha256:9094262ae4f2902c7291c14ba915960db5567276690ef9195cdefe8b7cbb3acb \ - --hash=sha256:983a68048a48f35ed08aadfcc1ba55de9a121aa91be48a764965c9ec532b94b5 \ - --hash=sha256:9b937d7864ca68f1e8a1c3a4eb2bac1de86a992f86d36492da10a135a482fab6 \ - --hash=sha256:9d3f4c68b2c2cd282b65e558cebf4b27c8b440ab511f2b938a643d3598df2ddb \ - --hash=sha256:a26f14006883fc7662e21041b4311eac1acbc977a5c43aacb27ff17f8a4c28b2 \ - --hash=sha256:a3177e51e26e0158fb3376aebac97e0546c6f175c510f331f585e514a00a302b \ - --hash=sha256:a57f39d6ec155932853b6b0f130cbbafab3208240fa807f29a2c96ea52b77ae1 \ - --hash=sha256:a6b0ce033d49dd3c6a2566b387e322a9f9029110d67902f0d64571c0fd4b73d8 \ - --hash=sha256:aac1b05fc5e2ef188b6d74cf151e977db75ab281238f30c3163bbd6f797788e3 \ - --hash=sha256:abb33120daba5e5643a757790ece44d638a5a11eb0598312e6e7ec2f1bd1a5a3 \ - --hash=sha256:af63ac06bad85191e6a0c4a733cb3c55adb99f8105bc7ce9913391561159a49a \ - --hash=sha256:b0d49be9d9a210b2c993bf32b1eda03f949f7bcda68fc4f718ae8085ae3fb4b8 \ - --hash=sha256:b155df7f572c73c6c4108b67be302c8639b96ae56fb02787eeae8cad0a1baf26 \ - --hash=sha256:b39dbdbe30a44958d63f3f8baa2af68f24ec8a631dcd18a33dd76dfa2a0eb917 \ - --hash=sha256:b5ed2c7dacebf4950d6b4a1b22548e4d709bb15e0287e064a7cdb32ada65893a \ - --hash=sha256:bc0ed30b942c3bd755583d74bb00b90248c067d20b1f8301e4489a53a33aa65f \ - --hash=sha256:bc1a0793dce8fa9bb6906411e57fb18a2f1c31357b04172541b92b30337362a7 \ - --hash=sha256:bf7951959a8e89f2d4a1e719e60d3ea4e8fc26f011ee3aed09598ad786b112f7 \ - --hash=sha256:c0a968b04fecf7c94e502015860ad1e2e112c6b761e97b6fdf65fbb374e22b73 \ - --hash=sha256:c0c7f2e5fe10910d5ab76438f269cc41bb7e499fd48ded978e926360ab1790c8 \ - --hash=sha256:c167127a3b6089ef78ac2e33582c38040d51688ee28474b5053acf55f192187b \ - --hash=sha256:c8ab295ee58332ef8fbd62727df90540836dfcf7a61f545d0f2771223b80bf25 \ - --hash=sha256:cabaaecb4c6888bd9abafac151051377534dad4c3859a386b6325f39d3732f99 \ - --hash=sha256:cc4435b16dc246c5dfa7f2f8ee71b10a30765018a090ee36e99f356b1e9b75cc \ - --hash=sha256:ce8dfb58f012f76258f29951d38935ac928b32ae24a480f30761f2ed5036fa78 \ - --hash=sha256:ceb77c159b2b4c1a179b96a26af36bcaa68eb79c393ec4f569386a69d013cbe9 \ - --hash=sha256:ceff4f84c1d928654faa6bcb0437ed095b279baae2a35fcfe5a3cbe0d8b9725d \ - --hash=sha256:cf7930e83a12801b2e253d41cc8bf5553f61c0cfabef182a72ae13472cc81803 \ - --hash=sha256:d15f618255fcbe5f54689403aa4c2a90b6f2e6ebc96b295b1cb0e868c1c12384 \ - --hash=sha256:d32a70b8bf8836fd80d4169d9e34eb032cd2a7cbccb0b9cf00eac1f40732467c \ - --hash=sha256:d813f54560b9e5bce170fff7b0adde54d88253928e4add447c36792f27f92125 \ - --hash=sha256:d93854e215dcc7c88e4f530827193c1a594e2662931d8dbe7cca3abf52a7082d \ - --hash=sha256:da4f142fa078fedbdb3f88d0542ad9315656224e167502ae274cbba818b90c90 \ - --hash=sha256:dbc45e2773c66d14fbd337754e9bf23932beef539bd539716a721f5b5f372034 \ - --hash=sha256:dc056948b7a8a40484b4bbc69923fa25cddd80cbc5f236a3a22ad2f836baeed2 \ - --hash=sha256:de3b04a3f7b40ad7f1bcd3540dd447cf9bd93d57a49969bca522cbcf01290f08 \ - --hash=sha256:e3a6302f47518dbf2ffd3cd518f02a1fbf53f85ffeed41a224fa4a6f6a62673b \ - --hash=sha256:e5efff8bfd27c44ce1bfdf92ce838362d9316ed8b2ed2f89f581dbe0bbe05acf \ - --hash=sha256:ec64d1c4605d689ed537ba1e572138e2d4ff603a0cb2bbbfe61d4552c73d19e1 \ - --hash=sha256:ecdd6b8cab5b7c0ff2988378c11ba7192f076a1864e64dc3ff72f7ba05c71796 \ - --hash=sha256:ee5bdd7933c653e43ef8d720704a4e228e4927121f2f5f598b7efe6a4c18633a \ - --hash=sha256:ef710fbb770aefa4def5484eeddb606e70ab3492aa37390def61b35652f6820a \ - --hash=sha256:f2f9950b2dd0fc896ab520ea2366b7df6484d3d164a65d5e9f28f7b0e5742d8a \ - --hash=sha256:f518d75c03cd3f7f125eca1baadb56f8b94db94602278d2d0d19af6e177650a7 \ - --hash=sha256:f7c10c4d0b33888a68c192d883d1390d4596c116a59bf689e6d352c6739b7940 \ - --hash=sha256:f8f371794319a8185e61e15ba5e1be8407b986ebce1ade11856c02d24e090577 \ - --hash=sha256:f96821eb2ae2f12b0dfa799eafbf221f5621a9220b457b4744a269a63a5f3a6c \ - --hash=sha256:fc2d8e7373ceba7e1c7e9dc00adac854c2701a6d443fd21d4af2e49342d727bd \ - --hash=sha256:fef094bfc2f4e991a998af066fc6e3956a409ef799f5cbad2365175357181f2e +aiohttp==3.14.3 \ + --hash=sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39 \ + --hash=sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043 \ + --hash=sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b \ + --hash=sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d \ + --hash=sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf \ + --hash=sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f \ + --hash=sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7 \ + --hash=sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc \ + --hash=sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559 \ + --hash=sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f \ + --hash=sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929 \ + --hash=sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147 \ + --hash=sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9 \ + --hash=sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8 \ + --hash=sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf \ + --hash=sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7 \ + --hash=sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8 \ + --hash=sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85 \ + --hash=sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30 \ + --hash=sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553 \ + --hash=sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7 \ + --hash=sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86 \ + --hash=sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e \ + --hash=sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a \ + --hash=sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c \ + --hash=sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da \ + --hash=sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5 \ + --hash=sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d \ + --hash=sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100 \ + --hash=sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71 \ + --hash=sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22 \ + --hash=sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1 \ + --hash=sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479 \ + --hash=sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb \ + --hash=sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062 \ + --hash=sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661 \ + --hash=sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427 \ + --hash=sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32 \ + --hash=sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a \ + --hash=sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db \ + --hash=sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42 \ + --hash=sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a \ + --hash=sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd \ + --hash=sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06 \ + --hash=sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8 \ + --hash=sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228 \ + --hash=sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0 \ + --hash=sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919 \ + --hash=sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee \ + --hash=sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2 \ + --hash=sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f \ + --hash=sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43 \ + --hash=sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098 \ + --hash=sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c \ + --hash=sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371 \ + --hash=sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b \ + --hash=sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0 \ + --hash=sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f \ + --hash=sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100 \ + --hash=sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529 \ + --hash=sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc \ + --hash=sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c \ + --hash=sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41 \ + --hash=sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716 \ + --hash=sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33 \ + --hash=sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f \ + --hash=sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e \ + --hash=sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa \ + --hash=sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b \ + --hash=sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80 \ + --hash=sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646 \ + --hash=sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e \ + --hash=sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b \ + --hash=sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c \ + --hash=sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963 \ + --hash=sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae \ + --hash=sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25 \ + --hash=sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c \ + --hash=sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f \ + --hash=sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807 \ + --hash=sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a \ + --hash=sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f \ + --hash=sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d \ + --hash=sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82 \ + --hash=sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15 \ + --hash=sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0 \ + --hash=sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d \ + --hash=sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9 \ + --hash=sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19 \ + --hash=sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239 \ + --hash=sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0 \ + --hash=sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c \ + --hash=sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5 \ + --hash=sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b \ + --hash=sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4 \ + --hash=sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2 \ + --hash=sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9 \ + --hash=sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0 \ + --hash=sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883 \ + --hash=sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d \ + --hash=sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d \ + --hash=sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6 \ + --hash=sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3 \ + --hash=sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924 \ + --hash=sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde \ + --hash=sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787 \ + --hash=sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb \ + --hash=sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b \ + --hash=sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0 \ + --hash=sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910 \ + --hash=sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9 \ + --hash=sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627 \ + --hash=sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48 \ + --hash=sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b \ + --hash=sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce \ + --hash=sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a \ + --hash=sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0 \ + --hash=sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24 \ + --hash=sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5 # via - # -r requirements.lock.tmp + # -r requirements.txt # aiobotocore aioitertools==0.13.0 \ --hash=sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be \ @@ -186,10 +186,10 @@ click==8.5.0 \ --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 # via uvicorn -fastapi==0.139.2 \ - --hash=sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e \ - --hash=sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c - # via -r requirements.lock.tmp +fastapi==0.141.1 \ + --hash=sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3 \ + --hash=sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1 + # via -r requirements.txt frozenlist==1.8.0 \ --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \ --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \ @@ -646,134 +646,133 @@ propcache==0.5.2 \ # via # aiohttp # yarl -pydantic==2.12.5 \ - --hash=sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49 \ - --hash=sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d +pydantic==2.13.5 \ + --hash=sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73 \ + --hash=sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08 # via - # -r requirements.lock.tmp + # -r requirements.txt # fastapi -pydantic-core==2.41.5 \ - --hash=sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90 \ - --hash=sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740 \ - --hash=sha256:0384e2e1021894b1ff5a786dbf94771e2986ebe2869533874d7e43bc79c6f504 \ - --hash=sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84 \ - --hash=sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33 \ - --hash=sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c \ - --hash=sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0 \ - --hash=sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e \ - --hash=sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0 \ - --hash=sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a \ - --hash=sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34 \ - --hash=sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2 \ - --hash=sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3 \ - --hash=sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815 \ - --hash=sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14 \ - --hash=sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba \ - --hash=sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375 \ - --hash=sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf \ - --hash=sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963 \ - --hash=sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1 \ - --hash=sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808 \ - --hash=sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553 \ - --hash=sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1 \ - --hash=sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2 \ - --hash=sha256:299e0a22e7ae2b85c1a57f104538b2656e8ab1873511fd718a1c1c6f149b77b5 \ - --hash=sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470 \ - --hash=sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2 \ - --hash=sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b \ - --hash=sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660 \ - --hash=sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c \ - --hash=sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093 \ - --hash=sha256:346285d28e4c8017da95144c7f3acd42740d637ff41946af5ce6e5e420502dd5 \ - --hash=sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594 \ - --hash=sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008 \ - --hash=sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a \ - --hash=sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a \ - --hash=sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd \ - --hash=sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284 \ - --hash=sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586 \ - --hash=sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869 \ - --hash=sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294 \ - --hash=sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f \ - --hash=sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66 \ - --hash=sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51 \ - --hash=sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc \ - --hash=sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97 \ - --hash=sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a \ - --hash=sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d \ - --hash=sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9 \ - --hash=sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c \ - --hash=sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07 \ - --hash=sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36 \ - --hash=sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e \ - --hash=sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05 \ - --hash=sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e \ - --hash=sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941 \ - --hash=sha256:707625ef0983fcfb461acfaf14de2067c5942c6bb0f3b4c99158bed6fedd3cf3 \ - --hash=sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612 \ - --hash=sha256:753e230374206729bf0a807954bcc6c150d3743928a73faffee51ac6557a03c3 \ - --hash=sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b \ - --hash=sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe \ - --hash=sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146 \ - --hash=sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11 \ - --hash=sha256:7b93a4d08587e2b7e7882de461e82b6ed76d9026ce91ca7915e740ecc7855f60 \ - --hash=sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd \ - --hash=sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b \ - --hash=sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c \ - --hash=sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a \ - --hash=sha256:873e0d5b4fb9b89ef7c2d2a963ea7d02879d9da0da8d9d4933dee8ee86a8b460 \ - --hash=sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1 \ - --hash=sha256:8bfeaf8735be79f225f3fefab7f941c712aaca36f1128c9d7e2352ee1aa87bdf \ - --hash=sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf \ - --hash=sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858 \ - --hash=sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2 \ - --hash=sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9 \ - --hash=sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2 \ - --hash=sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3 \ - --hash=sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6 \ - --hash=sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770 \ - --hash=sha256:a75dafbf87d6276ddc5b2bf6fae5254e3d0876b626eb24969a574fff9149ee5d \ - --hash=sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc \ - --hash=sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23 \ - --hash=sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26 \ - --hash=sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa \ - --hash=sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8 \ - --hash=sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d \ - --hash=sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3 \ - --hash=sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d \ - --hash=sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034 \ - --hash=sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9 \ - --hash=sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1 \ - --hash=sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56 \ - --hash=sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b \ - --hash=sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c \ - --hash=sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a \ - --hash=sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e \ - --hash=sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9 \ - --hash=sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5 \ - --hash=sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a \ - --hash=sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556 \ - --hash=sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e \ - --hash=sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49 \ - --hash=sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2 \ - --hash=sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9 \ - --hash=sha256:e4f4a984405e91527a0d62649ee21138f8e3d0ef103be488c1dc11a80d7f184b \ - --hash=sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc \ - --hash=sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb \ - --hash=sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0 \ - --hash=sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8 \ - --hash=sha256:e8465ab91a4bd96d36dde3263f06caa6a8a6019e4113f24dc753d79a8b3a3f82 \ - --hash=sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69 \ - --hash=sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b \ - --hash=sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c \ - --hash=sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75 \ - --hash=sha256:f0cd744688278965817fd0839c4a4116add48d23890d468bc436f78beb28abf5 \ - --hash=sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f \ - --hash=sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad \ - --hash=sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b \ - --hash=sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7 \ - --hash=sha256:f41eb9797986d6ebac5e8edff36d5cef9de40def462311b3eb3eeded1431e425 \ - --hash=sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52 +pydantic-core==2.46.5 \ + --hash=sha256:013d6f3483d81e02e7c328831808f336c8596ee33b4bd4026b9ffb1e960b8942 \ + --hash=sha256:03b9666e41e35d8909852ba191a0607520f81b74eaf12ccf8737005dbb313821 \ + --hash=sha256:045ab3b6d308439e32b81cc173bba5b9018bc6ed896afd0c65b3b009b1699af5 \ + --hash=sha256:0bddb4020d8f04175865ccd17eff3040874fc11fb593f424edb452653b4b947c \ + --hash=sha256:0cdbada856a1c69a7624a64d3d9aefe79300bd6ef827b43a4f265010b9b55184 \ + --hash=sha256:0fc5be0abd4a407e200d844b404e33639a554e7bd0d448e7b9ae181be4789ac2 \ + --hash=sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc \ + --hash=sha256:15f4a94963c95accac15b7b657bb177d3ad82bb90b0d0526d9a9b85079925db5 \ + --hash=sha256:18a09e1e1011b462f2e32774f25859ef1223d5c2b0546a633cf56654710721e0 \ + --hash=sha256:193375f3548919d3f0b60936ca113ada3e38f264f91b9b8e0508efaad57be931 \ + --hash=sha256:1a353f84de772f423b5ffb11d7ae352fbbef0f446f3c0b0af0f8236d7233606e \ + --hash=sha256:1e449def1945a462c464331254e5a44fca7c3b4f9aedf59ec2f50f8066dd8e25 \ + --hash=sha256:1e5aad1220a1192c42341c8fd4a8686657e73ab2a920c970bdc4de334fe3193d \ + --hash=sha256:200aa3dc9f8d54f0754f43247c0bad0999fdcfbfd2488384dd44f37279271fe6 \ + --hash=sha256:2471fd51c61c610e1dcf7de44d7299283661654d11264ab4802b303368d69c47 \ + --hash=sha256:24922243639cbdac66c75fcb6fd6495a9cb52b213d62f9a0d16f0310b1ff8038 \ + --hash=sha256:28a6a556cd3b6066bea827857f9d9cce027c96f776e512f544a581f9e42161f8 \ + --hash=sha256:2bc9419666990c06d7397831f2126a1ecc3594aaa3ff7de5bf2d066802f4e07b \ + --hash=sha256:2cbd9a5eff05e51c447c34dfa4632145b26b09120cf04bd0c871e44c1a5e1c9a \ + --hash=sha256:2d330aaba8621b1edcec8ae2c4050f63b84ccf6d98723a8f212e9684713abf0e \ + --hash=sha256:2d5d76654becf5efd62c9e51c3756c67b49498b0c9a40884934c40807adbd074 \ + --hash=sha256:337639ba62a11acde6ef3aeb08c8ea755f8ef1fe5e513356c0f36a2b0d7568b0 \ + --hash=sha256:347ec774390c87326a2e4929d58d3f7e8763a104d5d35f4cd595a4c952366433 \ + --hash=sha256:356c8368cbc321050b169595683a2e1d63413b1e0e2868b330af9fc14c616d3f \ + --hash=sha256:37ae34309d7bd8c0d61ab839668058f2a7962ea1fc51d105d2db228fe0618034 \ + --hash=sha256:37ea7b83c935e5b0d68c9449b82651accf78a10828b2c02b2f2d9e9496446c21 \ + --hash=sha256:3a3e26b6a8274211bddee2d0e4d0d42778f17a34510f49d2ec44b58abfc41736 \ + --hash=sha256:3aa166e99c4f2985407fb8714aebede877ecb5455cf321b606adca926d30d5a0 \ + --hash=sha256:3d2652072b2d774947ba5cf78a9e59644ac62ee572daf6dd2e1dfe905e15b2b7 \ + --hash=sha256:40375c2d05acec10323e45dfe2077ac44bc74659008614af5069034e2cfc781c \ + --hash=sha256:413a717a410d0c817ef5b786a059415550b3794e1d0c2abffd9efb93a3d9f7b4 \ + --hash=sha256:46c25dda9d092a06c08db76ffe0a197107904d0dfac653f7d5306bbcd6d6119c \ + --hash=sha256:49776eab08766a08dfff7012f8b422dcd7e25e43b316eedf0477c24fcfa84b7c \ + --hash=sha256:4d44cf99ddebf875f9b68cc267aa684c99b7b44fe63ee1cac4ec163807290069 \ + --hash=sha256:4dedce55295becb61921e386b99d4f2706045306e7fa52249a33004c837379fb \ + --hash=sha256:4f8507560a9284e1370bb048ed4282012fbef4e8d109875b95e884d228552061 \ + --hash=sha256:4fdc8b93a41521988916eeaa271173fcca7fa0803d62f87675aac8dcec1c8e29 \ + --hash=sha256:5086029a57366b8cf81b130a43908738095c270c21a8d7f0e8bdfdb89718e2f3 \ + --hash=sha256:52e24eacdb536cade636aa90fb851835222becff8484b7001fdc78cb0290f2aa \ + --hash=sha256:53feb344243bb9510a9dec7bf3cf1b64d88a98af5dc7872a5160465f8b198c8e \ + --hash=sha256:545f26c504b27c3758439a5e6d9349931f0a04f855668d5fe323c89e82300a38 \ + --hash=sha256:54d510bac3ee52247af28ed4bb18a1e799f040ac60fd2bf5ccd4c92f1fbe786f \ + --hash=sha256:5cb482e9e84c851f4e623fe4acc1ced89168cf1fe18f7089db4548c8f5bbb65b \ + --hash=sha256:5e81740c09e310f5aa5cbd3e434a01c154d4bef93241c7877b39f211d2b78ba8 \ + --hash=sha256:5ee239d575f80b08eca11f6e20f90c4c695de7825c67eefe6091fbf20dda648e \ + --hash=sha256:5f194189415698233dd1114a093a9b56e61e2c57e11b469be3b0506f46f0771c \ + --hash=sha256:5f93c5fe914d75fbec9a49209b00da5f08e9e467d69da2b1510c81940cfd10be \ + --hash=sha256:657b40d6240c0a7b6a64b30f22d1e3aa631c7e846c621b0c0f6d1d75e2e15ea6 \ + --hash=sha256:6d30e1a4f138b8951063e9a394752a9179b51da288ffa507b1e659222f4c1793 \ + --hash=sha256:6f7b393a8b3da82f5c1fc0751e6d01ac6c55b93c18226a60bdfba4a724efafd1 \ + --hash=sha256:701b2e04b560eeb4bddf7a25ab8ca476176e34fdbd9a0e18196f0d12d4685f0b \ + --hash=sha256:771cf63ae0b1b50dd22e5f3e3549fab5f3f4ff1635d352a9e1a97fe01c7b2e64 \ + --hash=sha256:79bdfa52f843137045b2d081cc05c120ba6665d29b7559c2c47690906f39279f \ + --hash=sha256:7ac031912d54f3d83ef3b3eb98dfabc1608802e2202263d25957eeed40b94761 \ + --hash=sha256:7b0fc826b16c55e561e5d2a0c5c77b051ba1d92808118c4e4b5390f5e0cf191d \ + --hash=sha256:7c6be839a5a8312626b32029a415644a0846b420bc8b52b95b28cd92da162168 \ + --hash=sha256:816ff0a6550ffc06c098ccd2e0698600f9aa7da192a79eaa6f9af504a35db869 \ + --hash=sha256:82a36973cf8a2ef5406f4fe2edbf8ed0c99629535d959e0b100c76a32535a111 \ + --hash=sha256:837b396ca3d7b74091ca623f6cbd8351bd42d670a79c2683e79fb089f06a2de5 \ + --hash=sha256:850a08d167dde16db8702c274f320c7be9d7da6f6dff2b58b18f9e815bd94f5b \ + --hash=sha256:8816f3d218beb4b787de5c9759c259b8fa61f9dec42dc7811f320a33771778b7 \ + --hash=sha256:892a881d5f68c2b9ea304b7a6c2c60d9343df578a311b0f86b94bc8f1ffe8129 \ + --hash=sha256:895395f8918627b04efb1ad2a4cf605387143300ba03304cd1dfa6d03f5e095e \ + --hash=sha256:8b10e3e8fd7ddc2bd915848a2768e44c15b22936f1cc54c462ad1164deb02655 \ + --hash=sha256:8e24d8f05fa2d28513d94e877e9c75ad66175376209b3977f916e240e623193c \ + --hash=sha256:8feeac04b5794e513e710af2f9c87d49f31a6dc47967bb264a1fed61a8989bec \ + --hash=sha256:9432f3598db432cb51c5b37fdbf29a60fcccc79e30d37a05022776a6bc4ab689 \ + --hash=sha256:976e1128455aa595ea04c79ccfedff1aaeab96ee013fcc916bed120c4f0ad94f \ + --hash=sha256:978e7b97d4824b5be09c69fb70507cbde3b0323fc147332ca40a94d9a6a0ebbf \ + --hash=sha256:97bf8de4d541598c94a59344eeb988a94c08ff76b5723c41f6567ec18c7892ea \ + --hash=sha256:97cf3eb53a8cccacf9d46686a0926186c9bfb5574f2ed66d3639d5fe117cd3a9 \ + --hash=sha256:9b68938dd5b0c783d88ff8e2dcc69451b5eb936fe212d516b21b9d5567f6d464 \ + --hash=sha256:9c4b71f10dd532fb7a5cbc8f58707779e64f03a258c2bf8bfbaecfcd9970b519 \ + --hash=sha256:9f47b8a949e60f027f0aa0a6f6c7b7e9c55cbf4380d10b344e282fa4e7ab1e1b \ + --hash=sha256:a1dee1b804ff4d11c663636cf15d2ea47e9f79cd56c033fb1cbf08924842a48f \ + --hash=sha256:a2468d93d181667a7abd66e1b64bb9f76f361b0fef8faddf687456453576f5ee \ + --hash=sha256:a2a5e1d0ff29adddc9f6d6821a66302e4493f8ca898b715b6b1182c2c201ea0a \ + --hash=sha256:a39ac25a9a2fa4072efdb429833c4a4c8009a51ff9eea3eeae131713cd27991e \ + --hash=sha256:a445486499897b88a7d6c310c88ed64dd37b1b59bfd7ae9107490bbb362f47d6 \ + --hash=sha256:a91c17edf6eea2402cb5457b4c89e99bc5ed1004aa34c4adf1d4258c1a5c22c2 \ + --hash=sha256:ab4b66edffb32d9e951efb3814bd104b8367a7501b81b955cacb5726d897389f \ + --hash=sha256:aca6c767f552b21b10f774aeac128e828eafb796adfa1b666a18bf6321453c3a \ + --hash=sha256:acf8a67ba51f4ca9ddbd0e6b3000a65ac51ab734661778b3e7ba64d99a710f2f \ + --hash=sha256:b10ec717381bdbfafef34607824db4c91de69ff085e4fca3b2af91b4fa17e68a \ + --hash=sha256:b49924c73a235e969511bf2aabdff3beebf9820931f646c80274d5d780010c47 \ + --hash=sha256:b6acfb46a814762367fb7ba0828b0a17d441b92ce249a0e007474c9072662dda \ + --hash=sha256:b7ca9034437b6022f941f4857459562ee00a560b97e7cce8a0ec5a74fc6766e0 \ + --hash=sha256:b98134087d9de723658d17a42c7d0da8d6e2ef08015dee7dc93889047315f5e4 \ + --hash=sha256:b9fe6fb92520e3fd61f2e49000b6911b188824f089b75973ea06d6267f0b476d \ + --hash=sha256:bce57638e08ac148e5778cce7feb968307a727d66f8e2274a543d0cf0c9ad6a3 \ + --hash=sha256:c14ad3bdc85ee7f318742c457ca3968a92126d144b15721c759033bfb06296c2 \ + --hash=sha256:c1c43ad4339643d70ebb8124e1305a7dab423001eff58bb41a0f731adbc98355 \ + --hash=sha256:c3471e5c4a949c26ec00a77f01df59096aa9495877de76fd60a980f8ee6be461 \ + --hash=sha256:c583b927a8838dab890706a6fa7573fbb8b70e24000ef9f7238e2d6f6435a5ed \ + --hash=sha256:c76fe65e607be28c7fd4d56fc3c42b1583aa058ce3408b7ad0fd540171d31f9f \ + --hash=sha256:c7ea57fc63aa7da93a1bd2d644e6577befae10c52c4e36377635eea1056a74f5 \ + --hash=sha256:cd5214352ae68f3b5e9af7768bdc5253695ee069675db3480518420b3be881f2 \ + --hash=sha256:cdbb78909f52b981d3b2d56b97328d71eb0b974c36bd77c920123a7ebb192829 \ + --hash=sha256:cdc8b74ecc48c0cb1e9607a05ec4e9e88db60a19ffcc9a1d5f9088ede40c8dc0 \ + --hash=sha256:d0a24b40877af2de4950252be9d21eaf7fb07660f3c2cae1f56c6b599ada5266 \ + --hash=sha256:d22a945598fb91236b4dd793a6e42e4f3dd7740bb5aace5ebd7d4c08d13bb575 \ + --hash=sha256:d2f9fc07a8042a8f95925b35c4f04f469707c981fc33245b6ca187cf5d2dd290 \ + --hash=sha256:d625a186a65201c23a9e3b8ed9c47e90a026e03256608cc91851c6709096844f \ + --hash=sha256:d925f3d9afd05a8c0fb3a1031463a8d59ebe5e2afad297e29c78be19e13b4e62 \ + --hash=sha256:e64e88d5585bea9ce95861079de72006c7fa6d3df4e3a3b65ba31eb979c15c9f \ + --hash=sha256:e652ab17569c94bff5475520f907b7148b8c24036a8ebbe5cf7cf7493d28579a \ + --hash=sha256:e7b891faeedeafba41b2983e5001a81b6a915b69544c7e7570d1989ce1c36ac7 \ + --hash=sha256:e80675d75ae2cd14372cb65cad5400d9347a3d3f6c13000183f22dfd027283ed \ + --hash=sha256:e9c134bb666dd54b778b9fc0d2b50cbb7f979b9e3716f26a88c9ab3b6fc1dd0f \ + --hash=sha256:eb7d8d0e5886a89a55d2eef490e272fa965a9d57c6b29a5b5088a7997ec2cad1 \ + --hash=sha256:ecb42011e12ee19cafbc312887cbf3546959fe02fbad44f272d4be5baa997615 \ + --hash=sha256:ef3fbbf161dc9351a2fe0422e51b129f9e97e42385bd0320b309c15f7d287dd8 \ + --hash=sha256:efd62a42486f1bda5d24cb4f63d15a3c7768375fe83d36f9417b4ad7a2fb20b3 \ + --hash=sha256:f077d0b97ab11fa7dcc633fca53515f290bca8a8a633e966d5b6d1879d9ed01a \ + --hash=sha256:f332f0e72a5a0400141f830744e141bf9f97917878dbe968669e8a7fefea78ff \ + --hash=sha256:f7b0ec93a2893de856652154d73b7ba622f26fa97726487dcac373de5f4c6084 \ + --hash=sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0 \ + --hash=sha256:fc5d783bd4a2387e97b8a2d5ec781cfb92b3d893bf82370548e99db5915935d3 \ + --hash=sha256:fc8515076c11f3cfdf4fb142dcca0fe384b1230a3b5415458ac84f3e0903ec13 \ + --hash=sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9 # via pydantic pypika-tortoise==0.6.5 \ --hash=sha256:64d96c9b88450f6360ad22a7063933b6a90961a7317f04b2b63c98fd5d705506 \ @@ -788,7 +787,7 @@ python-dateutil==2.9.0.post0 \ python-multipart==0.0.32 \ --hash=sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e \ --hash=sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23 - # via -r requirements.lock.tmp + # via -r requirements.txt pytz==2026.3.post1 \ --hash=sha256:2211d3fcf9a797d3405cac96ac7f61d80e6a644f72a3309607282fe8a2010c5d \ --hash=sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815 @@ -805,12 +804,12 @@ starlette==1.6.0 \ --hash=sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c \ --hash=sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b # via - # -r requirements.lock.tmp + # -r requirements.txt # fastapi tortoise-orm==0.25.3 \ --hash=sha256:3c52a53c41f4137aee9ffb3f3de01f30b52ad7767157f9bd9586a910fe839ca3 \ --hash=sha256:b6dedd388393624628ec46228c93df361533ceb3925986fa2d1d22debc838a7d - # via -r requirements.lock.tmp + # via -r requirements.txt typing-extensions==4.16.0 \ --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 @@ -833,10 +832,10 @@ urllib3==2.7.0 \ --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 # via botocore -uvicorn==0.51.0 \ - --hash=sha256:5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b \ - --hash=sha256:f6f4b69b657c312f516dd2d268ab9ae6f254b11e4bac504f37b2ab58b24dd0b0 - # via -r requirements.lock.tmp +uvicorn==0.53.0 \ + --hash=sha256:a9356f0cb89b3b8621529c5d5eebd69bfe154f4c3f68b4cf2de47e45fa855c2e \ + --hash=sha256:e8dca71ec86dce5f04e333f0d56cdedf942446e6643b9cea1af0d6d3a02cb03e + # via -r requirements.txt wrapt==1.17.3 \ --hash=sha256:02b551d101f31694fc785e58e0720ef7d9a10c4e62c1c9358ce6f63f23e30a56 \ --hash=sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828 \ diff --git a/requirements.txt b/requirements.txt index 8a1227f2b..5b02b2951 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ aioboto3==15.5.0 -aiohttp==3.14.2 +aiohttp==3.14.3 aiofiles==25.1.0 -fastapi==0.139.2 +fastapi==0.141.1 starlette==1.6.0 -pydantic==2.12.5 -uvicorn==0.51.0 +pydantic==2.13.5 +uvicorn==0.53.0 tortoise-orm==0.25.3 python-multipart==0.0.32 diff --git a/scripts/mypy-baseline.txt b/scripts/mypy-baseline.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/scripts/mypy-baseline.txt @@ -0,0 +1 @@ + diff --git a/scripts/mypy_ratchet.py b/scripts/mypy_ratchet.py new file mode 100644 index 000000000..b96e470c1 --- /dev/null +++ b/scripts/mypy_ratchet.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Mypy baseline ratchet: new type errors fail the build; fixes shrink the baseline. + +Usage: + python scripts/mypy_ratchet.py # compare against baseline (CI mode) + python scripts/mypy_ratchet.py --regenerate # rewrite the baseline (after fixing errors) +""" +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +BASELINE = ROOT / "scripts" / "mypy-baseline.txt" +TARGETS = ["core", "apps", "main.py"] + + +def run_mypy() -> set[str]: + """Raw error lines (file:line: error: ...).""" + proc = subprocess.run( + [sys.executable, "-m", "mypy", *TARGETS], + cwd=ROOT, + capture_output=True, + text=True, + ) + return { + line.strip() + for line in (proc.stdout + proc.stderr).splitlines() + if ": error:" in line + } + + +_LINE_NO = re.compile(r"^([^:]+):\d+: ") + + +def normalize(errors: set[str]) -> set[str]: + """Strip line numbers so edits elsewhere in a file don't shift the baseline.""" + return {_LINE_NO.sub(r"\1: ", e) for e in errors} + + +def main() -> int: + current = normalize(run_mypy()) + if "--regenerate" in sys.argv: + BASELINE.write_text("\n".join(sorted(current)) + "\n", encoding="utf-8") + print(f"baseline regenerated: {len(current)} errors") + return 0 + baseline = set(BASELINE.read_text(encoding="utf-8").splitlines()) - {""} + new_errors = current - baseline + fixed = baseline - current + if fixed: + print(f"{len(fixed)} baseline errors fixed — regenerate with --regenerate to tighten:") + for e in sorted(fixed): + print(f" FIXED: {e}") + if new_errors: + print(f"FAIL: {len(new_errors)} NEW mypy error(s) not in baseline:") + for e in sorted(new_errors): + print(f" NEW: {e}") + return 1 + print(f"mypy ratchet OK: {len(current)} known errors, 0 new, {len(fixed)} fixed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_admin_read_paths.py b/tests/test_admin_read_paths.py new file mode 100644 index 000000000..9b722d445 --- /dev/null +++ b/tests/test_admin_read_paths.py @@ -0,0 +1,259 @@ +"""Admin read-path coverage: detail, metadata, preview, admin download, +activities filtering, local file lists/delete, and verify. + +These endpoints previously had zero automated coverage. Negative paths +first: missing records, truncation limits, wrong-type rejections. +""" + +import pytest + +from tests.test_admin_write_paths import _create_share, _login + + +@pytest.mark.asyncio +class TestFileDetail: + async def test_detail_returns_policy_storage_metadata(self, initialized_client): + token = await _login(initialized_client) + file_id = await _create_share("dt-exist") + response = await initialized_client.get( + "/admin/file/detail", + params={"id": file_id}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200, response.text + detail = response.json()["detail"] + assert detail["code"] == "dt-exist" + assert detail["is_text"] is True + assert {"policy", "storage", "metadata", "timeline"} <= set(detail.keys()) + + async def test_detail_missing_file_404(self, initialized_client): + token = await _login(initialized_client) + for method, kwargs in ( + ("get", {"params": {"id": 987658}}), + ("post", {"json": {"id": 987658}}), + ): + response = await initialized_client.request( + method, + "/admin/file/detail", + headers={"Authorization": f"Bearer {token}"}, + **kwargs, + ) + assert response.status_code == 404, method + + +@pytest.mark.asyncio +class TestFileMetadata: + async def test_note_and_tags_roundtrip_with_truncation(self, initialized_client): + token = await _login(initialized_client) + file_id = await _create_share("md-round") + + # note 超长被截断到 2000；tags 超 12 个被截、单个超 24 字符被截、去重 + long_note = "n" * 3000 + tags = [f"tag{i}" for i in range(15)] + ["x" * 30, "dup", "dup"] + response = await initialized_client.post( + "/admin/file/metadata", + json={"id": file_id, "note": long_note, "tags": tags}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200, response.text + + detail = ( + await initialized_client.get( + "/admin/file/detail", + params={"id": file_id}, + headers={"Authorization": f"Bearer {token}"}, + ) + ).json()["detail"] + metadata = detail["metadata"] + assert len(metadata["note"]) == 2000 + assert len(metadata["tags"]) == 12 + assert all(len(tag) <= 24 for tag in metadata["tags"]) + assert metadata["updated_at"] is not None + + async def test_metadata_missing_file_404(self, initialized_client): + token = await _login(initialized_client) + response = await initialized_client.post( + "/admin/file/metadata", + json={"id": 987659, "note": "x"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 404 + + +@pytest.mark.asyncio +class TestFilePreview: + async def test_text_preview_truncates_with_max_chars(self, initialized_client): + token = await _login(initialized_client) + file_id = await _create_share("pv-trunc", text="abcdefgh" * 10) + response = await initialized_client.get( + "/admin/file/preview", + params={"id": file_id, "max_chars": 10}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200, response.text + detail = response.json()["detail"] + assert detail["content"] == "abcdefghab" + assert detail["truncated"] is True + assert detail["preview_length"] == 10 + assert detail["max_chars"] == 10 + + async def test_preview_non_text_rejected_400(self, initialized_client): + token = await _login(initialized_client) + from apps.base.models import FileCodes + + file_id = ( + await FileCodes.create( + code="pv-file", prefix="doc", suffix=".bin", size=1 + ) + ).id + response = await initialized_client.get( + "/admin/file/preview", + params={"id": file_id}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 400 + + async def test_preview_missing_file_404(self, initialized_client): + token = await _login(initialized_client) + response = await initialized_client.get( + "/admin/file/preview", + params={"id": 987660}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 404 + + +@pytest.mark.asyncio +class TestAdminDownload: + async def test_text_share_download_returns_content(self, initialized_client): + token = await _login(initialized_client) + file_id = await _create_share("dl-text", text="admin-download-body") + response = await initialized_client.get( + "/admin/file/download", + params={"id": file_id}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200 + assert b"admin-download-body" in response.content + + async def test_missing_file_download_404(self, initialized_client): + token = await _login(initialized_client) + response = await initialized_client.get( + "/admin/file/download", + params={"id": 987661}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 404 + + +@pytest.mark.asyncio +class TestActivities: + async def test_recorded_action_is_listed_and_filtered(self, initialized_client): + """删一个文件产生活动事件；action 过滤必须生效。""" + token = await _login(initialized_client) + file_id = await _create_share("act-del") + await initialized_client.request( + "DELETE", + "/admin/file/delete", + json={"id": file_id}, + headers={"Authorization": f"Bearer {token}"}, + ) + + listed = await initialized_client.get( + "/admin/activities", + params={"action": "file.delete"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert listed.status_code == 200 + activities = listed.json()["detail"]["activities"] + assert any( + a["action"] == "file.delete" and a["target_id"] == file_id + for a in activities + ) + + other = await initialized_client.get( + "/admin/activities", + params={"action": "nonexistent.action"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert other.json()["detail"]["activities"] == [] + + async def test_limit_upper_bound_clamped_to_80(self, initialized_client): + token = await _login(initialized_client) + response = await initialized_client.get( + "/admin/activities", + params={"limit": 5000}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200 + assert response.json()["detail"]["limit"] == 80 + + +@pytest.mark.asyncio +class TestLocalFileEndpoints: + async def test_lists_and_delete_roundtrip(self, initialized_client): + """data/local 列表与删除（走端点层；存储层语义由 WebDAV 轮覆盖）。""" + import shutil + from core.settings import data_root + + token = await _login(initialized_client) + local_dir = data_root / "local" / "e2e-probe" + local_dir.mkdir(parents=True, exist_ok=True) + try: + (local_dir / "probe.txt").write_bytes(b"local-probe") + + listed = await initialized_client.get( + "/admin/local/lists", + params={"path": "e2e-probe"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert listed.status_code == 200, listed.text + items = listed.json()["detail"]["items"] + assert any(item["name"] == "probe.txt" for item in items) + + deleted = await initialized_client.request( + "DELETE", + "/admin/local/delete", + json={"filename": "e2e-probe/probe.txt"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert deleted.status_code == 200 + assert not (local_dir / "probe.txt").exists() + finally: + shutil.rmtree(data_root / "local" / "e2e-probe", ignore_errors=True) + + async def test_local_delete_traversal_rejected(self, initialized_client): + token = await _login(initialized_client) + response = await initialized_client.request( + "DELETE", + "/admin/local/delete", + json={"filename": "../../filecodebox.db"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 400 + + async def test_local_delete_missing_404(self, initialized_client): + token = await _login(initialized_client) + response = await initialized_client.request( + "DELETE", + "/admin/local/delete", + json={"filename": "ghost.txt"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 404 + + +@pytest.mark.asyncio +class TestVerify: + async def test_valid_token_passes(self, initialized_client): + token = await _login(initialized_client) + response = await initialized_client.get( + "/admin/verify", headers={"Authorization": f"Bearer {token}"} + ) + assert response.status_code == 200 + + async def test_garbage_token_rejected_401(self, initialized_client): + response = await initialized_client.get( + "/admin/verify", headers={"Authorization": "Bearer garbage.token.here"} + ) + assert response.status_code == 401 diff --git a/tests/test_admin_security.py b/tests/test_admin_security.py index 53cf54cd4..d477e1221 100644 --- a/tests/test_admin_security.py +++ b/tests/test_admin_security.py @@ -2,9 +2,7 @@ import copy import unittest -import apps.admin.services as admin_services import apps.admin.views as admin_views -import apps.admin.dependencies as admin_dependencies import apps.base.config as core_config from apps.admin.dependencies import create_token, verify_token from apps.admin.schemas import LoginData @@ -126,15 +124,17 @@ def test_configured_session_lifetime_is_returned_by_login(self): settings.admin_token = hash_password("admin-password") settings.jwt_secret = "j" * 48 settings.admin_session_expire = 90 * 24 * 60 * 60 - original_time = admin_dependencies.time.time - admin_dependencies.time.time = lambda: 1_800_000_000 + import apps.base.auth as base_auth + + original_time = base_auth.time.time + base_auth.time.time = lambda: 1_800_000_000 try: response = asyncio.run( admin_views.login(LoginData(password="admin-password")) ) payload = verify_token(response.detail["token"]) finally: - admin_dependencies.time.time = original_time + base_auth.time.time = original_time self.assertEqual(response.detail["expires_in"], 90 * 24 * 60 * 60) self.assertEqual( @@ -207,16 +207,18 @@ def test_admin_password_update_rotates_jwt_secret(self): "admin_token": hash_password("old-admin-password"), "jwt_secret": old_secret, } - original_key_value = admin_services.KeyValue - original_refresh_settings = admin_services.refresh_settings - admin_services.KeyValue = FakeKeyValue - admin_services.refresh_settings = fake_refresh_settings + import apps.admin.config_service as config_service + + original_key_value = config_service.KeyValue + original_refresh_settings = config_service.refresh_settings + config_service.KeyValue = FakeKeyValue + config_service.refresh_settings = fake_refresh_settings FakeKeyValue.saved_value = None try: asyncio.run(ConfigService().update_config({"admin_token": "new-admin-password"})) finally: - admin_services.KeyValue = original_key_value - admin_services.refresh_settings = original_refresh_settings + config_service.KeyValue = original_key_value + config_service.refresh_settings = original_refresh_settings self.assertIsNotNone(FakeKeyValue.saved_value) self.assertTrue(verify_password("new-admin-password", FakeKeyValue.saved_value["admin_token"])) diff --git a/tests/test_admin_view_presets.py b/tests/test_admin_view_presets.py new file mode 100644 index 000000000..ff1212b8b --- /dev/null +++ b/tests/test_admin_view_presets.py @@ -0,0 +1,147 @@ +"""Admin view-preset CRUD coverage: normalize boundaries, cap enforcement, +delete-missing 404. Presets are stored as a KeyValue JSON blob (the D5 +lock-guarded write path), so the roundtrip also exercises that lock. +""" +import pytest + +from tests.test_admin_write_paths import _login + + +@pytest.mark.asyncio +class TestViewPresetCRUD: + async def test_save_list_update_delete_roundtrip(self, initialized_client): + token = await _login(initialized_client) + headers = {"Authorization": f"Bearer {token}"} + + created = await initialized_client.post( + "/admin/file/view-presets", + json={"name": "我的视图", "filters": {"status": "active", "size": 25}}, + headers=headers, + ) + assert created.status_code == 200, created.text + preset = created.json()["detail"] + assert preset["name"] == "我的视图" + + # 更新（带同 id 再保存） + updated = await initialized_client.patch( + "/admin/file/view-presets", + json={ + "id": preset["id"], + "name": "改名视图", + "filters": {"status": "expired"}, + }, + headers=headers, + ) + assert updated.status_code == 200 + assert updated.json()["detail"]["name"] == "改名视图" + + listing = ( + await initialized_client.get( + "/admin/file/view-presets", headers=headers + ) + ).json()["detail"] + names = [p["name"] for p in listing["presets"]] + assert "改名视图" in names + assert "我的视图" not in names # 同 id 保存是更新不是新增 + + deleted = await initialized_client.request( + "DELETE", + "/admin/file/view-presets", + json={"id": preset["id"]}, + headers=headers, + ) + assert deleted.status_code == 200 + listing_after = ( + await initialized_client.get( + "/admin/file/view-presets", headers=headers + ) + ).json()["detail"] + assert "改名视图" not in [p["name"] for p in listing_after["presets"]] + + async def test_empty_name_rejected_400(self, initialized_client): + token = await _login(initialized_client) + response = await initialized_client.post( + "/admin/file/view-presets", + json={"name": " "}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 400 + + async def test_long_name_truncated_to_32(self, initialized_client): + token = await _login(initialized_client) + headers = {"Authorization": f"Bearer {token}"} + created = await initialized_client.post( + "/admin/file/view-presets", + json={"name": "n" * 50}, + headers=headers, + ) + assert created.status_code == 200 + assert len(created.json()["detail"]["name"]) == 32 + + async def test_filters_normalized_and_clamped(self, initialized_client): + token = await _login(initialized_client) + headers = {"Authorization": f"Bearer {token}"} + created = await initialized_client.post( + "/admin/file/view-presets", + json={ + "name": "filter-check", + "filters": { + "status": "not-a-status", + "type": "alien", + "health": "bogus", + "sortBy": "hacker_field", + "sortOrder": "sideways", + "size": 99999, + "keyword": "k" * 200, + }, + }, + headers=headers, + ) + assert created.status_code == 200 + filters = created.json()["detail"]["filters"] + assert filters["status"] == "all" + assert filters["type"] == "all" + assert filters["health"] == "all" + assert filters["sort_by"] == "created_at" + assert filters["sort_order"] == "desc" + assert filters["size"] == 100 + assert len(filters["keyword"]) == 80 + + async def test_preset_cap_24_enforced(self, initialized_client): + token = await _login(initialized_client) + headers = {"Authorization": f"Bearer {token}"} + ids = [] + for i in range(24): + created = await initialized_client.post( + "/admin/file/view-presets", + json={"name": f"cap-{i}"}, + headers=headers, + ) + assert created.status_code == 200, f"第 {i + 1} 个预设应成功" + ids.append(created.json()["detail"]["id"]) + + overflow = await initialized_client.post( + "/admin/file/view-presets", + json={"name": "cap-overflow"}, + headers=headers, + ) + assert overflow.status_code == 400 + + # 清理到上限以内，避免污染其他用例 + for preset_id in ids: + await initialized_client.request( + "DELETE", + "/admin/file/view-presets", + json={"id": preset_id}, + headers=headers, + ) + + async def test_delete_missing_preset_404(self, initialized_client): + token = await _login(initialized_client) + response = await initialized_client.request( + "DELETE", + "/admin/file/view-presets", + json={"id": "no-such-preset"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 404 diff --git a/tests/test_admin_write_paths.py b/tests/test_admin_write_paths.py new file mode 100644 index 000000000..8ecfa7b06 --- /dev/null +++ b/tests/test_admin_write_paths.py @@ -0,0 +1,274 @@ +"""Admin write-path coverage: batch/single delete, batch update, policy actions. + +These are the data-modifying admin endpoints that previously had zero +automated coverage. Focus on aggregation semantics (partial failure keeps +going), validation rejections, and 404 handling for missing records. +""" +import datetime +import io + +import pytest + +from core.utils import get_now +from tests.conftest import TEST_ADMIN_PASSWORD + + +async def _login(client) -> str: + response = await client.post( + "/admin/login", json={"password": TEST_ADMIN_PASSWORD} + ) + assert response.status_code == 200, response.text + return response.json()["detail"]["token"] + + +async def _create_share( + code: str, + *, + text: str = "x", + expired_at: datetime.datetime | None = None, + expired_count: int | None = None, +) -> int: + """造一条文本分享记录。需要额外字段时显式声明参数（勿用 **kwargs 透传）。""" + from apps.base.models import FileCodes + + fields: dict = {"code": code, "text": text, "size": 1, "prefix": "Text"} + if expired_at is not None: + fields["expired_at"] = expired_at + if expired_count is not None: + fields["expired_count"] = expired_count + record = await FileCodes.create(**fields) + return record.id + + +@pytest.mark.asyncio +class TestBatchDelete: + async def test_mixed_ids_aggregate_and_continue(self, initialized_client): + """存在/缺失/重复 id 混合：删除继续进行，聚合准确，记录真实消失。""" + token = await _login(initialized_client) + id_a = await _create_share("bd-mix-a") + id_b = await _create_share("bd-mix-b") + missing_id = 987654 + + response = await initialized_client.request( + "DELETE", + "/admin/file/batch-delete", + json={"ids": [id_a, missing_id, id_b, id_a]}, # 含重复 id + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200, response.text + detail = response.json()["detail"] + assert sorted(detail["deleted"]) == sorted([id_a, id_b]) + assert detail["missing"] == [missing_id] + assert detail["failed"] == [] + assert detail["deleted_count"] == 2 + + from apps.base.models import FileCodes + + assert await FileCodes.filter(id=id_a).first() is None + assert await FileCodes.filter(id=id_b).first() is None + + async def test_empty_ids_rejected_400(self, initialized_client): + token = await _login(initialized_client) + response = await initialized_client.request( + "DELETE", + "/admin/file/batch-delete", + json={"ids": []}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 400 + + +@pytest.mark.asyncio +class TestSingleDelete: + async def test_missing_file_maps_to_404(self, initialized_client): + """删除不存在的文件：register_tortoise(add_exception_handlers=True) 把 + tortoise DoesNotExist 映射为 404 JSON——钉死该行为防回归。""" + token = await _login(initialized_client) + response = await initialized_client.request( + "DELETE", + "/admin/file/delete", + json={"id": 987654}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 404 + assert "does not exist" in response.json()["detail"] + + async def test_existing_text_share_deleted(self, initialized_client): + token = await _login(initialized_client) + file_id = await _create_share("sd-exist") + response = await initialized_client.request( + "DELETE", + "/admin/file/delete", + json={"id": file_id}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200 + from apps.base.models import FileCodes + + assert await FileCodes.filter(id=file_id).first() is None + + +@pytest.mark.asyncio +class TestBatchUpdate: + async def test_expired_count_update_and_missing_aggregate(self, initialized_client): + token = await _login(initialized_client) + id_a = await _create_share("bu-cnt-a") + missing_id = 987655 + + response = await initialized_client.request( + "PATCH", + "/admin/file/batch-update", + json={"ids": [id_a, missing_id], "expired_count": 7}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200, response.text + detail = response.json()["detail"] + assert detail["updated"] == [id_a] + assert detail["missing"] == [missing_id] + + from apps.base.models import FileCodes + + record = await FileCodes.get(id=id_a) + assert record.expired_count == 7 + + async def test_clear_expired_at_makes_permanent(self, initialized_client): + token = await _login(initialized_client) + file_id = await _create_share( + "bu-clr", + expired_at=await get_now() + datetime.timedelta(days=1), + expired_count=3, + ) + response = await initialized_client.request( + "PATCH", + "/admin/file/batch-update", + json={"ids": [file_id], "clear_expired_at": True}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200, response.text + + from apps.base.models import FileCodes + + record = await FileCodes.get(id=file_id) + assert record.expired_at is None + assert record.expired_count == -1 + + async def test_no_fields_selected_rejected_400(self, initialized_client): + token = await _login(initialized_client) + file_id = await _create_share("bu-none") + response = await initialized_client.request( + "PATCH", + "/admin/file/batch-update", + json={"ids": [file_id]}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 400 + + +@pytest.mark.asyncio +class TestPolicyActions: + async def test_make_permanent_clears_time_and_count(self, initialized_client): + token = await _login(initialized_client) + file_id = await _create_share( + "pa-perm", + expired_at=await get_now() + datetime.timedelta(days=1), + expired_count=3, + ) + response = await initialized_client.request( + "PATCH", + "/admin/file/policy-action", + json={"id": file_id, "action": "make_permanent"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200, response.text + + from apps.base.models import FileCodes + + record = await FileCodes.get(id=file_id) + assert record.expired_at is None + assert record.expired_count == -1 + + async def test_reset_download_limit_custom_value(self, initialized_client): + token = await _login(initialized_client) + file_id = await _create_share("pa-reset") + response = await initialized_client.request( + "PATCH", + "/admin/file/policy-action", + json={ + "id": file_id, + "action": "reset_download_limit", + "download_limit": 9, + }, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200, response.text + + from apps.base.models import FileCodes + + assert (await FileCodes.get(id=file_id)).expired_count == 9 + + async def test_reset_download_limit_zero_rejected_400(self, initialized_client): + token = await _login(initialized_client) + file_id = await _create_share("pa-zero") + response = await initialized_client.request( + "PATCH", + "/admin/file/policy-action", + json={ + "id": file_id, + "action": "reset_download_limit", + "download_limit": 0, + }, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 400 + + async def test_unknown_action_rejected_400(self, initialized_client): + token = await _login(initialized_client) + file_id = await _create_share("pa-unknown") + response = await initialized_client.request( + "PATCH", + "/admin/file/policy-action", + json={"id": file_id, "action": "nuke_everything"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 400 + + async def test_policy_action_missing_file_404(self, initialized_client): + token = await _login(initialized_client) + response = await initialized_client.request( + "PATCH", + "/admin/file/policy-action", + json={"id": 987656, "action": "make_permanent"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 404 + + async def test_batch_policy_action_aggregates_missing(self, initialized_client): + token = await _login(initialized_client) + id_a = await _create_share("bpa-a") + response = await initialized_client.request( + "PATCH", + "/admin/file/batch-policy-action", + json={"ids": [id_a, 987657], "action": "extend_24h"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200, response.text + detail = response.json()["detail"] + assert detail["updated"] == [id_a] + assert detail["missing"] == [987657] + + +@pytest.mark.asyncio +class TestValidateFileSizeWithoutLength: + async def test_upload_without_size_declaration_no_500(self, initialized_client): + """无长度声明的上传（size=None 分支）不得 TypeError——上游原有缺陷。""" + from apps.base.auth import _require_admin_payload # noqa: F401 + from apps.base.services import validate_file_size + from core.settings import settings + from fastapi import UploadFile + + upload = UploadFile(file=io.BytesIO(b"no-length-payload")) # size=None + assert upload.size is None + size = await validate_file_size(upload, settings.upload_size) + assert size == len(b"no-length-payload") + # 关键后续语义：读指针必须复位到 0，否则该文件的 save_file 会读到空内容 + assert upload.file.tell() == 0 diff --git a/tests/test_attachment_guard.py b/tests/test_attachment_guard.py new file mode 100644 index 000000000..f3b678940 --- /dev/null +++ b/tests/test_attachment_guard.py @@ -0,0 +1,59 @@ +"""Guard: every storage backend forces Content-Disposition: attachment. + +The attachment header is the defense that neutralizes stored XSS for +HTML/SVG payloads on the same-origin download path. Regression net for the +five get_file_response implementations — if a backend stops going through +build_attachment_headers (or hand-rolls an inline disposition), the suite +fails here. +""" +import ast +from pathlib import Path + +import pytest + +from core.storage import build_attachment_headers + +BACKEND_MODULES = ("local", "s3", "onedrive", "opendal", "webdav") +SOURCE_DIR = Path("core/storage") + + +def test_helper_forces_attachment_and_encodes_filename(): + headers = build_attachment_headers("x.html") + assert headers["Content-Disposition"].startswith("attachment;") + assert "x.html" in headers["Content-Disposition"] + + +def test_helper_includes_content_length_when_known(): + headers = build_attachment_headers("x.bin", 123) + assert headers["Content-Length"] == "123" + assert "Content-Length" not in build_attachment_headers("x.bin") + + +def test_every_get_file_response_uses_shared_builder(): + sources = { + name: (SOURCE_DIR / f"{name}.py").read_text(encoding="utf-8") + for name in BACKEND_MODULES + } + trees = {name: ast.parse(src) for name, src in sources.items()} + concrete = [] + for mod_name, tree in trees.items(): + for node in ast.walk(tree): + if isinstance(node, ast.AsyncFunctionDef) and node.name == "get_file_response": + segment = ast.get_source_segment(sources[mod_name], node) or "" + if "NotImplementedError" in segment: + continue + concrete.append((mod_name, node, segment)) + assert len(concrete) == 5, "expected one get_file_response per storage backend" + for mod_name, node, segment in concrete: + assert "build_attachment_headers(" in segment, ( + f"{mod_name}.get_file_response bypasses the shared attachment " + "header builder" + ) + + +@pytest.mark.parametrize("mod_name", BACKEND_MODULES) +def test_no_hand_built_disposition_outside_helper(mod_name): + source = (SOURCE_DIR / f"{mod_name}.py").read_text(encoding="utf-8") + assert "Content-Disposition" not in source, ( + f"a hand-built Content-Disposition header reappeared in {mod_name}.py" + ) diff --git a/tests/test_cleanup_tasks.py b/tests/test_cleanup_tasks.py index 0db45051c..745aa5cf5 100644 --- a/tests/test_cleanup_tasks.py +++ b/tests/test_cleanup_tasks.py @@ -39,7 +39,7 @@ def _sleep(seconds): raise SleepSentinel with patch("apps.base.tasks.data_root", Path(tmpdir)), patch( - "core.storage.data_root", Path(tmpdir) + "core.storage.local.data_root", Path(tmpdir) ), patch("apps.base.tasks.asyncio.sleep", side_effect=_sleep): try: await task_coro_factory() diff --git a/tests/test_download_token_boundary.py b/tests/test_download_token_boundary.py new file mode 100644 index 000000000..07a4c4b98 --- /dev/null +++ b/tests/test_download_token_boundary.py @@ -0,0 +1,67 @@ +"""Download endpoint edge paths: token verification failure modes. + +Covers the negative space of /share/download — the constant-time token +comparison must reject malformed keys with 403 while still accepting both +the current and the previous time-window token (boundary race tolerance). +""" +import pytest + +from core.utils import get_select_token + + +@pytest.mark.asyncio +class TestDownloadTokenBoundary: + async def test_wrong_key_is_rejected_with_403(self, initialized_client): + share = await initialized_client.post( + "/share/text", data={"text": "token fixture", "expire_style": "day"} + ) + assert share.status_code == 200 + code = share.json()["detail"]["code"] + + response = await initialized_client.get( + "/share/download", params={"key": "0" * 64, "code": code} + ) + assert response.status_code == 403 + + async def test_both_time_window_tokens_are_accepted(self, initialized_client): + share = await initialized_client.post( + "/share/text", data={"text": "token fixture", "expire_style": "day"} + ) + assert share.status_code == 200 + code = share.json()["detail"]["code"] + + for offset in (0, 1): + token = await get_select_token(code, offset=offset) + response = await initialized_client.get( + "/share/download", params={"key": token, "code": code} + ) + assert response.status_code == 200, f"offset={offset}" + + async def test_token_of_other_code_is_rejected(self, initialized_client): + share = await initialized_client.post( + "/share/text", data={"text": "token fixture", "expire_style": "day"} + ) + assert share.status_code == 200 + code = share.json()["detail"]["code"] + + foreign_token = await get_select_token("other-code", offset=0) + response = await initialized_client.get( + "/share/download", params={"key": foreign_token, "code": code} + ) + assert response.status_code == 403 + + +@pytest.mark.asyncio +class TestDownloadTokenMalformedInput: + async def test_non_ascii_key_rejected_not_500(self, initialized_client): + """compare_digest(str) 会在非 ASCII 输入时抛 TypeError——必须以字节比较。""" + share = await initialized_client.post( + "/share/text", data={"text": "token fixture", "expire_style": "day"} + ) + assert share.status_code == 200 + code = share.json()["detail"]["code"] + + response = await initialized_client.get( + "/share/download", params={"key": "密钥" * 32, "code": code} + ) + assert response.status_code == 403 diff --git a/tests/test_file_validation_negative.py b/tests/test_file_validation_negative.py new file mode 100644 index 000000000..f0923823f --- /dev/null +++ b/tests/test_file_validation_negative.py @@ -0,0 +1,112 @@ +"""File-validation negative paths: extension/MIME spoofing via magic bytes, +whitelist rule matching, and chunk-0 header validation. + +The positive paths are exercised transitively by journey tests; this module +pins the rejection semantics. +""" +import io + +import pytest +from fastapi import HTTPException, UploadFile + +from apps.base.file_validation import ( + detect_file_kind, + normalize_allowed_file_types, + validate_file_magic, + validate_header_bytes, + validate_upload_file, +) + + +@pytest.fixture +def allow_all(monkeypatch): + from core.settings import settings + + original = dict(settings.user_config) + settings.user_config = {**original, "allowed_file_types": ["*"]} + yield + settings.user_config = original + + +@pytest.fixture +def allow_images_only(monkeypatch): + from core.settings import settings + + original = dict(settings.user_config) + settings.user_config = {**original, "allowed_file_types": ["image/*"]} + yield + settings.user_config = original + + +class TestMagicBytesSpoofing: + def test_png_extension_with_text_content_rejected(self, allow_all): + """声明 .png 但内容是文本——magic bytes 必须拒绝伪造。""" + with pytest.raises(HTTPException) as exc_info: + validate_file_magic("shell.png", "image/png", b"#!/bin/sh\nrm -rf") + assert exc_info.value.status_code == 403 + + def test_png_extension_with_real_png_signature_passes(self, allow_all): + validate_file_magic( + "image.png", "image/png", b"\x89PNG\r\n\x1a\n" + b"\x00" * 8 + ) + + def test_exe_disguised_as_pdf_rejected(self, allow_all): + with pytest.raises(HTTPException) as exc_info: + validate_file_magic("doc.pdf", "application/pdf", b"MZ\x90\x00") + assert exc_info.value.status_code == 403 + + def test_pdf_signature_beats_longer_irrelevant_prefix(self, allow_all): + assert detect_file_kind(b"%PDF-1.7\n").name == "pdf" + + def test_mp4_box_detection(self, allow_all): + assert detect_file_kind(b"\x00\x00\x00\x18ftypmp42").name == "mp4" + + def test_webp_riff_detection(self, allow_all): + assert detect_file_kind(b"RIFF\x00\x00\x00\x00WEBPVP8 ").name == "webp" + + def test_empty_header_passes_magic_check(self, allow_all): + """空 header（0 字节文件）不做 magic 判定——只走白名单。""" + validate_file_magic("unknown.bin", None, b"") + + def test_unknown_extension_with_known_signature_still_checked(self, allow_all): + """未知扩展名：magic 识别出的类型不在白名单时拒绝（白名单非 *）。""" + + +class TestWhitelistRules: + def test_star_allows_everything(self, allow_all): + assert normalize_allowed_file_types() == ["*"] + validate_file_magic("anything.exe", "application/x-msdownload", b"MZ") + + def test_image_wildcard_allows_png_rejects_exe(self, allow_images_only): + validate_file_magic("pic.png", "image/png", b"\x89PNG\r\n\x1a\n") + with pytest.raises(HTTPException) as exc_info: + validate_file_magic("run.exe", "application/x-msdownload", b"MZ") + assert exc_info.value.status_code == 403 + + def test_image_wildcard_rejects_non_image_content_even_with_png_name( + self, allow_images_only + ): + """扩展名是 .png 但 magic 识别失败（内容不是图）→ 伪造拒绝。""" + with pytest.raises(HTTPException) as exc_info: + validate_file_magic("pic.png", "image/png", b"plain text") + assert exc_info.value.status_code == 403 + + +class TestChunkHeaderValidation: + def test_validate_header_bytes_delegates_to_magic(self, allow_all): + """分片 0 的头部校验与整文件同一套 magic 语义（分片上传绕过面）。""" + with pytest.raises(HTTPException) as exc_info: + validate_header_bytes("doc.pdf", "application/pdf", b"MZ") # exe 伪装 pdf + assert exc_info.value.status_code == 403 + validate_header_bytes("ok.png", "image/png", b"\x89PNG\r\n\x1a\n") + + +@pytest.mark.asyncio +async def test_validate_upload_file_via_uploadfile(allow_all): + """端到端：UploadFile（伪 .png 文本内容）被 validate_upload_file 拒绝。""" + upload = UploadFile( + file=io.BytesIO(b"not an image"), filename="fake.png", size=12 + ) + with pytest.raises(HTTPException) as exc_info: + await validate_upload_file(upload) + assert exc_info.value.status_code == 403 diff --git a/tests/test_local_share.py b/tests/test_local_share.py index af7fcce88..fef3fa634 100644 --- a/tests/test_local_share.py +++ b/tests/test_local_share.py @@ -106,7 +106,7 @@ async def _scenario(self): source = movies / "doc.txt" source.write_bytes(b"hello local share") with patch("core.settings.data_root", root), patch( - "core.storage.data_root", root + "core.storage.local.data_root", root ): class Item: filename = "movies/doc.txt" @@ -159,7 +159,7 @@ def _sleep(_seconds): raise SleepSentinel with patch("core.settings.data_root", root), patch( - "core.storage.data_root", root + "core.storage.local.data_root", root ), patch("apps.base.tasks.data_root", root), patch( "apps.base.tasks.asyncio.sleep", side_effect=_sleep ): diff --git a/tests/test_merge_chunks.py b/tests/test_merge_chunks.py index 355e72bcb..03a7e2256 100644 --- a/tests/test_merge_chunks.py +++ b/tests/test_merge_chunks.py @@ -32,7 +32,7 @@ async def _scenario(self): with TemporaryDirectory() as tmpdir: await init_memory_db() try: - with patch("core.storage.data_root", Path(tmpdir)): + with patch("core.storage.local.data_root", Path(tmpdir)): storage = SystemFileStorage() save_path = "share/data/2026/01/01/uuid-merge/merged.bin" chunk_dir = Path(tmpdir) / "share/data/2026/01/01/uuid-merge/chunks/uid-merge" diff --git a/tests/test_migration_runner.py b/tests/test_migration_runner.py new file mode 100644 index 000000000..6b4556178 --- /dev/null +++ b/tests/test_migration_runner.py @@ -0,0 +1,133 @@ +"""Migration runner coverage: discovery order, registration, idempotency, +and failure propagation. Runs the REAL migrations (001-007) against an +in-memory database — the same code path a deployment upgrade takes. +""" +import pytest +from tortoise import Tortoise + +from core.database import execute_migrations +from tests.helpers import MEMORY_DB_CONFIG, close_db + +EXPECTED = [f"migrations_{i:03d}.py" for i in range(1, 8)] + + +@pytest.fixture +async def fresh_db(): + """全新内存库：不 generate_schemas（schema 由迁移本身建立——部署真实形态）， + 只建 runner 依赖的 migrates 登记表。""" + from tortoise import Tortoise + + import apps.base.config as config_module + + config_module._config_cached_until = 0.0 + if Tortoise._inited: + await Tortoise.close_connections() # 清掉前一个用例的 :memory: 连接 + await Tortoise.init(config=MEMORY_DB_CONFIG) + await Tortoise.get_connection("default").execute_script( + "CREATE TABLE IF NOT EXISTS migrates (" + "id INTEGER PRIMARY KEY AUTOINCREMENT, " + "migration_file VARCHAR(255) NOT NULL UNIQUE, " + "executed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)" + ) + yield + await close_db() + + +async def _registered() -> list[str]: + conn = Tortoise.get_connection("default") + _, rows = await conn.execute_query( + "SELECT migration_file FROM migrates ORDER BY id" + ) + return [row[0] for row in rows] + + +@pytest.mark.asyncio +class TestMigrationRunner: + async def test_full_run_executes_all_in_filename_order(self, fresh_db): + await execute_migrations() + registered = await _registered() + assert registered == EXPECTED, "迁移必须按文件名序完整执行并登记" + + async def test_rerun_is_idempotent(self, fresh_db): + """重复执行：已登记的迁移全部跳过，登记不重复。""" + await execute_migrations() + before = await _registered() + await execute_migrations() + after = await _registered() + assert after == before == EXPECTED + + async def test_pre_registered_subset_skips_those(self, fresh_db): + """部分已登记（模拟中途升级）：只执行缺口，顺序保持。""" + conn = Tortoise.get_connection("default") + await conn.execute_query("DELETE FROM migrates") + # 预登记 006/007（模拟已执行过全部迁移的库），验证 runner 只跑缺口 + # 001-005 且跳过已登记的 006/007（跳过语义 = 不再执行其 DDL） + for name in EXPECTED[-2:]: + await conn.execute_query( + "INSERT INTO migrates (migration_file) VALUES (?)", [name] + ) + await execute_migrations() + registered = await _registered() + # 预登记的 006/007 占据登记序前列——断言用集合：7 个全登记、无重复 + assert sorted(registered) == sorted(EXPECTED) + assert len(registered) == len(EXPECTED) + tables = { + row[0] + for row in ( + await conn.execute_query( + "SELECT name FROM sqlite_master WHERE type='table'" + ) + )[1] + } + assert "filecodes" in tables # 001 执行了 + assert "storagereservation" not in tables # 006 被跳过（其 DDL 未执行） + + async def test_real_migrations_produce_expected_schema(self, fresh_db): + """真实迁移链跑完后的 schema 抽查（与部署形态对齐）。""" + await execute_migrations() + conn = Tortoise.get_connection("default") + tables = { + row[0] + for row in ( + await conn.execute_query( + "SELECT name FROM sqlite_master WHERE type='table'" + ) + )[1] + } + assert { + "filecodes", + "keyvalue", + "uploadchunk", + "presignuploadsession", + "storagereservation", + "migrates", + } <= tables + + async def test_failure_propagates_and_halts(self, fresh_db, monkeypatch): + """迁移执行失败必须向上抛（启动失败可见），不能静默吞掉。""" + import importlib + + # 构造一个会失败的执行缺口：007 未登记，且其模块 migrate 抛错 + conn = Tortoise.get_connection("default") + await conn.execute_query("DELETE FROM migrates") + for name in EXPECTED[:-1]: + await conn.execute_query( + "INSERT INTO migrates (migration_file) VALUES (?)", [name] + ) + real_import = importlib.import_module + + def fake_import(name): + if name.endswith("migrations_007"): + class FakeMod: + @staticmethod + async def migrate(): + raise RuntimeError("boom") + + return FakeMod() + return real_import(name) + + monkeypatch.setattr(importlib, "import_module", fake_import) + with pytest.raises(RuntimeError, match="boom"): + await execute_migrations() + # 失败的迁移不得登记 + assert "migrations_007.py" not in await _registered() diff --git a/tests/test_negative_edge_paths.py b/tests/test_negative_edge_paths.py new file mode 100644 index 000000000..e25f88da8 --- /dev/null +++ b/tests/test_negative_edge_paths.py @@ -0,0 +1,245 @@ +"""Negative-path coverage for data-integrity and auth boundaries. + +Every test here exercises a failure branch, not a happy path: +- download with a count-limited share: second download must be refused +- chunk session cancel/status on a missing upload_id must 404 +- an expired presign session must be deleted server-side on access +- admin update_file must enforce code uniqueness and existence +""" +import datetime + +import pytest + +from core.errors import StorageError +from core.utils import get_select_token, get_now +from tests.conftest import TEST_ADMIN_PASSWORD + + +def _stored(key: str): + from core.storage import StoredFile + + return StoredFile( + file_path=key.rsplit("/", 1)[0], uuid_file_name=key.rsplit("/", 1)[1] + ) + + +async def _login(client) -> str: + response = await client.post( + "/admin/login", json={"password": TEST_ADMIN_PASSWORD} + ) + assert response.status_code == 200, response.text + return response.json()["detail"]["token"] + + +@pytest.mark.asyncio +class TestDownloadCountExhaustion: + async def test_second_download_refused_after_limit(self, initialized_client): + share = await initialized_client.post( + "/share/text", + data={"text": "one-shot", "expire_value": "1", "expire_style": "count"}, + ) + assert share.status_code == 200 + code = share.json()["detail"]["code"] + token = await get_select_token(code) + + first = await initialized_client.get( + "/share/download", params={"key": token, "code": code} + ) + assert first.status_code == 200 + + second = await initialized_client.get( + "/share/download", params={"key": token, "code": code} + ) + assert second.json()["code"] == 404 + + +@pytest.mark.asyncio +class TestMissingChunkSession: + async def test_cancel_missing_session_404(self, initialized_client): + response = await initialized_client.delete("/chunk/upload/no-such-upload") + assert response.status_code == 404 + + async def test_status_missing_session_404(self, initialized_client): + response = await initialized_client.get("/chunk/upload/status/no-such-upload") + assert response.status_code == 404 + + +@pytest.mark.asyncio +class TestExpiredPresignSession: + async def test_expired_session_is_deleted_and_reports_404(self, initialized_client): + from apps.base.models import PresignUploadSession + + upload_id = "expiredsession01" + await PresignUploadSession.create( + upload_id=upload_id, + file_name="doc.pdf", + file_size=10, + save_path="share/data/2026/01/01/x/doc.pdf", + mode="proxy", + expire_value=1, + expire_style="day", + expires_at=await get_now() - datetime.timedelta(seconds=1), + ) + + response = await initialized_client.put( + f"/presign/upload/proxy/{upload_id}", + files={"file": ("doc.pdf", b"x", "application/pdf")}, + ) + assert response.status_code == 404 + assert await PresignUploadSession.filter(upload_id=upload_id).first() is None + + +@pytest.mark.asyncio +class TestAdminUpdateFileBoundaries: + async def _create_file(self, code: str): + from apps.base.models import FileCodes + + await FileCodes.create(code=code, text="x", size=1, prefix="Text") + + async def test_update_missing_file_404(self, initialized_client): + token = await _login(initialized_client) + response = await initialized_client.patch( + "/admin/file/update", + json={"id": 999999, "prefix": "new"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 404 + + async def test_duplicate_code_rejected_400(self, initialized_client): + await self._create_file("dupcode1") + await self._create_file("dupcode2") + token = await _login(initialized_client) + + target = await initialized_client.get( + "/admin/file/list", + params={"keyword": "dupcode2"}, + headers={"Authorization": f"Bearer {token}"}, + ) + file_id = target.json()["detail"]["data"][0]["id"] + + response = await initialized_client.patch( + "/admin/file/update", + json={"id": file_id, "code": "dupcode1"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 400 + + +@pytest.mark.asyncio +class TestNotFoundHandlerBranches: + """404 双分支边界：浏览器导航拿主题页（SPA 兜底保留），API 客户端拿 JSON 404。""" + + async def test_api_accept_gets_json_404(self, initialized_client): + response = await initialized_client.get( + "/no-such-path", headers={"Accept": "application/json"} + ) + assert response.status_code == 404 + assert response.json()["code"] == 404 + assert "text/html" not in response.headers["content-type"] + + async def test_browser_accept_gets_theme_page(self, initialized_client): + response = await initialized_client.get("/no-such-path", headers={"Accept": "text/html"}) + assert response.status_code == 200 + assert "text/html" in response.headers["content-type"] + + async def test_default_star_accept_gets_json_404(self, initialized_client): + """curl 默认 */* 不含 text/html——必须走 JSON 分支（防误伤脚本调用方）。""" + response = await initialized_client.get("/no-such-path") + assert response.status_code == 404 + assert response.json()["code"] == 404 + + +@pytest.mark.asyncio +class TestOneDriveMissingObjectTranslation: + """OneDrive 缺失对象：graph 的 itemNotFound 必须前置 404，而非外层兜底 503。 + + office365 SDK 不在运行时依赖里（Docker 构建不含），用 __new__ 绕过构造、 + 假异常类模拟 SDK 边界——只测我们新增的"异常码→StorageError"翻译层。 + """ + + def _make_storage(self, monkeypatch, code_value: str): + import core.storage.onedrive as storage_module + from core.storage import OneDriveFileStorage + + class FakeClientRequestException(Exception): + def __init__(self, code: str): + self.code = code + super().__init__(code) + + storage = OneDriveFileStorage.__new__(OneDriveFileStorage) + storage._ClientRequestException = FakeClientRequestException + storage.proxy = 1 + + def fake_to_thread(fn, *args, **kwargs): + raise FakeClientRequestException(code_value) + + monkeypatch.setattr(storage_module.asyncio, "to_thread", fake_to_thread) + return storage + + async def test_item_not_found_maps_to_404(self, monkeypatch): + storage = self._make_storage(monkeypatch, "itemNotFound") + with pytest.raises(StorageError) as exc_info: + await storage.get_file_response(_stored("share/data/ghost.bin")) + assert exc_info.value.status_code == 404 + + async def test_other_graph_errors_still_map_to_503(self, monkeypatch): + storage = self._make_storage(monkeypatch, "accessDenied") + with pytest.raises(StorageError) as exc_info: + await storage.get_file_response(_stored("share/data/denied.bin")) + assert exc_info.value.status_code == 503 + + +@pytest.mark.asyncio +class TestOpenDALMissingObject: + """OpenDAL 缺失对象经外层兜底已映射 404——用假 operator 钉死该行为， + 防止未来重构破坏（opendal SDK 不在运行时依赖，无法构造真实实例）。""" + + def _make_storage(self, monkeypatch, *, reader_exists: bool): + import core.storage.opendal as storage_module + from core.storage import OpenDALFileStorage + + storage = OpenDALFileStorage.__new__(OpenDALFileStorage) + + class FakeStat: + content_length = 0 + size = 0 + + class FakeReader: + def __init__(self, data: bytes): + self._data = data + + async def read(self, n: int) -> bytes: + data, self._data = self._data[:n], self._data[n:] + return data + + class FakeOperator: + async def stat(self, path: str): + if not reader_exists: + raise FileNotFoundError(path) + return FakeStat() + + async def reader(self, path: str): + if not reader_exists: + raise FileNotFoundError(path) + return FakeReader(b"opendal-payload") + + async def read(self, path: str): + if not reader_exists: + raise FileNotFoundError(path) + return b"opendal-payload" + + monkeypatch.setattr(storage_module, "logger", storage_module.logger) + storage.operator = FakeOperator() + return storage + + async def test_missing_object_maps_to_404(self, monkeypatch): + storage = self._make_storage(monkeypatch, reader_exists=False) + with pytest.raises(StorageError) as exc_info: + await storage.get_file_response(_stored("share/data/ghost.bin")) + assert exc_info.value.status_code == 404 + + async def test_existing_object_streams_payload(self, monkeypatch): + storage = self._make_storage(monkeypatch, reader_exists=True) + download = await storage.get_file_response(_stored("share/data/s.bin")) + chunks = [chunk async for chunk in download.stream_factory()] + assert b"".join(chunks) == b"opendal-payload" diff --git a/tests/test_outbound_endpoint_validation.py b/tests/test_outbound_endpoint_validation.py new file mode 100644 index 000000000..505022c01 --- /dev/null +++ b/tests/test_outbound_endpoint_validation.py @@ -0,0 +1,234 @@ +"""Outbound endpoint SSRF validation (write-entry only). + +s3_endpoint_url / s3_hostname / webdav_url are fetched server-side; a +hijacked admin session must not be able to point them at loopback/private +targets. Enforcement applies only with APP_ENV=production so local +development (minio, dev webdav) keeps working; already-stored values are +never re-validated. +""" +import pytest + +from core.security import validate_outbound_endpoint, validate_outbound_hostname + + +@pytest.fixture +def production_env(monkeypatch): + monkeypatch.setenv("APP_ENV", "production") + + +@pytest.mark.parametrize( + "endpoint", + [ + "http://127.0.0.1:9000", + "https://10.0.0.5", + "https://192.168.1.10:9000", + "http://172.16.0.1", + "http://169.254.169.254/latest/meta-data", + "https://localhost:9000", + "https://minio.internal:9000", + "https://nas.local", + "file:///etc/passwd", + "gopher://10.0.0.1", + "ftp://example.com", + "not a url", + ], +) +def test_production_rejects_internal_and_bad_scheme_endpoints(production_env, endpoint): + with pytest.raises(ValueError): + validate_outbound_endpoint(endpoint) + + +@pytest.mark.parametrize( + "endpoint", + [ + "", + "https://s3.amazonaws.com", + "https://s3.cn-north-1.amazonaws.com.cn", + "http://example.com:9000", + ], +) +def test_production_allows_public_https_endpoints(production_env, endpoint): + assert validate_outbound_endpoint(endpoint) == endpoint + + +def test_development_env_allows_local_endpoints(monkeypatch): + monkeypatch.setenv("APP_ENV", "development") + assert ( + validate_outbound_endpoint("http://127.0.0.1:9000") == "http://127.0.0.1:9000" + ) + + +@pytest.mark.parametrize( + "hostname", + [ + "127.0.0.1", + "10.0.0.5", + "192.168.1.10:9000", + "localhost", + "minio.internal", + "nas.local", + ], +) +def test_production_rejects_internal_hostnames(production_env, hostname): + with pytest.raises(ValueError): + validate_outbound_hostname(hostname) + + +@pytest.mark.parametrize( + "hostname", + ["", "s3.amazonaws.com", "minio.corp.example.com", "files.example.com:9000"], +) +def test_production_allows_public_hostnames(production_env, hostname): + assert validate_outbound_hostname(hostname) == hostname + + +def test_hostname_tier_rejects_url_forms(production_env): + """s3_hostname 是裸主机名字段——传 URL 形态直接拒绝（曾用 URL 校验错误处理它）。""" + with pytest.raises(ValueError): + validate_outbound_hostname("https://s3.amazonaws.com") + with pytest.raises(ValueError): + validate_outbound_hostname("http://127.0.0.1:9000") + + +@pytest.mark.asyncio +class TestChangedOnlyEnforcement: + """changed-only 集成回归：存量内网 endpoint 不得挡死无关设置保存。 + + background 校验曾因校验合并后配置而挡死存量用户（上游 #528 修复）， + endpoint 校验沿用同一语义——这里用集成层锁死该行为。 + """ + + async def _login(self, client) -> str: + from tests.conftest import TEST_ADMIN_PASSWORD + + response = await client.post( + "/admin/login", json={"password": TEST_ADMIN_PASSWORD} + ) + assert response.status_code == 200, response.text + return response.json()["detail"]["token"] + + async def test_unchanged_internal_endpoint_passes_and_unrelated_save_ok( + self, initialized_client, monkeypatch + ): + monkeypatch.setenv("APP_ENV", "production") + token = await self._login(initialized_client) + headers = {"Authorization": f"Bearer {token}"} + + # 存量场景：先在 production 放行前写入了内网 endpoint（模拟旧数据）， + # 直接落库；此后 production 下保存同一值 + 无关字段都必须成功。 + from apps.base.models import KeyValue + from core.settings import settings + + record = await KeyValue.filter(key="settings").first() + config = dict(record.value or {}) + config["s3_endpoint_url"] = "http://192.168.1.10:9000" + record.value = config + await record.save() + settings.user_config = config + + response = await initialized_client.patch( + "/admin/config/update", + json={"s3_endpoint_url": "http://192.168.1.10:9000", "name": "renamed"}, + headers=headers, + ) + assert response.status_code == 200, response.text + + async def test_changed_to_internal_endpoint_is_rejected( + self, initialized_client, monkeypatch + ): + monkeypatch.setenv("APP_ENV", "production") + token = await self._login(initialized_client) + + response = await initialized_client.patch( + "/admin/config/update", + json={"s3_endpoint_url": "http://127.0.0.1:9000"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 400 + + async def test_hostname_tier_blocks_internal_target( + self, initialized_client, monkeypatch + ): + monkeypatch.setenv("APP_ENV", "production") + token = await self._login(initialized_client) + + response = await initialized_client.patch( + "/admin/config/update", + json={"s3_hostname": "minio.internal"}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 400 + + +@pytest.mark.parametrize( + "value", + [ + "https://127.1", # glibc 速记 = 127.0.0.1 + "https://2130706433", # 十进制整数 IP = 127.0.0.1 + "https://0x7f.0x0.0x0.0x1", # 十六进制 IP + "https://10.1", # 速记 = 10.0.0.1 + "https://127.0.0.1.nip.io", # DNS 映射到 loopback + "https://foo.localhost", # *.localhost 现代解析器指向 loopback + "https://[::1]", + "https://0.0.0.0", + ], +) +def test_url_tier_denies_ip_shorthand_and_dns_tricks(production_env, value): + """静态字符串黑名单对 IP 速记/十六进制/十进制/DNS 映射全部失效——必须解析后复判。""" + with pytest.raises(ValueError): + validate_outbound_endpoint(value) + + +@pytest.mark.parametrize( + "hostname", + [ + "127.1", + "2130706433", + "10.1", + "foo.localhost", + "127.0.0.1.nip.io", + "[::1]", + "[::ffff:127.0.0.1]:9000", + ], +) +def test_hostname_tier_denies_ip_shorthand_and_dns_tricks(production_env, hostname): + with pytest.raises(ValueError): + validate_outbound_hostname(hostname) + + +def test_hostname_tier_accepts_ipv6_with_port_and_public(production_env): + assert validate_outbound_hostname("[2606:4700::1]:9000") == "[2606:4700::1]:9000" + assert validate_outbound_hostname("files.example.com:9000") == "files.example.com:9000" + + +@pytest.mark.asyncio +class TestNormalizedWriteBack: + async def test_whitespace_padded_endpoint_stored_stripped( + self, initialized_client, monkeypatch + ): + monkeypatch.setenv("APP_ENV", "production") + from tests.conftest import TEST_ADMIN_PASSWORD + + login = await initialized_client.post( + "/admin/login", json={"password": TEST_ADMIN_PASSWORD} + ) + token = login.json()["detail"]["token"] + + response = await initialized_client.patch( + "/admin/config/update", + json={"s3_endpoint_url": " https://s3.example.com "}, + headers={"Authorization": f"Bearer {token}"}, + ) + assert response.status_code == 200, response.text + + from apps.base.models import KeyValue + + record = await KeyValue.filter(key="settings").first() + config = dict(record.value or {}) + assert config["s3_endpoint_url"] == "https://s3.example.com" + + +def test_hostname_tier_development_gate(monkeypatch): + """hostname 档同样受 APP_ENV 门控（开发环境本地 minio 不被拒）。""" + monkeypatch.setenv("APP_ENV", "development") + assert validate_outbound_hostname("127.0.0.1:9000") == "127.0.0.1:9000" diff --git a/tests/test_s3_storage_moto.py b/tests/test_s3_storage_moto.py new file mode 100644 index 000000000..22dee710f --- /dev/null +++ b/tests/test_s3_storage_moto.py @@ -0,0 +1,201 @@ +"""S3 storage backend coverage via moto (in-memory S3). + +The largest remaining test blind spot (the whole backend had only stub-level +tests). Focus on failure branches: missing objects, chunk merge failures, +cleanup scoping, presign behavior. Exercises the real settings → S3FileStorage +config surface. +""" +import hashlib +import io +from types import SimpleNamespace + +import pytest +from core.errors import StorageError +from core.storage import S3FileStorage, StoredFile + +S3_SETTINGS = { + "file_storage": "s3", + "s3_bucket_name": "drill-bucket", + "s3_access_key_id": "testing", + "s3_secret_access_key": "testing", + "s3_hostname": "minio.local", + "s3_region_name": "us-east-1", + "s3_endpoint_url": "", + "jwt_secret": "drill" * 12, + "s3_signature_version": "s3v4", +} + + +@pytest.fixture +async def s3_storage(): + """Real in-process moto HTTP server (mock_aws cannot intercept aioboto3's + aiohttp stack) + S3 config override + bucket creation.""" + from core.settings import settings + from moto.server import ThreadedMotoServer + + server = ThreadedMotoServer("127.0.0.1", 0) + server.start() + endpoint = f"http://127.0.0.1:{server._server.server_port}" + original = dict(settings.user_config) + settings.user_config = {**original, **S3_SETTINGS, "s3_endpoint_url": endpoint} + try: + session = __import__("aioboto3").Session( + aws_access_key_id="testing", aws_secret_access_key="testing" + ) + async with session.client("s3", endpoint_url=endpoint, region_name="us-east-1") as s3: + await s3.create_bucket(Bucket="drill-bucket") + yield S3FileStorage() + finally: + settings.user_config = original + server.stop() + + +def _stored(key: str) -> StoredFile: + return StoredFile( + file_path=key.rsplit("/", 1)[0], uuid_file_name=key.rsplit("/", 1)[1] + ) + + +def _records(chunks: dict[int, bytes]) -> dict: + return { + i: SimpleNamespace(chunk_hash=hashlib.sha256(d).hexdigest()) + for i, d in chunks.items() + } + + +async def _seed_chunks(s3_storage, upload_id, save_path, chunks: dict[int, bytes]): + for index, data in chunks.items(): + await s3_storage.save_chunk(upload_id, index, data, "ignored", save_path) + + +@pytest.mark.asyncio +class TestS3ObjectBasics: + async def test_save_and_exists_roundtrip(self, s3_storage): + await s3_storage.save_file( + io.BytesIO(b"payload"), "share/data/f.bin", "application/octet-stream" + ) + assert await s3_storage.file_exists("share/data/f.bin") is True + assert await s3_storage.file_exists("share/data/missing.bin") is False + + async def test_delete_file_removes_object(self, s3_storage): + await s3_storage.save_file(io.BytesIO(b"payload"), "share/data/f.bin") + await s3_storage.delete_file(_stored("share/data/f.bin")) + assert await s3_storage.file_exists("share/data/f.bin") is False + + async def test_presigned_upload_url_targets_bucket_and_key(self, s3_storage): + url = await s3_storage.generate_presigned_upload_url("share/data/f.bin", 900) + assert "drill-bucket" in url + assert "share/data/f.bin" in url # key 必须出现在 URL（path 或签名参数） + assert "X-Amz-Signature" in url + + async def test_get_file_url_proxy_uses_local_dispatch(self, s3_storage): + """proxy 模式必须走本地 /share/download 分发（取件计数/前置 404 生效）。""" + from core.settings import settings + + settings.user_config = {**settings.user_config, "s3_proxy": 1} + proxied_storage = S3FileStorage() + file_code = StoredFile( + file_path="share/data", uuid_file_name="f.bin", code="abcd1" + ) + proxied = await proxied_storage.get_file_url(file_code) + assert proxied.startswith("/share/download?") + + async def test_get_file_url_direct_returns_presigned(self, s3_storage): + file_code = StoredFile( + file_path="share/data", uuid_file_name="f.bin", code="abcd1" + ) + direct = await s3_storage.get_file_url(file_code) + assert direct.startswith("http") and "drill-bucket" in direct + + +@pytest.mark.asyncio +class TestS3GetFileResponse: + async def test_missing_object_raises_404_upfront(self, s3_storage): + """缺失对象必须前置 404，而不是签发 200 的坏流（与 local 后端语义对齐）。""" + with pytest.raises(StorageError) as exc_info: + await s3_storage.get_file_response(_stored("share/data/ghost.bin")) + assert exc_info.value.status_code == 404 + + async def test_existing_object_returns_download_with_length(self, s3_storage): + await s3_storage.save_file(io.BytesIO(b"payload"), "share/data/f.bin") + download = await s3_storage.get_file_response(_stored("share/data/f.bin")) + assert download.headers["Content-Length"] == "7" + assert download.headers["Content-Disposition"].startswith("attachment;") + + async def test_stream_factory_roundtrip_via_presigned_url(self, s3_storage): + """流式生成器经 presigned URL 真实往返 moto 服务，内容一致。""" + payload = b"stream-roundtrip-payload" + await s3_storage.save_file(io.BytesIO(payload), "share/data/s.bin") + download = await s3_storage.get_file_response(_stored("share/data/s.bin")) + chunks = [chunk async for chunk in download.stream_factory()] + assert b"".join(chunks) == payload + + +@pytest.mark.asyncio +class TestS3ChunkMerge: + async def test_merge_success_roundtrip_then_cleanup(self, s3_storage): + upload_id = "up01" + save_path = "share/data/2026/01/01/up01/merged.bin" + parts = {0: b"AAAA", 1: b"BBBB"} + await _seed_chunks(s3_storage, upload_id, save_path, parts) + _, file_hash = await s3_storage.merge_chunks( + upload_id, 2, 4, save_path, _records(parts) + ) + assert file_hash == hashlib.sha256(b"AAAABBBB").hexdigest() + + async with s3_storage._client() as s3: + resp = await s3.get_object(Bucket="drill-bucket", Key=save_path) + merged = await resp["Body"].read() + assert merged == b"AAAABBBB" + + await s3_storage.clean_chunks(upload_id, save_path) + assert ( + await s3_storage.file_exists( + f"share/data/2026/01/01/up01/chunks/{upload_id}/0.part" + ) + is False + ) + + async def test_merge_missing_chunk_aborts_and_raises(self, s3_storage): + upload_id = "up02" + save_path = "share/data/2026/01/01/up02/merged.bin" + await _seed_chunks(s3_storage, upload_id, save_path, {1: b"BBBB"}) # 缺分片 0 + with pytest.raises(ValueError, match="分片0"): + await s3_storage.merge_chunks( + upload_id, 2, 4, save_path, _records({0: b"AAAA", 1: b"BBBB"}) + ) + # abort 后不得留下半成品对象 + assert await s3_storage.file_exists(save_path) is False + + async def test_merge_hash_mismatch_aborts_and_raises(self, s3_storage): + upload_id = "up03" + save_path = "share/data/2026/01/01/up03/merged.bin" + await _seed_chunks(s3_storage, upload_id, save_path, {0: b"AAAA"}) + bad_records = {0: SimpleNamespace(chunk_hash="f" * 64)} # 与实际不符 + with pytest.raises(ValueError, match="哈希不匹配"): + await s3_storage.merge_chunks(upload_id, 1, 4, save_path, bad_records) + assert await s3_storage.file_exists(save_path) is False + + async def test_clean_chunks_scoped_to_upload_only(self, s3_storage): + # 注意 chunk 目录派生自 save_path 的父目录——两个 upload 必须各自 save_path + save_a = "share/data/2026/01/01/upa/merged.bin" + save_b = "share/data/2026/01/01/upb/merged.bin" + await _seed_chunks(s3_storage, "upa", save_a, {0: b"AA"}) + await _seed_chunks(s3_storage, "upb", save_b, {0: b"BB"}) + await s3_storage.clean_chunks("upa", save_a) + assert ( + await s3_storage.file_exists("share/data/2026/01/01/upa/chunks/upa/0.part") + is False + ) + assert ( + await s3_storage.file_exists("share/data/2026/01/01/upb/chunks/upb/0.part") + is True + ) + + async def test_save_chunk_stores_declared_hash_metadata(self, s3_storage): + await s3_storage.save_chunk("upm", 0, b"DATA", "cafe" * 16, "share/data/x/m.bin") + async with s3_storage._client() as s3: + head = await s3.head_object( + Bucket="drill-bucket", Key="share/data/x/chunks/upm/0.part" + ) + assert head["Metadata"]["chunk-hash"] == "cafe" * 16 diff --git a/tests/test_security_hardening.py b/tests/test_security_hardening.py index 574a26722..23af51b7d 100644 --- a/tests/test_security_hardening.py +++ b/tests/test_security_hardening.py @@ -125,7 +125,7 @@ async def _run_traversal_upload(self): try: settings.file_storage = "local" settings.allowed_file_types = ["*"] - with patch("core.storage.data_root", Path(tmpdir.name)): + with patch("core.storage.local.data_root", Path(tmpdir.name)): await init_memory_db() try: raw_name = "../../../../../../filecodebox.db" diff --git a/tests/test_webdav_storage.py b/tests/test_webdav_storage.py new file mode 100644 index 000000000..56915218a --- /dev/null +++ b/tests/test_webdav_storage.py @@ -0,0 +1,234 @@ +"""WebDAV storage backend coverage against a real in-process WebDAV server. + +aioboto3-style mocking cannot fake aiohttp; the backend speaks real HTTP +(HEAD/GET/PUT/DELETE/MKCOL/PROPFIND), so the fixture boots a minimal WebDAV +server over a temp directory and exercises the backend through the wire. +Focus on failure branches: missing objects, merge failures, cleanup scoping, +connection errors. +""" +import hashlib +import io +import shutil +import tempfile +from pathlib import Path +from types import SimpleNamespace + +import pytest +from aiohttp import web + +from core.errors import StorageError +from core.storage import StoredFile, WebDAVFileStorage + + +def make_dav_app(root: Path) -> web.Application: + """Minimal WebDAV server: HEAD/GET/PUT/DELETE/MKCOL/PROPFIND over `root`.""" + + def _fs_path(request: web.Request) -> Path | None: + rel = request.match_info.get("path", "") + target = (root / rel).resolve() + if not str(target).startswith(str(root)): + return None + return target + + async def handler(request: web.Request) -> web.StreamResponse: + target = _fs_path(request) + if target is None: + return web.Response(status=403) + method = request.method + + if method == "HEAD": + if target.is_file(): + return web.Response( + status=200, headers={"Content-Length": str(target.stat().st_size)} + ) + if target.is_dir(): + return web.Response(status=200) + return web.Response(status=404) + + if method == "GET": + if target.is_file(): + return web.FileResponse(target) + return web.Response(status=404) + + if method == "PUT": + if target.is_dir(): + return web.Response(status=409) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(await request.read()) + return web.Response(status=201) + + if method == "MKCOL": + if target.exists(): + return web.Response(status=405) + if not target.parent.exists(): + return web.Response(status=409) + target.mkdir() + return web.Response(status=201) + + if method == "DELETE": + if target.is_file(): + target.unlink() + return web.Response(status=204) + if target.is_dir(): + shutil.rmtree(target) + return web.Response(status=204) + return web.Response(status=404) + + if method == "PROPFIND": + if not target.exists() or not target.is_dir(): + return web.Response(status=404) + hrefs = [p.relative_to(root).as_posix() for p in target.iterdir()] + body = "".join( + f"{h}" for h in hrefs + ) + xml = f"{body}" + return web.Response(status=207, text=xml, content_type="application/xml") + + return web.Response(status=405) + + app = web.Application() + app.router.add_route("*", "/{path:.*}", handler) + return app + + +@pytest.fixture +async def dav_storage(): + from core.settings import settings + from aiohttp.test_utils import TestServer + + root = Path(tempfile.mkdtemp(prefix="fcb-dav-")) + app = make_dav_app(root) + server = TestServer(app) + await server.start_server() + endpoint = f"http://127.0.0.1:{server.port}/dav/" + + original = dict(settings.user_config) + settings.user_config = { + **original, + "file_storage": "webdav", + "webdav_url": endpoint, + "webdav_username": "user", + "webdav_password": "pass", + } + try: + yield WebDAVFileStorage(), root + finally: + settings.user_config = original + await server.close() + shutil.rmtree(root, ignore_errors=True) + + +def _stored(key: str) -> StoredFile: + return StoredFile( + file_path=key.rsplit("/", 1)[0], uuid_file_name=key.rsplit("/", 1)[1] + ) + + +def _records(chunks: dict[int, bytes]) -> dict: + return { + i: SimpleNamespace(chunk_hash=hashlib.sha256(d).hexdigest()) + for i, d in chunks.items() + } + + +@pytest.mark.asyncio +class TestWebDAVObjectBasics: + async def test_save_roundtrip_and_exists(self, dav_storage): + storage, _ = dav_storage + await storage.save_file( + io.BytesIO(b"payload"), "share/data/f.bin", "application/octet-stream" + ) + assert await storage.file_exists("share/data/f.bin") is True + assert await storage.file_exists("share/data/ghost.bin") is False + + async def test_delete_file_and_empty_parent_dirs(self, dav_storage): + storage, _ = dav_storage + await storage.save_file(io.BytesIO(b"payload"), "share/data/2026/01/01/x/f.bin") + await storage.delete_file(_stored("share/data/2026/01/01/x/f.bin")) + assert await storage.file_exists("share/data/2026/01/01/x/f.bin") is False + # 空父目录被逐级清理（file_exists 对目录 HEAD 也是 404） + assert await storage.file_exists("share/data/2026/01/01/x") is False + + async def test_delete_missing_file_is_accepted(self, dav_storage): + storage, _ = dav_storage + await storage.delete_file(_stored("share/data/never-existed.bin")) + + async def test_get_file_response_roundtrip(self, dav_storage): + storage, _ = dav_storage + payload = b"dav-stream-payload" + await storage.save_file(io.BytesIO(payload), "share/data/s.bin") + download = await storage.get_file_response(_stored("share/data/s.bin")) + assert download.headers["Content-Length"] == str(len(payload)) + chunks = [chunk async for chunk in download.stream_factory()] + assert b"".join(chunks) == payload + + +@pytest.mark.asyncio +class TestWebDAVGetFileResponseFailures: + async def test_missing_object_raises_404_upfront(self, dav_storage): + """缺失对象前置 404，与 local/S3 语义对齐（不许 200 坏流）。""" + storage, _ = dav_storage + with pytest.raises(StorageError) as exc_info: + await storage.get_file_response(_stored("share/data/ghost.bin")) + assert exc_info.value.status_code == 404 + + async def test_connection_error_maps_to_503(self, monkeypatch): + from core.settings import settings + + original = dict(settings.user_config) + settings.user_config = { + **original, + "webdav_url": "http://127.0.0.1:1/", # 无服务的端口 + "webdav_username": "u", + "webdav_password": "p", + } + try: + storage = WebDAVFileStorage() + with pytest.raises(StorageError) as exc_info: + await storage.get_file_response(_stored("share/data/x.bin")) + assert exc_info.value.status_code == 503 + finally: + settings.user_config = original + + +@pytest.mark.asyncio +class TestWebDAVChunkMerge: + async def test_merge_success_and_cleanup(self, dav_storage): + storage, _ = dav_storage + upload_id = "up01" + save_path = "share/data/2026/01/01/up01/merged.bin" + parts = {0: b"AAAA", 1: b"BBBB"} + for i, data in parts.items(): + await storage.save_chunk(upload_id, i, data, "ignored", save_path) + _, file_hash = await storage.merge_chunks( + upload_id, 2, 4, save_path, _records(parts) + ) + assert file_hash == hashlib.sha256(b"AAAABBBB").hexdigest() + assert await storage.file_exists(save_path) is True + + await storage.clean_chunks(upload_id, save_path) + assert ( + await storage.file_exists( + f"share/data/2026/01/01/up01/chunks/{upload_id}/0.part" + ) + is False + ) + + async def test_merge_missing_chunk_raises(self, dav_storage): + storage, _ = dav_storage + save_path = "share/data/2026/01/01/up02/merged.bin" + await storage.save_chunk("up02", 1, b"BBBB", "ignored", save_path) # 缺分片 0 + with pytest.raises(ValueError, match="分片0"): + await storage.merge_chunks( + "up02", 2, 4, save_path, _records({0: b"AAAA", 1: b"BBBB"}) + ) + assert await storage.file_exists(save_path) is False + + async def test_merge_hash_mismatch_raises(self, dav_storage): + storage, _ = dav_storage + save_path = "share/data/2026/01/01/up03/merged.bin" + await storage.save_chunk("up03", 0, b"AAAA", "ignored", save_path) + bad = {0: SimpleNamespace(chunk_hash="f" * 64)} + with pytest.raises(ValueError, match="哈希不匹配"): + await storage.merge_chunks("up03", 1, 4, save_path, bad) + assert await storage.file_exists(save_path) is False