diff --git a/tests/integration/test_mcp_bridge_routes.py b/tests/integration/test_mcp_bridge_routes.py new file mode 100644 index 0000000..25d3d97 --- /dev/null +++ b/tests/integration/test_mcp_bridge_routes.py @@ -0,0 +1,270 @@ +"""Integration tests for MCP HTTP bridge routes.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path + +import pytest +import yaml +from httpx import ASGITransport, AsyncClient + +from web2api.main import create_app +from web2api.schemas import ( + ApiResponse, + ErrorCode, + ErrorResponse, + MetadataResponse, + PaginationResponse, + SiteInfo, +) + + +class FakePool: + """Pool stub used for MCP bridge route tests.""" + + async def start(self) -> None: + return None + + async def stop(self) -> None: + return None + + @property + def health(self) -> dict[str, int | bool]: + return { + "browser_connected": True, + "total_contexts": 1, + "available_contexts": 1, + "queue_size": 0, + "total_requests_served": 0, + } + + +def _write_recipe(recipes_dir: Path, slug: str) -> None: + recipe_dir = recipes_dir / slug + recipe_dir.mkdir(parents=True, exist_ok=True) + (recipe_dir / "recipe.yaml").write_text( + yaml.safe_dump( + { + "name": slug.title(), + "slug": slug, + "base_url": "https://example.com", + "description": f"{slug} fixture recipe", + "endpoints": { + "read": { + "url": "https://example.com/items?page={page}", + "items": { + "container": ".item", + "fields": {"title": {"selector": ".title"}}, + }, + "pagination": {"type": "page_param", "param": "page"}, + }, + }, + } + ), + encoding="utf-8", + ) + + +def _success_response(*, slug: str, endpoint: str, page: int) -> ApiResponse: + return ApiResponse( + site=SiteInfo(name=slug.title(), slug=slug, url="https://example.com"), + endpoint=endpoint, + query=None, + items=[], + pagination=PaginationResponse( + current_page=page, + has_next=False, + has_prev=page > 1, + total_pages=None, + total_items=None, + ), + metadata=MetadataResponse( + scraped_at=datetime.now(UTC), + response_time_ms=1, + item_count=0, + cached=False, + ), + error=None, + ) + + +def _error_response(*, slug: str, endpoint: str, page: int, code: ErrorCode) -> ApiResponse: + return ApiResponse( + site=SiteInfo(name=slug.title(), slug=slug, url="https://example.com"), + endpoint=endpoint, + query=None, + items=[], + pagination=PaginationResponse( + current_page=page, + has_next=False, + has_prev=page > 1, + total_pages=None, + total_items=None, + ), + metadata=MetadataResponse( + scraped_at=datetime.now(UTC), + response_time_ms=1, + item_count=0, + cached=False, + ), + error=ErrorResponse(code=code, message="boom", details=None), + ) + + +@pytest.mark.asyncio +async def test_mcp_call_tool_honors_page_param( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + recipes_dir = tmp_path / "recipes" + _write_recipe(recipes_dir, "demo") + + seen_pages: list[int] = [] + + async def fake_scrape( + *, + pool: FakePool, + recipe, + endpoint: str, + page: int = 1, + query: str | None = None, + extra_params: dict[str, str] | None = None, + scrape_timeout: float = 30.0, + ) -> ApiResponse: + _ = pool, query, extra_params, scrape_timeout + seen_pages.append(page) + return _success_response(slug=recipe.config.slug, endpoint=endpoint, page=page) + + monkeypatch.setattr("web2api.main.scrape", fake_scrape) + + app = create_app(recipes_dir=recipes_dir, pool=FakePool()) + async with app.router.lifespan_context(app): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + resp = await client.post("/mcp/tools/demo_read", json={"page": 3}) + + assert resp.status_code == 200 + assert seen_pages == [3] + + +@pytest.mark.asyncio +async def test_mcp_call_tool_error_uses_mapped_status_code( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + recipes_dir = tmp_path / "recipes" + _write_recipe(recipes_dir, "demo") + + async def fake_scrape( + *, + pool: FakePool, + recipe, + endpoint: str, + page: int = 1, + query: str | None = None, + extra_params: dict[str, str] | None = None, + scrape_timeout: float = 30.0, + ) -> ApiResponse: + _ = pool, query, extra_params, scrape_timeout + return _error_response( + slug=recipe.config.slug, + endpoint=endpoint, + page=page, + code="INVALID_PARAMS", + ) + + monkeypatch.setattr("web2api.main.scrape", fake_scrape) + + app = create_app(recipes_dir=recipes_dir, pool=FakePool()) + async with app.router.lifespan_context(app): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + resp = await client.post("/mcp/tools/demo_read", json={"q": "x"}) + + assert resp.status_code == 400 + assert resp.json()["result"] == "Error: boom" + + +@pytest.mark.asyncio +async def test_mcp_call_tool_filtered_enforces_exclude( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + recipes_dir = tmp_path / "recipes" + _write_recipe(recipes_dir, "demo") + + called = False + + async def fake_scrape( + *, + pool: FakePool, + recipe, + endpoint: str, + page: int = 1, + query: str | None = None, + extra_params: dict[str, str] | None = None, + scrape_timeout: float = 30.0, + ) -> ApiResponse: + _ = pool, query, extra_params, scrape_timeout + nonlocal called + called = True + return _success_response(slug=recipe.config.slug, endpoint=endpoint, page=page) + + monkeypatch.setattr("web2api.main.scrape", fake_scrape) + + app = create_app(recipes_dir=recipes_dir, pool=FakePool()) + async with app.router.lifespan_context(app): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + resp = await client.post( + "/mcp/exclude/demo/tools/demo_read", + json={"q": "x"}, + ) + + assert resp.status_code == 404 + assert called is False + + +@pytest.mark.asyncio +async def test_mcp_call_tool_filtered_enforces_only( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + recipes_dir = tmp_path / "recipes" + _write_recipe(recipes_dir, "demo") + + seen_pages: list[int] = [] + + async def fake_scrape( + *, + pool: FakePool, + recipe, + endpoint: str, + page: int = 1, + query: str | None = None, + extra_params: dict[str, str] | None = None, + scrape_timeout: float = 30.0, + ) -> ApiResponse: + _ = pool, query, extra_params, scrape_timeout + seen_pages.append(page) + return _success_response(slug=recipe.config.slug, endpoint=endpoint, page=page) + + monkeypatch.setattr("web2api.main.scrape", fake_scrape) + + app = create_app(recipes_dir=recipes_dir, pool=FakePool()) + async with app.router.lifespan_context(app): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + blocked = await client.post( + "/mcp/only/other/tools/demo_read", + json={"q": "x"}, + ) + allowed = await client.post( + "/mcp/only/demo/tools/demo_read", + json={"page": 2}, + ) + + assert blocked.status_code == 404 + assert allowed.status_code == 200 + assert seen_pages == [2] diff --git a/tests/unit/test_mcp_bridge.py b/tests/unit/test_mcp_bridge.py new file mode 100644 index 0000000..50cbc72 --- /dev/null +++ b/tests/unit/test_mcp_bridge.py @@ -0,0 +1,79 @@ +"""Unit tests for MCP HTTP bridge tool resolution.""" + +from __future__ import annotations + +from pathlib import Path + +from web2api.config import RecipeConfig +from web2api.mcp_bridge import _resolve_tool +from web2api.registry import Recipe + + +class _FakeRegistry: + """Minimal registry stub exposing ``list_all``.""" + + def __init__(self, recipes: list[Recipe]) -> None: + self._recipes = recipes + + def list_all(self) -> list[Recipe]: + return self._recipes + + +def _endpoint(**overrides: object) -> dict[str, object]: + endpoint: dict[str, object] = { + "url": "https://example.com/items?page={page}", + "items": {"container": ".item", "fields": {"title": {"selector": ".title"}}}, + "pagination": {"type": "page_param", "param": "page"}, + } + endpoint.update(overrides) + return endpoint + + +def _recipe(slug: str, endpoints: dict[str, object]) -> Recipe: + config = RecipeConfig.model_validate( + { + "name": slug.title(), + "slug": slug, + "base_url": "https://example.com", + "description": "fixture recipe", + "endpoints": endpoints, + } + ) + return Recipe(config=config, scraper=None, path=Path(f"recipes/{slug}")) + + +def test_resolve_tool_standard_naming() -> None: + registry = _FakeRegistry([_recipe("demo", {"read": _endpoint()})]) + + assert _resolve_tool(registry, "demo__read") == ("demo", "read") + + +def test_resolve_tool_legacy_single_underscore_name() -> None: + registry = _FakeRegistry([_recipe("demo", {"read": _endpoint()})]) + + assert _resolve_tool(registry, "demo_read") == ("demo", "read") + + +def test_resolve_tool_handles_underscores_in_slug_and_endpoint() -> None: + registry = _FakeRegistry([_recipe("my_site", {"deep_read": _endpoint()})]) + + assert _resolve_tool(registry, "my_site__deep_read") == ("my_site", "deep_read") + + +def test_resolve_tool_handles_double_underscores_in_slug() -> None: + registry = _FakeRegistry([_recipe("my__site", {"read": _endpoint()})]) + + # Splitting on "__" would mis-parse this as ("my", "site__read"). + assert _resolve_tool(registry, "my__site__read") == ("my__site", "read") + + +def test_resolve_tool_custom_tool_name_override() -> None: + registry = _FakeRegistry([_recipe("allenai", {"molmo2": _endpoint(tool_name="molmo2_vision")})]) + + assert _resolve_tool(registry, "molmo2_vision") == ("allenai", "molmo2") + + +def test_resolve_tool_unknown_name_returns_none() -> None: + registry = _FakeRegistry([_recipe("demo", {"read": _endpoint()})]) + + assert _resolve_tool(registry, "missing_tool") == (None, None) diff --git a/tests/unit/test_schemas.py b/tests/unit/test_schemas.py new file mode 100644 index 0000000..43114be --- /dev/null +++ b/tests/unit/test_schemas.py @@ -0,0 +1,35 @@ +"""Unit tests for unified schema helpers.""" + +from __future__ import annotations + +import pytest + +from web2api.schemas import ErrorResponse, status_code_for_error + + +def _error(code: str) -> ErrorResponse: + return ErrorResponse(code=code, message="boom", details=None) + + +def test_status_code_for_error_returns_200_when_no_error() -> None: + assert status_code_for_error(None) == 200 + + +@pytest.mark.parametrize( + ("code", "expected"), + [ + ("SITE_NOT_FOUND", 404), + ("CAPABILITY_NOT_SUPPORTED", 400), + ("INVALID_PARAMS", 400), + ("SCRAPE_FAILED", 502), + ("SCRAPE_TIMEOUT", 504), + ("INTERNAL_ERROR", 500), + ], +) +def test_status_code_mapping(code: str, expected: int) -> None: + assert status_code_for_error(_error(code)) == expected + + +def test_status_code_defaults_to_500_for_unknown_code() -> None: + error = ErrorResponse.model_construct(code="UNKNOWN_CODE", message="boom", details=None) + assert status_code_for_error(error) == 500 diff --git a/tests/unit/test_scraper.py b/tests/unit/test_scraper.py new file mode 100644 index 0000000..ee64793 --- /dev/null +++ b/tests/unit/test_scraper.py @@ -0,0 +1,41 @@ +"""Unit tests for scraper base helpers (coercion + InvalidParamsError).""" + +from __future__ import annotations + +import pytest + +from web2api.scraper import InvalidParamsError, coerce_float, coerce_int + + +def test_coerce_int_parses_values() -> None: + assert coerce_int("20", name="count", default=10) == 20 + assert coerce_int(7, name="count", default=10) == 7 + + +def test_coerce_int_uses_default_for_missing_values() -> None: + assert coerce_int(None, name="count", default=10) == 10 + assert coerce_int("", name="count", default=10) == 10 + + +def test_coerce_int_raises_invalid_params_on_bad_input() -> None: + with pytest.raises(InvalidParamsError, match="count"): + coerce_int("abc", name="count", default=10) + + +def test_coerce_float_parses_values() -> None: + assert coerce_float("0.5", name="temperature") == 0.5 + assert coerce_float(2, name="temperature") == 2.0 + + +def test_coerce_float_uses_default_for_missing_values() -> None: + assert coerce_float(None, name="temperature") is None + assert coerce_float("", name="temperature", default=1.0) == 1.0 + + +def test_coerce_float_raises_invalid_params_on_bad_input() -> None: + with pytest.raises(InvalidParamsError, match="temperature"): + coerce_float("not-a-number", name="temperature") + + +def test_invalid_params_error_is_a_value_error() -> None: + assert issubclass(InvalidParamsError, ValueError) diff --git a/web2api/cache.py b/web2api/cache.py index a135fc5..00239b8 100644 --- a/web2api/cache.py +++ b/web2api/cache.py @@ -105,6 +105,11 @@ async def trigger_refresh( self._refresh_tasks.add(task) task.add_done_callback(self._refresh_tasks.discard) + async def clear(self) -> None: + """Drop all cached entries (e.g., after recipes change).""" + async with self._lock: + self._entries.clear() + async def stats(self) -> dict[str, int | float | bool]: """Return cache health and counters for diagnostics.""" now = monotonic() diff --git a/web2api/cli.py b/web2api/cli.py index 803d383..ff72667 100644 --- a/web2api/cli.py +++ b/web2api/cli.py @@ -17,6 +17,7 @@ build_dockerfile_snippet, build_entry_payload, build_install_commands, + catalog_entry_is_trusted, default_catalog_path, default_catalog_ref, default_catalog_source, @@ -783,7 +784,7 @@ def recipes_catalog_add( typer.echo(f"Catalog entry '{name}' was not found in {source_value}.", err=True) raise typer.Exit(code=1) - trusted_value = trusted or bool(spec.trusted) + trusted_value = trusted or catalog_entry_is_trusted(spec.trusted) target_dir = _recipes_dir_option(recipes_dir) diff --git a/web2api/engine.py b/web2api/engine.py index 8db2264..1361557 100644 --- a/web2api/engine.py +++ b/web2api/engine.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import json import logging import re from collections.abc import Callable @@ -27,7 +28,7 @@ PaginationResponse, SiteInfo, ) -from web2api.scraper import ScrapeResult +from web2api.scraper import InvalidParamsError, ScrapeResult logger = logging.getLogger(__name__) TransformHandler = Callable[[str, str], Any] @@ -191,6 +192,12 @@ async def detect_pagination( """Detect pagination values after extraction.""" has_prev = current_page > 1 if pagination.type in {"page_param", "offset_param"}: + # For param-based pagination a next-page selector is optional. When + # provided it gives an accurate ``has_next``; otherwise we fall back + # to the only available signal: the page produced at least one item. + if pagination.selector: + next_link = await page.query_selector(pagination.selector) + return next_link is not None, has_prev, None, None return item_count > 0, has_prev, None, None next_link = await page.query_selector(pagination.selector) @@ -325,6 +332,16 @@ async def _do_scrape() -> ScrapeResult: code="SCRAPE_TIMEOUT", message=f"scrape exceeded {scrape_timeout}s timeout", ) + except InvalidParamsError as exc: + return _error_response( + recipe=recipe, + endpoint=endpoint, + query=query, + current_page=current_page, + started_at=started_at, + code="INVALID_PARAMS", + message=str(exc), + ) except Exception as exc: # noqa: BLE001 return _error_response( recipe=recipe, @@ -452,7 +469,11 @@ async def _run_custom_scraper( def _normalize_items(raw_items: list[dict[str, Any]]) -> list[ItemResponse]: items: list[ItemResponse] = [] for raw_item in raw_items: - fields = {key: value for key, value in raw_item.items() if key not in {"title", "url"}} + fields: dict[str, Any] = { + key: _normalize_field_value(value) + for key, value in raw_item.items() + if key not in {"title", "url"} + } title = raw_item.get("title") url = raw_item.get("url") items.append( @@ -465,6 +486,18 @@ def _normalize_items(raw_items: list[dict[str, Any]]) -> list[ItemResponse]: return items +def _normalize_field_value(value: Any) -> Any: + """Coerce a scraped field value into the scalar-only ``FieldValue`` type. + + Custom scrapers may return nested lists/dicts; ``ItemResponse.fields`` + only allows scalars, so serialize anything non-scalar to a JSON string + instead of failing response validation. + """ + if value is None or isinstance(value, (str, int, float, bool)): + return value + return json.dumps(value, ensure_ascii=False, default=str) + + def _to_iso_date(value: str) -> str | None: value = value.strip() for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%b %d, %Y", "%B %d, %Y"): diff --git a/web2api/main.py b/web2api/main.py index d2fce5c..4472359 100644 --- a/web2api/main.py +++ b/web2api/main.py @@ -49,6 +49,7 @@ MetadataResponse, PaginationResponse, SiteInfo, + status_code_for_error, ) TEMPLATES = Jinja2Templates(directory=str(Path(__file__).resolve().parent / "templates")) @@ -71,20 +72,6 @@ def _env_bool(name: str, *, default: bool) -> bool: return raw.strip().lower() in {"1", "true", "yes", "on"} -def _status_code_for_error(error: ErrorResponse | None) -> int: - """Map unified API error payloads to HTTP status codes.""" - if error is None: - return 200 - return { - "SITE_NOT_FOUND": 404, - "CAPABILITY_NOT_SUPPORTED": 400, - "INVALID_PARAMS": 400, - "SCRAPE_FAILED": 502, - "SCRAPE_TIMEOUT": 504, - "INTERNAL_ERROR": 500, - }.get(error.code, 500) - - def _site_payload(recipe: Recipe) -> dict[str, Any]: """Build the site metadata payload returned by discovery endpoints.""" config = recipe.config @@ -126,6 +113,8 @@ def _build_error_response( code: ErrorCode, message: str, ) -> ApiResponse: + endpoint_config = recipe.config.endpoints.get(endpoint) + requires_query = endpoint_config.requires_query if endpoint_config is not None else False return ApiResponse( site=SiteInfo( name=recipe.config.name, @@ -133,7 +122,7 @@ def _build_error_response( url=recipe.config.base_url, ), endpoint=endpoint, - query=query if recipe.config.endpoints[endpoint].requires_query else None, + query=query if requires_query else None, items=[], pagination=PaginationResponse( current_page=current_page, @@ -309,7 +298,7 @@ async def _serve_recipe_endpoint( ) return JSONResponse( content=response.model_dump(mode="json"), - status_code=_status_code_for_error(response.error), + status_code=status_code_for_error(response.error), ) @@ -666,7 +655,7 @@ async def recipe_endpoint_post( endpoint: str, page: int = Query(default=1, ge=1), q: str | None = Query(default=None, max_length=_MAX_QUERY_LENGTH), - files: list[UploadFile] = File(default=[]), + files: list[UploadFile] | None = File(default=None), ) -> JSONResponse: """Serve recipe endpoints with file upload support (POST multipart).""" registry_state: RecipeRegistry = request.app.state.registry diff --git a/web2api/mcp_bridge.py b/web2api/mcp_bridge.py index f691d64..efa5725 100644 --- a/web2api/mcp_bridge.py +++ b/web2api/mcp_bridge.py @@ -16,8 +16,9 @@ from fastapi import FastAPI, HTTPException, Request from fastapi.responses import JSONResponse -from web2api.mcp_utils import build_tool_name, parse_tool_name +from web2api.mcp_utils import build_tool_name from web2api.registry import RecipeRegistry +from web2api.schemas import status_code_for_error logger = logging.getLogger(__name__) @@ -25,25 +26,19 @@ def _resolve_tool(registry: RecipeRegistry, tool_name: str) -> tuple[str | None, str | None]: """Resolve a tool name to (slug, endpoint_name). - Checks custom ``tool_name`` overrides first, then falls back to the - standard ``{slug}__{endpoint}`` naming convention. Legacy single-underscore - names are still accepted for existing clients. + Matches by recomputing each endpoint's tool name via ``build_tool_name``, + which handles both custom ``tool_name`` overrides and the standard + ``{slug}__{endpoint}`` convention — robust to slugs/endpoints that + themselves contain underscores. Legacy single-underscore names remain + accepted for existing clients. """ - # Check custom tool_name overrides for recipe in registry.list_all(): slug = recipe.config.slug for ep_name, ep_cfg in recipe.config.endpoints.items(): - if ep_cfg.tool_name and ep_cfg.tool_name == tool_name: + if build_tool_name(slug, ep_name, ep_cfg.tool_name) == tool_name: return slug, ep_name - # Fall back to standard naming - parsed = parse_tool_name(tool_name) - if parsed: - slug, ep_name = parsed - recipe = registry.get(slug) - if recipe and ep_name in recipe.config.endpoints: - return slug, ep_name - + # Legacy single-underscore names for backward compatibility. for recipe in registry.list_all(): for ep_name in recipe.config.endpoints: if tool_name == f"{recipe.config.slug}_{ep_name}": @@ -188,7 +183,16 @@ async def mcp_call_tool_filtered( filter_value: str, tool_name: str, ) -> JSONResponse: - """Call a tool via the filtered MCP path (routing is the same).""" + """Call a tool via the filtered MCP path, enforcing the filter.""" + registry: RecipeRegistry = request.app.state.registry + slugs = {s.strip() for s in filter_value.split(",") if s.strip()} + tool_slug = _tool_slug(registry, tool_name) + + if filter_type == "only" and (tool_slug is None or tool_slug not in slugs): + raise HTTPException(status_code=404, detail=f"Tool not found: {tool_name}") + if filter_type == "exclude" and tool_slug is not None and tool_slug in slugs: + raise HTTPException(status_code=404, detail=f"Tool not found: {tool_name}") + return await mcp_call_tool(request, tool_name) @app.post("/mcp/tools/{tool_name}") @@ -250,7 +254,10 @@ async def mcp_call_tool( error = response_data.get("error") if error: - return JSONResponse({"result": f"Error: {error.get('message', 'unknown error')}"}) + return JSONResponse( + {"result": f"Error: {error.get('message', 'unknown error')}"}, + status_code=status_code_for_error(response.error), + ) if len(items) == 1: fields = items[0].get("fields", {}) diff --git a/web2api/mcp_server.py b/web2api/mcp_server.py index fc9fc15..a499dd9 100644 --- a/web2api/mcp_server.py +++ b/web2api/mcp_server.py @@ -127,6 +127,7 @@ def _register_tool( param_docs: list[str] = [] if requires_q: param_docs.append("q: The search query or prompt (required)") + param_docs.append("page: 1-based page number (optional, default 1)") for pname, pcfg in extra_params.items(): pdesc = pcfg.get("description", "") suffix = " (required)" if pcfg.get("required") else " (optional)" diff --git a/web2api/recipe_admin_api.py b/web2api/recipe_admin_api.py index 1586a6c..1157e93 100644 --- a/web2api/recipe_admin_api.py +++ b/web2api/recipe_admin_api.py @@ -13,6 +13,7 @@ from web2api.recipe_manager import ( build_entry_payload, + catalog_entry_is_trusted, check_recipe_updates, disable_recipe, discover_recipe_entries, @@ -53,6 +54,12 @@ async def _reload_registry_and_tools(app: FastAPI, *, app_version: str) -> None: ) app.state.registry = registry + # Drop cached responses so updated/installed recipes are served fresh. + response_cache = getattr(app.state, "response_cache", None) + clear_cache = getattr(response_cache, "clear", None) + if clear_cache is not None: + await clear_cache() + # Rebuild MCP tools so connected clients see the change try: from web2api.mcp_server import rebuild_mcp_tools @@ -172,7 +179,7 @@ async def recipes_manage_install(name: str, request: Request) -> JSONResponse: recipes_dir=request.app.state.recipes_dir, source_ref=spec.source_ref, source_subdir=spec.source_subdir, - trusted=bool(spec.trusted), + trusted=catalog_entry_is_trusted(spec.trusted), overwrite=False, record_source_type="catalog", ) diff --git a/web2api/recipe_manager.py b/web2api/recipe_manager.py index 47c4c77..3affadf 100644 --- a/web2api/recipe_manager.py +++ b/web2api/recipe_manager.py @@ -214,6 +214,16 @@ def entry_is_trusted(entry_record: dict[str, Any] | None) -> bool: return False +def catalog_entry_is_trusted(trusted: bool | None) -> bool: + """Resolve catalog trust with an explicit, safe default. + + A catalog entry without an explicit ``trusted`` flag is treated as + untrusted (``False``). This mirrors the install-time behavior used by + both the CLI and the admin API so the two cannot diverge. + """ + return trusted is True + + def recipe_origin(source_type: str | None) -> str: """Return normalized recipe origin from source type.""" if isinstance(source_type, str) and source_type in {"catalog", "git", "local"}: @@ -846,58 +856,6 @@ def compute_tree_hash(repo_dir: Path, subdir: str | None = None) -> str | None: return None -def fetch_remote_tree_hash( - source: str, - source_ref: str | None = None, - source_subdir: str | None = None, -) -> str | None: - """Fetch tree hash for a recipe directory from a remote git source. - - Does a lightweight fetch (``--depth 1``, ``--filter=blob:none``) to a temp - dir, then resolves the tree hash. Returns ``None`` on failure. - """ - with tempfile.TemporaryDirectory(prefix="web2api-hash-check-") as tmp_dir: - target = Path(tmp_dir) / "repo" - try: - subprocess.run( - ["git", "init", "--quiet", str(target)], - check=True, - text=True, - capture_output=True, - ) - subprocess.run( - ["git", "-C", str(target), "remote", "add", "origin", source], - check=True, - text=True, - capture_output=True, - ) - fetch_ref = source_ref or "HEAD" - subprocess.run( - [ - "git", "-C", str(target), - "fetch", "--quiet", "--depth", "1", "--filter=blob:none", - "origin", fetch_ref, - ], - check=True, - text=True, - capture_output=True, - ) - ref = "FETCH_HEAD^{tree}" - if source_subdir and source_subdir not in (".", ""): - cleaned = source_subdir.strip("/") - if cleaned and cleaned != ".": - ref = f"FETCH_HEAD:{cleaned}" - result = subprocess.run( - ["git", "-C", str(target), "rev-parse", ref], - check=True, - text=True, - capture_output=True, - ) - return result.stdout.strip() or None - except (subprocess.CalledProcessError, FileNotFoundError): - return None - - def check_recipe_updates(recipes_dir: Path) -> dict[str, bool | None]: """Check all managed git-sourced recipes for updates. diff --git a/web2api/schemas.py b/web2api/schemas.py index 1c1c2a0..af68b77 100644 --- a/web2api/schemas.py +++ b/web2api/schemas.py @@ -83,3 +83,17 @@ class ApiResponse(BaseModel): pagination: PaginationResponse metadata: MetadataResponse error: ErrorResponse | None = None + + +def status_code_for_error(error: ErrorResponse | None) -> int: + """Map unified API error payloads to HTTP status codes.""" + if error is None: + return 200 + return { + "SITE_NOT_FOUND": 404, + "CAPABILITY_NOT_SUPPORTED": 400, + "INVALID_PARAMS": 400, + "SCRAPE_FAILED": 502, + "SCRAPE_TIMEOUT": 504, + "INTERNAL_ERROR": 500, + }.get(error.code, 500) diff --git a/web2api/scraper.py b/web2api/scraper.py index a4323c6..2e86a28 100644 --- a/web2api/scraper.py +++ b/web2api/scraper.py @@ -21,6 +21,43 @@ class ScrapeResult: total_items: int | None = None +class InvalidParamsError(ValueError): + """Raised by a scraper when request parameters are malformed. + + The scraping engine maps this to an ``INVALID_PARAMS`` error response + instead of the generic ``SCRAPE_FAILED``. + """ + + +def coerce_int(value: Any, *, name: str, default: int) -> int: + """Coerce a user-supplied parameter to ``int``, raising on bad input.""" + if value in (None, ""): + return default + try: + return int(value) + except (TypeError, ValueError) as exc: + raise InvalidParamsError( + f"invalid {name} parameter: {value!r} (expected an integer)" + ) from exc + + +def coerce_float( + value: Any, + *, + name: str, + default: float | None = None, +) -> float | None: + """Coerce a user-supplied parameter to ``float``, raising on bad input.""" + if value in (None, ""): + return default + try: + return float(value) + except (TypeError, ValueError) as exc: + raise InvalidParamsError( + f"invalid {name} parameter: {value!r} (expected a number)" + ) from exc + + class BaseScraper(ABC): """Base class for optional recipe-specific scraper implementations.