Rework 0.5.0: harden runtime and contracts

.dockerignore

@@ -0,0 +1,14 @@
+.git
+.github
+.venv
+__pycache__
+*.py[cod]
+.pytest_cache
+.ruff_cache
+.coverage
+htmlcov
+graphify-out
+data
+tests
+docs
+*.egg-info

.github/workflows/ci.yml

@@ -0,0 +1,27 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install uv==0.6.11
+      - run: uv sync --frozen --extra dev
+      - run: uv run ruff check web2api tests
+      - run: uv run pytest tests/unit tests/integration --cov=web2api --cov-report=term-missing
+
+  docker-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: docker build -t web2api:ci .

.github/workflows/e2e.yml

@@ -0,0 +1,20 @@
+name: Docker E2E
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install uv==0.6.11
+      - run: uv sync --frozen --dev
+      - run: uv run pytest -m e2e tests/e2e -v

.gitignore

@@ -18,4 +18,3 @@ htmlcov/
 .ralph/
 dist_release/
 data/
-uv.lock

Dockerfile

@@ -6,16 +6,18 @@ ENV PYTHONUNBUFFERED=1
 WORKDIR /app
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends ca-certificates curl gnupg git \
-    && curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
-    && apt-get install -y --no-install-recommends nodejs \
-    && npm install -g @steipete/bird \
+    && apt-get install -y --no-install-recommends ca-certificates curl git \
     && rm -rf /var/lib/apt/lists/*
 
-COPY pyproject.toml ./
+COPY pyproject.toml uv.lock ./
+RUN pip install --no-cache-dir uv==0.6.11 \
+    && uv sync --frozen --no-dev --no-install-project
+
 COPY web2api/ ./web2api/
 
-RUN pip install --no-cache-dir . \
+ENV PATH="/app/.venv/bin:${PATH}"
+
+RUN uv sync --frozen --no-dev --no-editable \
     && playwright install --with-deps chromium
 
 RUN mkdir -p /data/recipes

README.md

@@ -73,6 +73,7 @@ curl -s http://localhost:8010/api/sites | jq
 ```bash
 git clone https://github.com/Endogen/web2api.git
 cd web2api
+export WEB2API_ADMIN_TOKEN="replace-with-a-long-random-token"
 docker compose up --build -d
 ```
 
@@ -110,6 +111,11 @@ docker compose exec web2api web2api recipes catalog add hackernews --yes
 
 Web2API can protect all HTTP routes except selected public paths with a shared access token.
 
+The recipe-management API is fail-closed even when general route authentication is disabled.
+Set `WEB2API_ADMIN_TOKEN` (or the general `WEB2API_ACCESS_TOKEN`) before using
+`/api/recipes/manage`. Development-only deployments can opt out with
+`WEB2API_ALLOW_UNAUTHENTICATED_ADMIN=true`.
+
 Set one of:
 - `WEB2API_ACCESS_TOKEN`
 - `WEB2API_ACCESS_TOKEN_FILE` (path to a file containing the token)
@@ -210,6 +216,8 @@ Catalog defaults come from:
 - `WEB2API_RECIPE_CATALOG_PATH` (catalog file path inside source, default `catalog.yaml`)
 If `WEB2API_RECIPE_CATALOG_SOURCE` is unset, Web2API uses the official remote repo
 `https://github.com/Endogen/web2api-recipes.git`.
+For repeatable production installs, set `WEB2API_RECIPE_CATALOG_REF` to an immutable commit SHA;
+the installer also records the installed recipe tree hash in its manifest.
 `recipes update` works only for recipes tracked in the manifest.
 
 Catalog entries can include optional setup hints:
@@ -368,7 +376,7 @@ Transport: Streamable HTTP
 ### How It Works
 
 - Each recipe endpoint registers as a separate MCP tool  
-  (e.g. `brave-search_search`, `deepl_de-en`, `allenai_olmo-32b`)
+  (e.g. `brave-search__search`, `deepl__de-en`, `allenai__olmo-32b`)
 - Tools include proper descriptions and typed parameter schemas
 - When recipes are installed/uninstalled via the admin API, tools rebuild automatically
 - Optional access-token protection is available via `WEB2API_ACCESS_TOKEN`
@@ -379,9 +387,13 @@ After installing the `brave-search` and `deepl` recipes:
 
 | Tool | Description | Parameters |
 |---|---|---|
-| `brave-search_search` | Web search via Brave | `q` (required) |
-| `deepl_de-en` | Translate German → English | `q` (required) |
-| `deepl_en-de` | Translate English → German | `q` (required) |
+| `brave-search__search` | Web search via Brave | `q` (required), `page` |
+| `deepl__de-en` | Translate German → English | `q` (required), `page` |
+| `deepl__en-de` | Translate English → German | `q` (required), `page` |
+
+Legacy single-underscore tool calls are still resolved by the HTTP bridge. Native tool discovery
+uses the unambiguous double-underscore form. For non-local MCP hostnames, configure
+`WEB2API_MCP_ALLOWED_HOSTS` and `WEB2API_MCP_ALLOWED_ORIGINS` explicitly.
 
 ### HTTP Bridge (Legacy)
 
@@ -546,13 +558,19 @@ endpoints:
 | `url` | yes | URL template with `{page}`, `{page_zero}`, `{query}` placeholders |
 | `description` | no | Human-readable endpoint description |
 | `requires_query` | no | If `true`, the `q` parameter is mandatory (default: `false`) |
+| `params` | no | Declared extra parameters; undeclared inputs are rejected |
+| `accepts_files` | no | Allow bounded multipart uploads for this endpoint (default: `false`) |
 | `actions` | no | Playwright actions to run before extraction |
 | `items` | yes | Container selector + field definitions |
 | `pagination` | yes | Pagination strategy (`page_param`, `offset_param`, or `next_link`) |
 
 Pagination notes:
 `{page}` resolves to `start + ((api_page - 1) * step)`.
 
+Extra parameters support `type` (`string`, `integer`, `number`, `boolean`), `required`, `enum`,
+`minimum`, `maximum`, `pattern`, `min_length`, `max_length`, and `example`. These constraints are
+used consistently by REST validation and MCP schemas.
+
 ### Actions
 
 | Type | Parameters |
@@ -602,6 +620,12 @@ class Scraper(BaseScraper):
 - `params` also includes validated extra query params (for example `count`)
 - Endpoints not handled by the scraper fall back to declarative YAML
 
+For a direct HTTP or CLI implementation, set `requires_browser = False`; its `page` argument is
+then `None` and it runs under the bounded direct-executor semaphore instead of consuming a browser
+context. Browser and direct outbound HTTP requests reject loopback, private, link-local, and
+reserved destinations by default. Operators can explicitly allow internal providers with
+`WEB2API_ALLOW_PRIVATE_NETWORK=true`.
+
 ### Plugin Metadata (Optional)
 
 Use `plugin.yaml` to declare install/runtime requirements for a recipe:
@@ -622,7 +646,7 @@ dependencies:
   apt:
     - nodejs
   npm:
-    - "@steipete/bird"
+    - "@steipete/bird@0.8.0"
 healthcheck:
   command: ["bird", "--version"]
 ```
@@ -651,6 +675,7 @@ Environment variables (with defaults):
 | `POOL_PAGE_TIMEOUT` | 15000 | Page navigation timeout (ms) |
 | `POOL_QUEUE_SIZE` | 20 | Max queued requests |
 | `SCRAPE_TIMEOUT` | 30 | Overall scrape timeout (seconds) |
+| `DIRECT_MAX_CONCURRENCY` | 20 | Maximum concurrent direct HTTP/CLI scraper calls |
 | `CACHE_ENABLED` | true | Enable in-memory response caching |
 | `CACHE_TTL_SECONDS` | 30 | Fresh cache duration in seconds |
 | `CACHE_STALE_TTL_SECONDS` | 120 | Stale-while-revalidate window in seconds |
@@ -662,7 +687,14 @@ Environment variables (with defaults):
 | `PLUGIN_ENFORCE_COMPATIBILITY` | false | Skip plugin recipes outside declared `web2api` version bounds |
 | `WEB2API_ACCESS_TOKEN` | empty | Shared access token for all routes except public paths |
 | `WEB2API_ACCESS_TOKEN_FILE` | empty | Path to file containing the access token (alternative to `WEB2API_ACCESS_TOKEN`) |
+| `WEB2API_ADMIN_TOKEN` | empty | Separate token for recipe-management routes; admin is disabled if no general/admin token is set |
+| `WEB2API_ALLOW_UNAUTHENTICATED_ADMIN` | false | Development-only opt-out for fail-closed admin routes |
 | `WEB2API_PUBLIC_PATHS` | empty | Extra public path patterns to allow without auth while token auth is enabled |
+| `WEB2API_ALLOW_PRIVATE_NETWORK` | false | Explicitly allow recipe traffic to private/internal network targets |
+| `WEB2API_MAX_UPLOAD_FILES` | 4 | Maximum multipart files per request |
+| `WEB2API_MAX_UPLOAD_BYTES` | 26214400 | Maximum bytes per uploaded file |
+| `WEB2API_MCP_ALLOWED_HOSTS` | local hosts | Allowed MCP Host header patterns |
+| `WEB2API_MCP_ALLOWED_ORIGINS` | local origins | Allowed MCP Origin patterns |
 | `BIRD_AUTH_TOKEN` | empty | X/Twitter auth token for `x` recipe |
 | `BIRD_CT0` | empty | X/Twitter ct0 token for `x` recipe |
 
@@ -671,6 +703,9 @@ Environment variables (with defaults):
 ```bash
 # Inside the container or with deps installed:
 pytest tests/unit tests/integration --timeout=30 -x -q
+
+# Explicit, bounded Docker/live-network suite
+pytest -m e2e tests/e2e -v
 ```
 
 ## Tech Stack

docker-compose.yml

@@ -15,6 +15,7 @@ services:
       POOL_PAGE_TIMEOUT: "${POOL_PAGE_TIMEOUT:-15000}"
       POOL_QUEUE_SIZE: "${POOL_QUEUE_SIZE:-20}"
       SCRAPE_TIMEOUT: "${SCRAPE_TIMEOUT:-120}"
+      DIRECT_MAX_CONCURRENCY: "${DIRECT_MAX_CONCURRENCY:-20}"
       CACHE_ENABLED: "${CACHE_ENABLED:-true}"
       CACHE_TTL_SECONDS: "${CACHE_TTL_SECONDS:-30}"
       CACHE_STALE_TTL_SECONDS: "${CACHE_STALE_TTL_SECONDS:-120}"
@@ -24,11 +25,21 @@ services:
       WEB2API_RECIPE_CATALOG_REF: "${WEB2API_RECIPE_CATALOG_REF:-}"
       WEB2API_RECIPE_CATALOG_PATH: "${WEB2API_RECIPE_CATALOG_PATH:-}"
       WEB2API_ACCESS_TOKEN: "${WEB2API_ACCESS_TOKEN:-}"
+      WEB2API_ADMIN_TOKEN: "${WEB2API_ADMIN_TOKEN:-}"
+      WEB2API_ALLOW_UNAUTHENTICATED_ADMIN: "${WEB2API_ALLOW_UNAUTHENTICATED_ADMIN:-false}"
+      WEB2API_ALLOW_PRIVATE_NETWORK: "${WEB2API_ALLOW_PRIVATE_NETWORK:-false}"
+      WEB2API_MAX_UPLOAD_FILES: "${WEB2API_MAX_UPLOAD_FILES:-4}"
+      WEB2API_MAX_UPLOAD_BYTES: "${WEB2API_MAX_UPLOAD_BYTES:-26214400}"
+      WEB2API_MCP_ALLOWED_HOSTS: "${WEB2API_MCP_ALLOWED_HOSTS:-127.0.0.1,127.0.0.1:*,localhost,localhost:*}"
+      WEB2API_MCP_ALLOWED_ORIGINS: "${WEB2API_MCP_ALLOWED_ORIGINS:-http://127.0.0.1,http://127.0.0.1:*,http://localhost,http://localhost:*}"
       WEB2API_PUBLIC_PATHS: "${WEB2API_PUBLIC_PATHS:-}"
       LOG_LEVEL: "${LOG_LEVEL:-info}"
       BIRD_AUTH_TOKEN: "${BIRD_AUTH_TOKEN:-}"
       BIRD_CT0: "${BIRD_CT0:-}"
       NVIDIA_API_KEY: "${NVIDIA_API_KEY:-}"
+      NOMINATIM_BASE_URL: "${NOMINATIM_BASE_URL:-}"
+      OSRM_BASE_URL: "${OSRM_BASE_URL:-}"
+      OPENSTREETMAP_USER_AGENT: "${OPENSTREETMAP_USER_AGENT:-}"
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
       interval: 10s

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "web2api"
-version = "0.4.1"
+version = "0.5.0"
 description = "Turn websites into REST APIs via live Playwright scraping."
 requires-python = ">=3.12"
 dependencies = [
@@ -52,3 +52,7 @@ select = ["E", "F", "I", "UP"]
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
+addopts = "-m 'not e2e'"
+markers = [
+  "e2e: Docker-backed live-network end-to-end tests",
+]

tests/conftest.py

@@ -5,7 +5,15 @@
 import sys
 from pathlib import Path
 
+import pytest
+
 PROJECT_ROOT = Path(__file__).resolve().parent.parent
 
 if str(PROJECT_ROOT) not in sys.path:
     sys.path.insert(0, str(PROJECT_ROOT))
+
+
+@pytest.fixture(autouse=True)
+def _allow_test_admin_without_a_token(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Keep legacy admin-route fixtures open unless a test opts into auth."""
+    monkeypatch.setenv("WEB2API_ALLOW_UNAUTHENTICATED_ADMIN", "true")

tests/e2e/test_e2e.py

@@ -36,6 +36,9 @@
     "docker daemon",
     "cannot connect to the docker engine",
 )
+COMPOSE_COMMAND_TIMEOUT_SECONDS = 300
+
+pytestmark = pytest.mark.e2e
 
 
 def _docker_compose_base_cmd() -> list[str]:
@@ -49,6 +52,7 @@ def _docker_compose_base_cmd() -> list[str]:
         capture_output=True,
         text=True,
         check=False,
+        timeout=30,
     )
     if version_result.returncode != 0:
         combined = f"{version_result.stdout}\n{version_result.stderr}".strip()
@@ -97,14 +101,21 @@ def _run_compose(
     env: dict[str, str],
     check: bool,
 ) -> subprocess.CompletedProcess[str]:
-    result = subprocess.run(
-        [*base_cmd, *args],
-        cwd=PROJECT_ROOT,
-        env=env,
-        capture_output=True,
-        text=True,
-        check=False,
-    )
+    try:
+        result = subprocess.run(
+            [*base_cmd, *args],
+            cwd=PROJECT_ROOT,
+            env=env,
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=COMPOSE_COMMAND_TIMEOUT_SECONDS,
+        )
+    except subprocess.TimeoutExpired as exc:
+        pytest.fail(
+            "docker compose "
+            f"{' '.join(args)} exceeded {COMPOSE_COMMAND_TIMEOUT_SECONDS}s: {exc}"
+        )
     if check and result.returncode != 0:
         combined = f"{result.stdout}\n{result.stderr}".strip()
         if _is_docker_unavailable(combined):
@@ -137,6 +148,7 @@ def dockerized_web2api() -> Iterator[str]:
     env["COMPOSE_PROJECT_NAME"] = compose_project
     host_port = _allocate_host_port()
     env["WEB2API_HOST_PORT"] = str(host_port)
+    env["WEB2API_ALLOW_UNAUTHENTICATED_ADMIN"] = "true"
     base_url = f"http://127.0.0.1:{host_port}"
 
     started = False