From 65492083e92fd814e750b352a313ca5110b41392 Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 2 Oct 2026 19:19:56 +0800 Subject: [PATCH 01/13] refactor: reuse parsing and bound site validation - reuse parsed canonical data and target lookups - validate version artifacts with two local workers - preserve all checks and report stage timings - test failure propagation and aggregate equivalence --- .github/workflows/hugo.yml | 2 +- scripts/test_versioning.py | 99 +++++++++++++++++++++++++++++ scripts/versioning.py | 126 ++++++++++++++++++++++++++++--------- 3 files changed, 195 insertions(+), 32 deletions(-) diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index 50cf9928b0..4c487451cf 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -296,7 +296,7 @@ jobs: extra+=(--asf-profile oink --asf-whoami asf-staging-oink) fi python3 scripts/versioning.py aggregate \ - --artifacts version-artifacts --artifact-prefix "$PREFIX-" \ + --artifacts version-artifacts --workers 2 --artifact-prefix "$PREFIX-" \ --artifact-suffix="-${GITHUB_RUN_ID}" \ --resolved-manifest resolved/resolved-versions.json \ --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ diff --git a/scripts/test_versioning.py b/scripts/test_versioning.py index 99cb51380d..92e6d6d832 100644 --- a/scripts/test_versioning.py +++ b/scripts/test_versioning.py @@ -48,6 +48,14 @@ def _walk_docs_nav_pages(nodes): yield from _walk_docs_nav_pages(node.get("children", [])) +def _validate_parallel_fixture(args): + """Picklable validator fixture: workers read inputs and must precede copying.""" + if (args.artifact.parent.parent / "aggregate").exists(): + raise AssertionError("aggregate output created before validation finished") + if (args.artifact / "index.html").read_text(encoding="utf-8") != args.version: + raise AssertionError("wrong artifact submitted to worker") + + class VersionUrlTest(unittest.TestCase): def test_historical_pruning_allows_footer_without_shared_routes(self) -> None: with tempfile.TemporaryDirectory() as temp_name: @@ -1781,6 +1789,97 @@ def test_aggregate_rejects_metadata_sha_before_copy(self) -> None: with self.assertRaises(SystemExit): versioning.aggregate(args) + def test_aggregate_parallel_validation_propagates_failure_before_output(self) -> None: + # Exercise real process transport, including SystemExit from fail(). + for workers in (1, 2): + with self.subTest(workers=workers), tempfile.TemporaryDirectory() as directory: + temp = Path(directory) + manifest = versioning.load_manifest(versioning.ROOT / "versions.json") + for entry in manifest["versions"]: + entry["sha"] = "a" * 40 + resolved = temp / "resolved.json" + resolved.write_text(json.dumps(manifest), encoding="utf-8") + source = temp / "artifacts/latest" + source.mkdir(parents=True) + # Identity is correct; worker validation must reject the wrong baseURL. + (source / ".version.json").write_text( + json.dumps(dict(manifest["versions"][0], baseURL="https://wrong.invalid/")), + encoding="utf-8", + ) + output = temp / "aggregate" + output.mkdir() + sentinel = output / "previous-output" + sentinel.write_text("preserve on validation failure", encoding="utf-8") + args = argparse.Namespace( + resolved_manifest=resolved, artifacts=temp / "artifacts", + artifact_prefix="", site_origin=ORIGIN, select="latest", + output=output, asf_profile=None, asf_whoami=None, workers=workers, + ) + with self.assertRaisesRegex(SystemExit, "version metadata does not match"): + versioning.aggregate(args) + self.assertEqual(sentinel.read_text(), "preserve on validation failure") + self.assertEqual(list(output.iterdir()), [sentinel]) + + def test_aggregate_parallel_copy_matches_serial_output(self) -> None: + # Keep the worker boundary real while isolating orchestration from the + # full site fixtures. Both modes must emit identical complete output. + outputs = [] + for workers in (1, 2): + with self.subTest(workers=workers), tempfile.TemporaryDirectory() as directory: + temp = Path(directory) + entries = [ + {"id": "latest", "publishPath": "", "sha": "a" * 40}, + {"id": "1.7", "publishPath": "versions/1.7", "sha": "b" * 40}, + ] + for entry in entries: + source = temp / "artifacts" / entry["id"] + source.mkdir(parents=True) + (source / ".version.json").write_text(json.dumps(entry), encoding="utf-8") + (source / "index.html").write_text(entry["id"], encoding="utf-8") + (source / "404.html").write_text("not found", encoding="utf-8") + for language in ("en", "cn"): + (source / language).mkdir() + (source / language / "sitemap.xml").write_text("", encoding="utf-8") + args = argparse.Namespace( + resolved_manifest=temp / "resolved.json", artifacts=temp / "artifacts", + artifact_prefix="", site_origin=ORIGIN, output=temp / "aggregate", + asf_profile=None, asf_whoami=None, workers=workers, + ) + with ( + mock.patch.object(versioning, "load_resolved_manifest", return_value={"versions": entries}), + mock.patch.object(versioning, "require_metadata_matches"), + mock.patch.object(versioning, "validate_artifact", _validate_parallel_fixture), + mock.patch.object(versioning, "load_version_routes", return_value={}), + mock.patch.object(versioning, "validate_aggregate_version_routes") as routes, + mock.patch.object(versioning, "validate_output_security") as security, + ): + versioning.aggregate(args) + routes.assert_called_once() + security.assert_called_once_with(args.output.resolve(), ORIGIN) + outputs.append({ + path.relative_to(args.output).as_posix(): path.read_bytes() + for path in args.output.rglob("*") if path.is_file() + }) + self.assertEqual(outputs[0], outputs[1]) + self.assertEqual(outputs[0]["versions/1.7/index.html"], b"1.7") + + def test_canonical_target_reuses_parsed_page(self) -> None: + entry = {"publishPath": "versions/1.7"} + for target, expected in (("cn/docs/guide/", "cn/docs/guide/"), ("about/", None)): + text = f'' + document = versioning.DocumentParser() + document.feed(text) + with mock.patch.object(versioning.DocumentParser, "feed", side_effect=AssertionError("reparsed")): + self.assertEqual(versioning._canonical_docs_target(document, entry), expected) + self.assertEqual(versioning.canonical_docs_target(text, entry), expected) + + def test_public_version_switch_still_rejects_invalid_routes(self) -> None: + manifest = versioning.load_manifest(versioning.ROOT / "versions.json") + routes = versioning.load_version_routes(manifest=manifest) + routes["schemaVersion"] = -1 + with self.assertRaises(SystemExit): + versioning.version_switch_options(manifest, routes, "latest", "docs/", ORIGIN) + def test_aggregate_flattened_artifact_checks_identity_and_selection(self) -> None: for changed_field, select in [(None, "latest"), ("id", "latest"), ("sha", "latest"), (None, None)]: diff --git a/scripts/versioning.py b/scripts/versioning.py index e812d8c0fc..536db7ed0c 100644 --- a/scripts/versioning.py +++ b/scripts/versioning.py @@ -19,6 +19,8 @@ from __future__ import annotations import argparse +from concurrent.futures import ProcessPoolExecutor +from contextlib import contextmanager import hashlib import html import html.parser @@ -30,6 +32,7 @@ import subprocess import sys import tempfile +import time import urllib.parse import xml.etree.ElementTree as ET from typing import NoReturn @@ -2582,6 +2585,11 @@ def canonical_docs_target(text: str, entry: dict) -> str | None: """Resolve the current canonical page to an unscoped route-map target.""" document = DocumentParser() document.feed(text) + return _canonical_docs_target(document, entry) + + +def _canonical_docs_target(document: DocumentParser, entry: dict) -> str | None: + """Read canonical routing from an already parsed page.""" if len(document.canonical) != 1: return None path = urllib.parse.urlsplit(document.canonical[0]).path @@ -2650,6 +2658,23 @@ def version_switch_options( ) -> list[dict]: """Return page-aware version choices with explicit missing-page fallbacks.""" validate_version_routes(route_map, manifest) + return _version_switch_options( + manifest, route_map, current_version, current_target, origin, + historical_origin, language=language, + ) + + +def _version_switch_options( + manifest: dict, + route_map: dict, + current_version: str, + current_target: str | None, + origin: str, + historical_origin: str | None = None, + *, + language: str | None = None, +) -> list[dict]: + """Use a route map already validated by load_version_routes or the public API.""" logical_id = None if current_target is not None: target_language, _ = docs_target_parts(current_target) @@ -3419,7 +3444,22 @@ def validate_social_image_metadata( fail(f"social image target is missing in {relative}: {value}") +@contextmanager +def timed_stage(label: str): + """Emit elapsed wall time even when a validation fails.""" + started = time.monotonic() + try: + yield + finally: + print(f"timing {label}: {time.monotonic() - started:.3f}s", flush=True) + + def validate_artifact(args: argparse.Namespace) -> None: + with timed_stage(f"version-validation[{args.version}]"): + _validate_artifact(args) + + +def _validate_artifact(args: argparse.Namespace) -> None: manifest = load_manifest(args.manifest) route_map = load_version_routes(manifest=manifest) entry = next( @@ -3516,6 +3556,15 @@ def validate_artifact(args: argparse.Namespace) -> None: ) contract_routes += 1 + # The artifact is read-only throughout validation; repeated navigation URLs + # can share filesystem lookups without sharing source-dependent URL checks. + target_cache: dict[str, bool] = {} + + def local_target_exists(relative: str) -> bool: + if relative not in target_cache: + target_cache[relative] = target_exists(root, relative) + return target_cache[relative] + def validate_url(value: str, source: pathlib.Path) -> None: nonlocal checked_urls if not value or value.startswith(("#", "?")): @@ -3591,7 +3640,7 @@ def validate_url(value: str, source: pathlib.Path) -> None: f"URL escapes version {entry['id']} in {source.relative_to(root)}: {value}" ) relative = path[len(prefix) :] if prefix else path - if not target_exists(root, relative): + if not local_target_exists(relative): fail(f"missing local target from {source.relative_to(root)}: {value}") checked_urls += 1 @@ -3778,8 +3827,8 @@ def validate_url(value: str, source: pathlib.Path) -> None: None, ) language = "cn" if relative.startswith("cn/") else "en" - current_target = canonical_docs_target(text, entry) - expected_options = version_switch_options( + current_target = _canonical_docs_target(document, entry) + expected_options = _version_switch_options( manifest, route_map, entry["id"], @@ -3895,7 +3944,7 @@ def validate_url(value: str, source: pathlib.Path) -> None: expected_hreflang = { language: url for language, url in equivalent_urls.items() - if target_exists(root, equivalent_paths[language]) + if local_target_exists(equivalent_paths[language]) } if actual_hreflang != expected_hreflang: fail( @@ -4290,17 +4339,18 @@ def write_error_documents(output: pathlib.Path, seen: set[str]) -> int: def validate_output_security(output: pathlib.Path, site_origin: str) -> None: """Re-scan a complete output tree before it can be published.""" - subprocess.run( - [ - sys.executable, - str(ROOT / "dist/validate-site-output.py"), - str(output), - site_origin, - "--security-only", - ], - cwd=ROOT, - check=True, - ) + with timed_stage(f"security[{output.name}]"): + subprocess.run( + [ + sys.executable, + str(ROOT / "dist/validate-site-output.py"), + str(output), + site_origin, + "--security-only", + ], + cwd=ROOT, + check=True, + ) def validate_aggregate_version_routes( @@ -4357,10 +4407,11 @@ def validate_aggregate_version_routes( def aggregate(args: argparse.Namespace) -> None: manifest = load_resolved_manifest(args.resolved_manifest) selected = selected_version_ids(getattr(args, "select", None), manifest) - output = prepare_output_directory(args.output, "aggregate output") - output.mkdir(parents=True) - seen: set[str] = set() - resolved = [] + workers = getattr(args, "workers", 1) + if workers not in {1, 2}: + fail("aggregate workers must be 1 or 2") + artifacts = [] + validations = [] for entry in manifest["versions"]: if entry["id"] not in selected: continue @@ -4377,7 +4428,7 @@ def aggregate(args: argparse.Namespace) -> None: fail(f"missing version metadata: {metadata_path}") metadata = json.loads(metadata_path.read_text(encoding="utf-8")) require_metadata_matches(entry, metadata, metadata_path) - validate_artifact( + validations.append( argparse.Namespace( manifest=args.resolved_manifest, version=entry["id"], @@ -4387,10 +4438,25 @@ def aggregate(args: argparse.Namespace) -> None: artifact=source, ) ) - destination = output / entry["publishPath"] - destination.mkdir(parents=True, exist_ok=True) - copy_without_collision(source, destination, seen) - resolved.append(metadata) + artifacts.append((entry, source, metadata)) + with timed_stage("aggregate-validation"): + if workers == 1: + for validation in validations: + validate_artifact(validation) + else: + with ProcessPoolExecutor(max_workers=workers) as executor: + # Consume every result: worker exceptions must prevent publication. + list(executor.map(validate_artifact, validations)) + output = prepare_output_directory(args.output, "aggregate output") + output.mkdir(parents=True) + seen: set[str] = set() + resolved = [] + with timed_stage("aggregate-copy"): + for entry, source, metadata in artifacts: + destination = output / entry["publishPath"] + destination.mkdir(parents=True, exist_ok=True) + copy_without_collision(source, destination, seen) + resolved.append(metadata) error_documents = write_error_documents(output, seen) write_aggregate_sitemap(output, args.site_origin, manifest) expected_sitemaps = sitemap_locations(output / "sitemap.xml") @@ -4430,13 +4496,10 @@ def aggregate(args: argparse.Namespace) -> None: json.dumps(route_map, ensure_ascii=False, sort_keys=False, indent=2) + "\n", encoding="utf-8", ) - validate_aggregate_version_routes( - output, - route_map, - selected, - manifest, - ) - validate_output_security(output, args.site_origin) + with timed_stage("aggregate-routes"): + validate_aggregate_version_routes(output, route_map, selected, manifest) + with timed_stage("aggregate-security"): + validate_output_security(output, args.site_origin) print( f"aggregated {len(resolved)} versions and {len(seen)} files " f"with {error_documents} error documents -> {output}" @@ -4505,6 +4568,7 @@ def parser() -> argparse.ArgumentParser: aggregate_parser.add_argument("--site-origin", required=True) aggregate_parser.add_argument("--historical-origin") aggregate_parser.add_argument("--select") + aggregate_parser.add_argument("--workers", type=int, choices=(1, 2), default=1) aggregate_parser.add_argument("--output", type=pathlib.Path, required=True) aggregate_parser.add_argument("--asf-profile") aggregate_parser.add_argument("--asf-whoami") From 6a2f26b32f6132fc14f5749fc88234a1df76b5ee Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 2 Oct 2026 19:24:48 +0800 Subject: [PATCH 02/13] refactor: reuse historical site build artifacts - group version builds on one bounded runner - reuse artifacts with exact inputs and tree digests - validate every restored version before aggregation - preserve dependency caches and rebuild invalid entries --- .github/workflows/hugo.yml | 79 ++++++----- contribution.md | 13 +- scripts/build_versions.py | 149 +++++++++++++++++++++ scripts/test_build_versions.py | 172 ++++++++++++++++++++++++ scripts/test_version_cache.py | 166 +++++++++++++++++++++++ scripts/version_cache.py | 189 +++++++++++++++++++++++++++ tests/e2e/workflow-contract.test.cjs | 18 ++- 7 files changed, 748 insertions(+), 38 deletions(-) create mode 100644 scripts/build_versions.py create mode 100644 scripts/test_build_versions.py create mode 100644 scripts/test_version_cache.py create mode 100644 scripts/version_cache.py diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index 4c487451cf..23713b5c0f 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -45,7 +45,6 @@ jobs: env: HUGO_CACHEDIR: /tmp/hugo-cache-${{ github.run_id }}-${{ github.run_attempt }}-prepare outputs: - versions: ${{ steps.matrix.outputs.versions }} selection: ${{ steps.plan.outputs.selection }} site_origin: ${{ steps.plan.outputs.site_origin }} historical_origin: ${{ steps.plan.outputs.historical_origin }} @@ -166,7 +165,7 @@ jobs: go mod download PYTHONDONTWRITEBYTECODE=1 \ python3 -m unittest discover -s scripts -p 'test_*.py' -v - - name: Resolve immutable version matrix + - name: Resolve immutable version manifest id: matrix env: LATEST_SHA: ${{ steps.plan.outputs.latest_sha }} @@ -175,7 +174,6 @@ jobs: python3 scripts/versioning.py prepare \ --latest-sha "$LATEST_SHA" --select "$SELECTION" \ --output resolved-versions.json - echo "versions=$(jq -c '.include' resolved-versions.json)" >> "$GITHUB_OUTPUT" - name: Upload resolved version manifest uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -189,16 +187,11 @@ jobs: build: needs: prepare runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 25 permissions: { contents: read } env: - HUGO_CACHEDIR: /tmp/hugo-cache-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.version.id }} - strategy: - fail-fast: false - max-parallel: 5 - matrix: - version: ${{ fromJSON(needs.prepare.outputs.versions) }} - name: Build ${{ matrix.version.id }} + HUGO_CACHEDIR: /tmp/hugo-resources + name: Build selected versions steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -208,13 +201,10 @@ jobs: - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.13" - - name: Fetch immutable resolved source - env: - RESOLVED_SHA: ${{ matrix.version.sha }} - run: | - [[ "$RESOLVED_SHA" =~ ^[0-9a-f]{40}$ ]] - git fetch --no-tags origin "$RESOLVED_SHA" - git cat-file -e "$RESOLVED_SHA^{commit}" + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: resolved-versions-${{ github.run_id }} + path: resolved - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: go.mod @@ -228,33 +218,57 @@ jobs: uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: ${{ env.HUGO_CACHEDIR }} - key: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-${{ matrix.version.id }}-${{ hashFiles('go.sum') }} - restore-keys: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-${{ matrix.version.id }}- + key: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-all-${{ hashFiles('go.sum') }} + restore-keys: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-all- - name: Verify pinned OINK module run: | test "$(go list -m -f '{{ .Path }}')" = "github.com/apache/hugegraph-doc" test "$(go list -m all | wc -l)" -eq 2 test -z "$(go list -m -f '{{ with .Replace }}{{ .Path }}{{ end }}' github.com/pgsty/oink)" python3 scripts/update_oink.py --check-baseline - - name: Build isolated version artifact + - name: Compute historical build fingerprints + id: cache-plan + env: + SITE_ORIGIN: ${{ needs.prepare.outputs.site_origin }} + HISTORICAL_ORIGIN: ${{ needs.prepare.outputs.historical_origin }} + run: | + python3 scripts/version_cache.py plan \ + --resolved-manifest resolved/resolved-versions.json \ + --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ + --hugo-version "$HUGO_VERSION" --go-version "$(go version)" \ + --cache-dir "$RUNNER_TEMP/history-cache" --output "$RUNNER_TEMP/cache-plan.json" + echo "key=$(jq -r .cacheKey "$RUNNER_TEMP/cache-plan.json")" >> "$GITHUB_OUTPUT" + - name: Restore historical artifact bundle + id: history-cache + if: github.event_name != 'workflow_dispatch' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ${{ runner.temp }}/history-cache + key: ${{ runner.os }}-${{ steps.cache-plan.outputs.key }} + # A previous bundle is only transport: each version must pass its + # exact input fingerprint and file digest before it can be reused. + restore-keys: ${{ runner.os }}-history-v1- + - name: Build and validate selected versions env: OINK_PYTHON: python3 SITE_ORIGIN: ${{ needs.prepare.outputs.site_origin }} HISTORICAL_ORIGIN: ${{ needs.prepare.outputs.historical_origin }} run: | - python3 scripts/versioning.py build \ - --version "${{ matrix.version.id }}" --sha "${{ matrix.version.sha }}" \ + python3 scripts/version_cache.py plan \ + --resolved-manifest resolved/resolved-versions.json \ --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ - --output "${RUNNER_TEMP}/version-public" - python3 scripts/versioning.py validate \ - --version "${{ matrix.version.id }}" --sha "${{ matrix.version.sha }}" \ + --hugo-version "$HUGO_VERSION" --go-version "$(go version)" \ + --cache-dir "$RUNNER_TEMP/history-cache" --output "$RUNNER_TEMP/cache-plan.json" + python3 scripts/build_versions.py \ + --resolved-manifest resolved/resolved-versions.json \ + --plan "$RUNNER_TEMP/cache-plan.json" --cache-dir "$RUNNER_TEMP/history-cache" \ --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ - --artifact "${RUNNER_TEMP}/version-public" - - name: Upload isolated version artifact + --workers 2 --output "$RUNNER_TEMP/version-artifacts" + - name: Upload selected version artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: ${{ needs.prepare.outputs.artifact_prefix }}-${{ matrix.version.id }}-${{ github.run_id }} - path: ${{ runner.temp }}/version-public + name: ${{ needs.prepare.outputs.artifact_prefix }}-versions-${{ github.run_id }} + path: ${{ runner.temp }}/version-artifacts include-hidden-files: true retention-days: 1 if-no-files-found: error @@ -282,7 +296,7 @@ jobs: path: resolved - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - pattern: ${{ needs.prepare.outputs.artifact_prefix }}-*-${{ github.run_id }} + name: ${{ needs.prepare.outputs.artifact_prefix }}-versions-${{ github.run_id }} path: version-artifacts - name: Assemble publishable site env: @@ -296,8 +310,7 @@ jobs: extra+=(--asf-profile oink --asf-whoami asf-staging-oink) fi python3 scripts/versioning.py aggregate \ - --artifacts version-artifacts --workers 2 --artifact-prefix "$PREFIX-" \ - --artifact-suffix="-${GITHUB_RUN_ID}" \ + --artifacts version-artifacts --workers 2 \ --resolved-manifest resolved/resolved-versions.json \ --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ --select "$SELECTION" \ diff --git a/contribution.md b/contribution.md index 3b239726c4..d7ff381af5 100644 --- a/contribution.md +++ b/contribution.md @@ -72,8 +72,17 @@ Cancelled runs skip report uploads and the final gate instead of holding the queue with `always()`. A queued job with no runner has not started testing; repeated reruns do not resolve runner capacity shortages. -Version builds remain parallel. Site assembly and blocking browser tests share -one runner and the same local artifact. The required `deploy` check still +Selected versions share one build runner with at most two builds in parallel. +Historical artifacts are reused only when both the complete build-input fingerprint +and artifact digest match. Missing or invalid cache entries rebuild automatically; +manual dispatches start without the artifact cache. Cache hits still run the full +version validator, and aggregation validates every selected version again. + +Assembly validates at most two versions concurrently, then writes the complete +site and runs the cross-version and security checks. Stage timings are printed as +`timing : s`; nested security timings are included in their parent +validation duration and must not be added twice. Site assembly and blocking browser +tests share one runner and the same local artifact. The required `deploy` check still requires all blocking jobs to succeed; visual captures remain advisory and publication alone receives write permission. For a test failure, rerun failed jobs after inspecting the cause; artifact names remain stable within the run. diff --git a/scripts/build_versions.py b/scripts/build_versions.py new file mode 100644 index 0000000000..6540646a24 --- /dev/null +++ b/scripts/build_versions.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Build selected versions with bounded concurrency and fully validate cache hits.""" + +import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import nullcontext +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import time + +import version_cache +import versioning + + +def log(version, message, started): + print(f"[version {version}] {message} ({time.monotonic() - started:.2f}s)", flush=True) + + +def run_version(entry, args, hugo_cache): + started = time.monotonic() + output = args.output / entry["id"] + environment = dict(os.environ, HUGO_CACHEDIR=str(hugo_cache / entry["id"])) + common = ["--version", entry["id"], "--sha", entry["sha"], + "--site-origin", args.site_origin, "--historical-origin", args.historical_origin] + + def command(operation): + target = "--output" if operation == "build" else "--artifact" + subprocess.run([sys.executable, str(versioning.ROOT / "scripts/versioning.py"), + operation, *common, target, str(output)], + cwd=versioning.ROOT, env=environment, check=True) + + restored = False + if entry["archived"] and not args.force_rebuild: + try: + restored = version_cache.restore(args.cache_dir, entry, output) + except OSError as error: + log(entry["id"], f"cache restore unavailable; rebuilding: {error}", started) + if restored: + log(entry["id"], "cache restored; validating", started) + try: + command("validate") + except subprocess.CalledProcessError: + log(entry["id"], "cached artifact failed validation; rebuilding", started) + else: + log(entry["id"], "cache validated", started) + return + log(entry["id"], "building", started) + command("build") + command("validate") + log(entry["id"], "build validated", started) + if entry["archived"]: + try: + version_cache.record(args.cache_dir, entry, output) + except (OSError, ValueError) as error: + log(entry["id"], f"cache save unavailable: {error}", started) + log(entry["id"], "complete", started) + + +def selected_entries(manifest, plan): + selected = manifest.get("include", manifest["versions"]) + if not selected or len({e["id"] for e in selected}) != len(selected): + raise ValueError("resolved selection must be nonempty and unique") + expected = {e["id"]: e for e in manifest["versions"]} + if any(e != expected.get(e["id"]) for e in selected): + raise ValueError("resolved selection does not match resolved versions") + entries = plan["versions"] + if [e["id"] for e in entries] != [e["id"] for e in selected]: + raise ValueError("cache plan does not match selected versions") + for entry, source in zip(entries, selected): + if any(entry.get(key) != value for key, value in source.items()): + raise ValueError("cache plan source differs from resolved manifest") + if not re.fullmatch(r"[0-9a-f]{64}", entry.get("buildKey", "")): + raise ValueError("cache plan has invalid build key") + return entries + + +def execute(args): + if not 1 <= args.workers <= 3: + raise ValueError("workers must be between 1 and 3") + manifest = versioning.load_resolved_manifest(args.resolved_manifest) + entries = selected_entries(manifest, json.loads(args.plan.read_text())) + output, cache = args.output.resolve(), args.cache_dir.resolve() + if output == cache or output in cache.parents or cache in output.parents: + raise ValueError("output and cache directories must not overlap") + configured_hugo_cache = os.environ.get("HUGO_CACHEDIR") + hugo_cache = Path(configured_hugo_cache).expanduser().resolve() if configured_hugo_cache else None + if hugo_cache is not None: + for other in (output, cache): + if hugo_cache == other or hugo_cache in other.parents or other in hugo_cache.parents: + raise ValueError("Hugo cache, artifact cache and output directories must not overlap") + args.output = versioning.prepare_output_directory(args.output, "versions output") + # Complete shared Git mutations before independent build subprocesses start. + for sha in dict.fromkeys(e["sha"] for e in entries): + try: + versioning.run(["git", "cat-file", "-e", f"{sha}^{{commit}}"]) + except subprocess.CalledProcessError: + versioning.run(["git", "fetch", "--no-tags", "origin", sha]) + print(f"version build workers: {args.workers}; available CPUs: {os.cpu_count()}", flush=True) + failures = [] + cache_context = (nullcontext(hugo_cache) if hugo_cache is not None + else tempfile.TemporaryDirectory(prefix="hugo-versions-")) + with cache_context as directory: + with ThreadPoolExecutor(max_workers=args.workers) as executor: + futures = {executor.submit(run_version, entry, args, Path(directory)): entry["id"] + for entry in entries} + for future in as_completed(futures): + try: + future.result() + except Exception as error: + failures.append(futures[future]) + print(f"[version {futures[future]}] FAILED: {error}", file=sys.stderr, flush=True) + if failures: + raise RuntimeError(f"failed versions: {', '.join(sorted(failures))}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("resolved-manifest", "plan", "cache-dir", "output"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--site-origin", required=True) + parser.add_argument("--historical-origin", required=True) + parser.add_argument("--workers", type=int, choices=range(1, 4), default=2) + parser.add_argument("--force-rebuild", action="store_true") + execute(parser.parse_args()) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_build_versions.py b/scripts/test_build_versions.py new file mode 100644 index 0000000000..5947752adb --- /dev/null +++ b/scripts/test_build_versions.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import os +from pathlib import Path +import subprocess +import tempfile +import threading +import unittest +from unittest import mock + +import build_versions + + +class BuildVersionsTest(unittest.TestCase): + def setUp(self): + self.entry = {"id": "1.7", "sha": "a" * 40, "archived": True, "buildKey": "b" * 64} + self.args = argparse.Namespace(output=Path("/tmp/versions-test"), + cache_dir=Path("/tmp/cache-test"), site_origin="https://example.com/", + historical_origin="https://example.com/", force_rebuild=False, workers=2) + + def run_one(self, restored, *, failure=None): + with mock.patch.object(build_versions.version_cache, "restore", return_value=restored) as restore, \ + mock.patch.object(build_versions.version_cache, "record") as record, \ + mock.patch.object(build_versions.subprocess, "run", side_effect=failure) as run: + build_versions.run_version(self.entry, self.args, Path("/tmp/hugo-test")) + return restore, record, run + + def test_cached_version_still_validated(self): + _, record, run = self.run_one(True) + self.assertEqual([c.args[0][2] for c in run.call_args_list], ["validate"]) + record.assert_not_called() + + def test_miss_builds_and_validates(self): + _, record, run = self.run_one(False) + self.assertEqual([c.args[0][2] for c in run.call_args_list], ["build", "validate"]) + record.assert_called_once() + self.assertEqual(run.call_args.kwargs["env"]["HUGO_CACHEDIR"], "/tmp/hugo-test/1.7") + + def test_invalid_cached_artifact_rebuilt_and_validated(self): + _, record, run = self.run_one(True, failure=[subprocess.CalledProcessError(1, "validate"), None, None]) + self.assertEqual([c.args[0][2] for c in run.call_args_list], ["validate", "build", "validate"]) + record.assert_called_once() + + def test_failed_fresh_validation_not_cached(self): + with mock.patch.object(build_versions.version_cache, "restore", return_value=False), \ + mock.patch.object(build_versions.version_cache, "record") as record, \ + mock.patch.object(build_versions.subprocess, "run", + side_effect=[None, subprocess.CalledProcessError(1, "validate")]): + with self.assertRaises(subprocess.CalledProcessError): + build_versions.run_version(self.entry, self.args, Path("/tmp/hugo-test")) + record.assert_not_called() + + def test_forced_rebuild_does_not_restore(self): + self.args.force_rebuild = True + restore, _, run = self.run_one(True) + restore.assert_not_called() + self.assertEqual([c.args[0][2] for c in run.call_args_list], ["build", "validate"]) + + def test_restore_io_failure_falls_back_to_build(self): + with mock.patch.object(build_versions.version_cache, "restore", side_effect=OSError("disk")), \ + mock.patch.object(build_versions.version_cache, "record"), \ + mock.patch.object(build_versions.subprocess, "run") as run: + build_versions.run_version(self.entry, self.args, Path("/tmp/hugo-test")) + self.assertEqual([c.args[0][2] for c in run.call_args_list], ["build", "validate"]) + + def test_corrupt_cache_symlink_does_not_fail_validated_build(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp).resolve() + self.args.output = root / "output" + self.args.cache_dir = root / "cache" + self.args.cache_dir.mkdir() + protected = root / "protected" + protected.mkdir() + marker = protected / "marker" + marker.write_text("keep") + (self.args.cache_dir / self.entry["id"]).symlink_to(protected, target_is_directory=True) + + def build(command, **_): + if command[2] == "build": + artifact = self.args.output / self.entry["id"] + artifact.mkdir(parents=True) + (artifact / ".version.json").write_text(json.dumps(self.entry)) + + with mock.patch.object(build_versions.subprocess, "run", side_effect=build) as run: + build_versions.run_version(self.entry, self.args, root / "hugo") + self.assertEqual([c.args[0][2] for c in run.call_args_list], ["build", "validate"]) + self.assertEqual(marker.read_text(), "keep") + self.assertEqual(list(protected.iterdir()), [marker]) + + def test_rejects_drifted_plan(self): + with self.assertRaises(ValueError): + build_versions.selected_entries({"versions": [self.entry]}, + {"versions": [dict(self.entry, sha="c" * 40)]}) + + def test_configured_hugo_cache_preserved_and_isolated_by_version(self): + entries = [self.entry, dict(self.entry, id="1.5")] + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + cache = root / "hugo-cache" + cache.mkdir() + marker = cache / "existing-resource" + marker.write_text("keep") + self.args.output = root / "output" + self.args.cache_dir = root / "artifacts" + self.args.plan = root / "plan.json" + self.args.plan.write_text(json.dumps({"versions": entries})) + self.args.resolved_manifest = root / "resolved.json" + with mock.patch.dict(os.environ, HUGO_CACHEDIR=str(cache)), \ + mock.patch.object(build_versions.versioning, "prepare_output_directory", return_value=self.args.output), \ + mock.patch.object(build_versions.versioning, "load_resolved_manifest", return_value={"versions": entries}), \ + mock.patch.object(build_versions.versioning, "run"), \ + mock.patch.object(build_versions.version_cache, "restore", return_value=False), \ + mock.patch.object(build_versions.version_cache, "record"), \ + mock.patch.object(build_versions.subprocess, "run") as run: + build_versions.execute(self.args) + self.assertEqual(marker.read_text(), "keep") + self.assertEqual({c.kwargs["env"]["HUGO_CACHEDIR"] for c in run.call_args_list}, + {str(cache.resolve() / "1.7"), str(cache.resolve() / "1.5")}) + + def test_hugo_cache_inside_output_rejected_before_cleanup(self): + with tempfile.TemporaryDirectory() as tmp: + self.args.plan = Path(tmp) / "plan.json" + self.args.plan.write_text(json.dumps({"versions": [self.entry]})) + self.args.resolved_manifest = Path(tmp) / "resolved.json" + with mock.patch.dict(os.environ, HUGO_CACHEDIR=str(self.args.output / "cache")), \ + mock.patch.object(build_versions.versioning, "load_resolved_manifest", return_value={"versions": [self.entry]}), \ + mock.patch.object(build_versions.versioning, "prepare_output_directory") as prepare: + with self.assertRaisesRegex(ValueError, "must not overlap"): + build_versions.execute(self.args) + prepare.assert_not_called() + + def test_bounded_workers_collect_failures_and_finish_other_versions(self): + entries = [dict(self.entry, id=str(i)) for i in range(4)] + gate = threading.Barrier(2, timeout=5) + finished = [] + def work(entry, *_): + gate.wait() + finished.append(entry["id"]) + if entry["id"] == "0": + raise RuntimeError("build failed") + with tempfile.TemporaryDirectory() as tmp: + self.args.plan = Path(tmp) / "plan.json" + self.args.plan.write_text(json.dumps({"versions": entries})) + self.args.resolved_manifest = Path(tmp) / "resolved.json" + with mock.patch.object(build_versions.versioning, "load_resolved_manifest", return_value={"versions": entries}), \ + mock.patch.object(build_versions.versioning, "prepare_output_directory", return_value=self.args.output), \ + mock.patch.object(build_versions.versioning, "run"), \ + mock.patch.object(build_versions, "run_version", side_effect=work): + with self.assertRaisesRegex(RuntimeError, "failed versions: 0"): + build_versions.execute(self.args) + self.assertEqual(sorted(finished), ["0", "1", "2", "3"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_version_cache.py b/scripts/test_version_cache.py new file mode 100644 index 0000000000..041ee3f1dd --- /dev/null +++ b/scripts/test_version_cache.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import json +import pathlib +import tempfile +import unittest + +import version_cache as cache + + +class VersionCacheTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = pathlib.Path(self.temporary.name).resolve() + self.bundle = self.root / "cache" + self.manifest = {"repository": "https://example.org/docs.git", "versions": [ + {"id": "latest", "sha": "a" * 40, "archived": False}, + {"id": "1.7", "sha": "b" * 40, "archived": True}, + {"id": "1.5", "sha": "c" * 40, "archived": True}]} + self.write("hugo.yaml", "theme: oink") + self.write("versions.json", json.dumps(self.manifest)) + self.write("content/en/docs/page.md", "latest body") + self.options = dict(site_origin="https://example.org/", historical_origin="https://example.org/", + hugo_version="0.165.0", go_version="1.27", webp_version="1.2", year=2026) + + def write(self, name, value): + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(value) + return path + + def plan(self): + return cache.make_plan(self.root, self.manifest, self.bundle, **self.options) + + def save(self): + entry = self.plan()["versions"][1] + self.write("artifact/.version.json", json.dumps({"id": "1.7", "sha": "b" * 40, "archived": True})) + self.write("artifact/index.html", "page") + cache.record(self.bundle, entry, self.root / "artifact") + return entry + + def test_body_and_latest_sha_do_not_invalidate_history(self): + before = self.plan() + self.write("content/en/docs/page.md", "updated body") + self.manifest["versions"][0]["sha"] = "d" * 40 + after = self.plan() + self.assertEqual(before["cacheKey"], after["cacheKey"]) + self.assertEqual(before["versions"][1:], after["versions"][1:]) + + def test_shell_changes_invalidate_history(self): + before = self.plan()["cacheKey"] + self.write("layouts/partials/test.html", "new shell") + self.assertNotEqual(before, self.plan()["cacheKey"]) + + def test_history_sha_only_invalidates_corresponding_version(self): + before = self.plan() + self.manifest["versions"][1]["sha"] = "d" * 40 + after = self.plan() + self.assertNotEqual(before["versions"][1]["buildKey"], after["versions"][1]["buildKey"]) + self.assertEqual(before["versions"][2], after["versions"][2]) + + def test_origins_toolchain_year_invalidate(self): + for key in self.options: + with self.subTest(key=key): + original = self.options[key] + before = self.plan()["cacheKey"] + self.options[key] = 2027 if key == "year" else original + "changed" + self.assertNotEqual(before, self.plan()["cacheKey"]) + self.options[key] = original + + def test_cache_round_trip_and_missing(self): + self.assertFalse(self.plan()["versions"][1]["reusable"]) + entry = self.save() + self.assertTrue(self.plan()["versions"][1]["reusable"]) + self.assertEqual([e["id"] for e in self.plan()["include"]], ["latest", "1.5"]) + self.assertTrue(cache.restore(self.bundle, entry, self.root / "restored")) + self.assertEqual((self.root / "restored/index.html").read_text(), "page") + self.assertFalse(self.plan()["versions"][0]["reusable"]) + + def test_corruption_and_receipt_mismatch_are_misses(self): + entry = self.save() + cached = self.bundle / "1.7/artifact/index.html" + cached.write_text("corrupt") + self.assertFalse(cache.reusable(self.bundle, entry)) + cached.write_text("page") + wrong = dict(entry, buildKey="wrong") + self.assertFalse(cache.reusable(self.bundle, wrong)) + (self.bundle / "1.7/receipt.json").write_text("invalid json") + self.assertFalse(cache.reusable(self.bundle, entry)) + + def test_symlink_and_traversal_rejected(self): + entry = self.save() + (self.bundle / "1.7/artifact/link").symlink_to(self.root / "hugo.yaml") + self.assertFalse(cache.reusable(self.bundle, entry)) + for version in ("../outside", "/tmp/outside", "..", "."): + with self.subTest(version=version), self.assertRaises(ValueError): + cache.version_directory(self.bundle, version) + + def test_plan_treats_symlink_cache_slot_as_miss_without_touching_target(self): + entry = self.save() + original = self.bundle / "1.7" + target = self.root / "external" + original.rename(target) + original.symlink_to(target, target_is_directory=True) + before = cache.snapshot(target) + plan = self.plan() + self.assertFalse(plan["versions"][1]["reusable"]) + self.assertIn("1.7", [e["id"] for e in plan["include"]]) + self.assertFalse(cache.restore(self.bundle, entry, self.root / "restored")) + with self.assertRaises(ValueError): + cache.record(self.bundle, entry, self.root / "artifact") + self.assertEqual(before, cache.snapshot(target)) + self.assertTrue(original.is_symlink()) + + def test_source_symlink_rejected(self): + (self.root / "data").symlink_to(self.root / "content", target_is_directory=True) + with self.assertRaises(ValueError): + self.plan() + + def test_shared_navigation_scripts_and_assets_invalidate(self): + for path in ("data/version_routes.json", "content/en/docs/SUMMARY.md", + "content/cn/docs/_nav/intro.md", "scripts/versioning.py", + "dist/url-contract.json", "static/favicon.svg"): + with self.subTest(path=path): + before = self.plan()["cacheKey"] + self.write(path, "changed") + self.assertNotEqual(before, self.plan()["cacheKey"]) + + def test_wrong_artifact_metadata_is_not_recorded(self): + entry = self.save() + self.write("artifact/.version.json", json.dumps({"id": "1.5", "sha": "b" * 40})) + with self.assertRaises(ValueError): + cache.record(self.bundle, entry, self.root / "artifact") + self.assertTrue(cache.reusable(self.bundle, entry)) + + def test_unarchived_version_need_not_be_named_latest(self): + self.manifest["versions"][0]["id"] = "current" + before = self.plan() + self.manifest["versions"][0]["sha"] = "d" * 40 + after = self.plan() + self.assertEqual(before["cacheKey"], after["cacheKey"]) + self.assertFalse(after["versions"][0]["reusable"]) + + def test_selection_is_honored(self): + self.manifest["include"] = [copy.deepcopy(self.manifest["versions"][0])] + self.assertEqual([e["id"] for e in self.plan()["versions"]], ["latest"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/version_cache.py b/scripts/version_cache.py new file mode 100644 index 0000000000..c7629cc511 --- /dev/null +++ b/scripts/version_cache.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exact historical build reuse; a receipt never replaces current validation.""" + +import argparse +import datetime +import hashlib +import json +import pathlib +import platform +import re +import shutil +import stat +import tempfile + +import versioning + + +SCHEMA = 1 + + +def digest_json(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def safe_path(path): + path = pathlib.Path(path).absolute() + if any(parent.is_symlink() for parent in (path, *path.parents)): + raise ValueError(f"symlink is not a cache input: {path}") + return path + + +def snapshot(path): + """Hash names, types and bytes, rejecting links and special files.""" + path = safe_path(path) + if not path.exists(): + return [["", "missing"]] + result = [] + for item in [path, *sorted(path.rglob("*"))] if path.is_dir() else [path]: + mode = item.lstat().st_mode + name = item.relative_to(path).as_posix() + if stat.S_ISDIR(mode): + result.append([name, "directory"]) + elif stat.S_ISREG(mode): + result.append([name, hashlib.sha256(item.read_bytes()).hexdigest()]) + else: + raise ValueError(f"unsupported cache input: {item}") + return result + + +def shared_digest(root): + paths = {*versioning.SHELL_FILES, *versioning.SHELL_DIRS, + *versioning.SHELL_CONTENT_DIRS, "versions.json", "dist", ".htaccess", ".asf.yaml"} + paths.update(p for p in versioning.SHELL_CONTENT if p.endswith("docs/SUMMARY.md")) + paths.update("static/" + p for p in (*versioning.SHELL_STATIC_FILES, *versioning.SHELL_STATIC_DIRS)) + paths.update(p.relative_to(root).as_posix() for p in (root / "scripts").glob("*.py")) + paths.update(p.relative_to(root).as_posix() for p in (root / "scripts").glob("*.sh")) + return digest_json([[p, snapshot(root / p)] for p in sorted(paths)]) + + +def validate_version_id(version): + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", version): + raise ValueError("invalid version identifier") + + +def version_directory(cache_dir, version): + validate_version_id(version) + return safe_path(pathlib.Path(cache_dir) / version) + + +def reusable(cache_dir, entry): + if not entry["archived"]: + return False + try: + directory = version_directory(cache_dir, entry["id"]) + receipt_path = safe_path(directory / "receipt.json") + receipt = json.loads(receipt_path.read_text()) + artifact = safe_path(directory / "artifact") + return (artifact.is_dir() and (artifact / ".version.json").is_file() + and receipt["schemaVersion"] == SCHEMA + and receipt["buildKey"] == entry["buildKey"] + and receipt["treeDigest"] == digest_json(snapshot(artifact))) + except (OSError, ValueError, KeyError, TypeError): + return False + + +def make_plan(root, manifest, cache_dir, *, site_origin, historical_origin, + hugo_version, go_version, webp_version, year=None): + common = {"schemaVersion": SCHEMA, "shared": shared_digest(root), + "siteOrigin": site_origin, "historicalOrigin": historical_origin, + "hugo": hugo_version, "go": go_version, "webp": webp_version, + "python": platform.python_version(), "system": platform.system(), + "machine": platform.machine(), "year": year or datetime.datetime.now(datetime.timezone.utc).year} + entries = [] + for source in manifest.get("include", manifest["versions"]): + entry = dict(source) + validate_version_id(entry["id"]) + entry["buildKey"] = digest_json({"common": common, "version": source, + "repository": manifest["repository"]}) + entry["reusable"] = reusable(cache_dir, entry) + entries.append(entry) + return {"schemaVersion": SCHEMA, "cacheKey": "history-v1-" + digest_json( + [[e["id"], e["buildKey"]] for e in entries if e["archived"]]), + "versions": entries, "include": [e for e in entries if not e["reusable"]]} + + +def record(cache_dir, entry, artifact): + if not entry["archived"]: + raise ValueError("latest artifacts are not reusable") + artifact = safe_path(artifact) + if not artifact.is_dir() or not (artifact / ".version.json").is_file(): + raise ValueError("artifact is missing version metadata") + tree_digest = digest_json(snapshot(artifact)) + metadata = json.loads((artifact / ".version.json").read_text()) + if any(metadata.get(field) != entry.get(field) for field in ("id", "sha")): + raise ValueError("artifact metadata does not match planned version") + destination = version_directory(cache_dir, entry["id"]) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="record-", dir=destination.parent) as temporary: + staging = pathlib.Path(temporary) / "version" + staging.mkdir() + shutil.copytree(artifact, staging / "artifact") + (staging / "receipt.json").write_text(json.dumps({"schemaVersion": SCHEMA, + "buildKey": entry["buildKey"], "treeDigest": tree_digest})) + if destination.exists(): + shutil.rmtree(destination) + staging.rename(destination) + + +def restore(cache_dir, entry, output): + if not reusable(cache_dir, entry): + return False + output = safe_path(output) + if output.exists(): + raise ValueError("restore output must not exist") + shutil.copytree(version_directory(cache_dir, entry["id"]) / "artifact", output) + return True + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + plan = commands.add_parser("plan") + plan.add_argument("--resolved-manifest", type=pathlib.Path, required=True) + plan.add_argument("--site-origin", required=True) + plan.add_argument("--historical-origin", required=True) + plan.add_argument("--hugo-version", default="0.165.0") + plan.add_argument("--go-version", required=True) + plan.add_argument("--webp-version", default="unspecified") + plan.add_argument("--output", type=pathlib.Path, required=True) + for name in ("restore", "record"): + command = commands.add_parser(name) + command.add_argument("--plan", type=pathlib.Path, required=True) + command.add_argument("--version", required=True) + command.add_argument("--output" if name == "restore" else "--artifact", type=pathlib.Path, required=True) + command.add_argument("--cache-dir", type=pathlib.Path, required=True) + plan.add_argument("--cache-dir", type=pathlib.Path, required=True) + args = parser.parse_args() + if args.command == "plan": + manifest = versioning.load_resolved_manifest(args.resolved_manifest) + result = make_plan(versioning.ROOT, manifest, args.cache_dir, + site_origin=args.site_origin, historical_origin=args.historical_origin, + hugo_version=args.hugo_version, go_version=args.go_version, webp_version=args.webp_version) + args.output.write_text(json.dumps(result, indent=2) + "\n") + else: + plan_data = json.loads(args.plan.read_text()) + entry = next(e for e in plan_data["versions"] if e["id"] == args.version) + if args.command == "record": + record(args.cache_dir, entry, args.artifact) + elif not restore(args.cache_dir, entry, args.output): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/workflow-contract.test.cjs b/tests/e2e/workflow-contract.test.cjs index 9550076944..89552dcb9e 100644 --- a/tests/e2e/workflow-contract.test.cjs +++ b/tests/e2e/workflow-contract.test.cjs @@ -60,9 +60,8 @@ test("trusted workflow bounds the candidate module graph without pinning its ver test("dependency artifacts keep stable names across selective reruns", () => { assert.match(workflow, /name: resolved-versions-\$\{\{ github\.run_id \}\}/); - assert.match(workflow, /name: \$\{\{ needs\.prepare\.outputs\.artifact_prefix \}\}-\$\{\{ matrix\.version\.id \}\}-\$\{\{ github\.run_id \}\}/); - assert.match(workflow, /pattern: \$\{\{ needs\.prepare\.outputs\.artifact_prefix \}\}-\*-\$\{\{ github\.run_id \}\}/); - assert.match(workflow, /--artifact-suffix="-\$\{GITHUB_RUN_ID\}"/); + assert.match(workflow, /name: \$\{\{ needs\.prepare\.outputs\.artifact_prefix \}\}-versions-\$\{\{ github\.run_id \}\}/); + assert.match(workflow, /--artifacts version-artifacts --workers 2/); assert.match(workflow, /name: hugegraph-site-\$\{\{ needs\.prepare\.outputs\.artifact_prefix \}\}-\$\{\{ github\.run_id \}\}/); assert.doesNotMatch(workflow, /name: (?:resolved-versions|hugegraph-site-[^\n]+)-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}/); assert.equal((workflow.match(/\n\s+overwrite: true/g) ?? []).length, 3); @@ -105,3 +104,16 @@ test("required gate rejects failed, skipped and cancelled prerequisites", () => } } }); + + +test("version work shares a runner with bounded concurrency and full validation", () => { + const build = jobBody("build"); + assert.doesNotMatch(build, /matrix:|strategy:/); + assert.match(build, /scripts\/build_versions\.py/); + assert.match(build, /--workers 2/); + assert.match(build, /name: Restore historical artifact bundle/); + assert.match(build, /if: github.event_name != 'workflow_dispatch'/); + assert.match(build, /scripts\/version_cache\.py plan/); + assert.match(jobBody("aggregate"), /scripts\/versioning\.py aggregate/); + assert.match(jobBody("publish"), /keep_files: false/); +}); From 98b8c78e27ca299129e1170610bda2511da1246a Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 2 Oct 2026 19:46:54 +0800 Subject: [PATCH 03/13] refactor: bound final security scan concurrency - scan HTML security contracts with two local workers - preserve ordered diagnostics and all validation checks - use three build slots on the four-CPU runner - verify serial and parallel results on full artifacts --- .github/workflows/hugo.yml | 2 +- contribution.md | 6 ++- dist/validate-site-output.py | 75 ++++++++++++++++++++++------ scripts/test_validate_site_output.py | 65 ++++++++++++++++++++++++ scripts/test_versioning.py | 7 +-- scripts/versioning.py | 7 ++- tests/e2e/workflow-contract.test.cjs | 2 +- 7 files changed, 140 insertions(+), 24 deletions(-) diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index 23713b5c0f..15d938ce82 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -263,7 +263,7 @@ jobs: --resolved-manifest resolved/resolved-versions.json \ --plan "$RUNNER_TEMP/cache-plan.json" --cache-dir "$RUNNER_TEMP/history-cache" \ --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ - --workers 2 --output "$RUNNER_TEMP/version-artifacts" + --workers 3 --output "$RUNNER_TEMP/version-artifacts" - name: Upload selected version artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: diff --git a/contribution.md b/contribution.md index d7ff381af5..1346c7137a 100644 --- a/contribution.md +++ b/contribution.md @@ -72,14 +72,16 @@ Cancelled runs skip report uploads and the final gate instead of holding the queue with `always()`. A queued job with no runner has not started testing; repeated reruns do not resolve runner capacity shortages. -Selected versions share one build runner with at most two builds in parallel. +Selected versions share one build runner with at most three builds in parallel. Historical artifacts are reused only when both the complete build-input fingerprint and artifact digest match. Missing or invalid cache entries rebuild automatically; manual dispatches start without the artifact cache. Cache hits still run the full version validator, and aggregation validates every selected version again. Assembly validates at most two versions concurrently, then writes the complete -site and runs the cross-version and security checks. Stage timings are printed as +site and runs the cross-version checks. The final security scan also uses at most +two processes; it starts after version validation finishes, so these pools do not +nest. All checks remain enabled. Stage timings are printed as `timing : s`; nested security timings are included in their parent validation duration and must not be added twice. Site assembly and blocking browser tests share one runner and the same local artifact. The required `deploy` check still diff --git a/dist/validate-site-output.py b/dist/validate-site-output.py index 4b9894037c..66f3852e55 100644 --- a/dist/validate-site-output.py +++ b/dist/validate-site-output.py @@ -19,6 +19,8 @@ from __future__ import annotations import argparse +from concurrent.futures import ProcessPoolExecutor +from functools import partial import html.parser import json import pathlib @@ -1091,6 +1093,37 @@ def document_url_shape_errors( return errors +def parse_and_check_page_security( + page: pathlib.Path, + root: pathlib.Path, + base_parts: urllib.parse.SplitResult, + error_paths: set[str], +) -> tuple[DocumentParser | None, list[str]]: + parser = DocumentParser() + try: + parser.feed(page.read_text(encoding="utf-8")) + except (OSError, UnicodeError) as exc: + return None, [f"cannot parse {page.relative_to(root)}: {exc}"] + + page_name = page.relative_to(root).as_posix() + errors = document_url_shape_errors(parser, page_name) + if errors: + return None, errors + errors.extend(document_security_errors(parser, page_name, base_parts)) + errors.extend(error_document_seo_errors(parser, page_name, error_paths)) + return parser, errors + + +def page_security_errors( + page: pathlib.Path, + root: pathlib.Path, + base_parts: urllib.parse.SplitResult, + error_paths: set[str], +) -> list[str]: + # Keep the parsed document local to the worker instead of serializing it. + return parse_and_check_page_security(page, root, base_parts, error_paths)[1] + + def main() -> int: argument_parser = argparse.ArgumentParser( description="Validate a generated HugeGraph documentation artifact." @@ -1105,6 +1138,13 @@ def main() -> int: "boundaries only; versioning.py validates isolated URL contracts." ), ) + argument_parser.add_argument( + "--workers", + type=int, + choices=(1, 2), + default=1, + help="Number of HTML security-check processes in --security-only mode.", + ) args = argument_parser.parse_args() root = args.public_dir.resolve() @@ -1148,23 +1188,28 @@ def main() -> int: f"route leaked into search: {ref}" ) - for page in html_files: - parser = DocumentParser() - try: - parser.feed(page.read_text(encoding="utf-8")) - except (OSError, UnicodeError) as exc: - errors.append(f"cannot parse {page.relative_to(root)}: {exc}") + if args.security_only: + check_page = partial( + page_security_errors, root=root, base_parts=base_parts, + error_paths=error_paths, + ) + if args.workers == 2 and html_files: + with ProcessPoolExecutor(max_workers=args.workers) as executor: + # map preserves sorted page order, including diagnostics. + for page_errors in executor.map(check_page, html_files): + errors.extend(page_errors) + else: + for page_errors in map(check_page, html_files): + errors.extend(page_errors) + + for page in [] if args.security_only else html_files: + parser, page_errors = parse_and_check_page_security( + page, root, base_parts, error_paths + ) + errors.extend(page_errors) + if parser is None: continue - page_name = page.relative_to(root).as_posix() - shape_errors = document_url_shape_errors(parser, page_name) - errors.extend(shape_errors) - if shape_errors: - continue - errors.extend(document_security_errors(parser, page_name, base_parts)) - errors.extend(error_document_seo_errors(parser, page_name, error_paths)) - if args.security_only: - continue errors.extend(toc_accessibility_errors(parser, page_name)) try: alias_target = refresh_target(parser) diff --git a/scripts/test_validate_site_output.py b/scripts/test_validate_site_output.py index 7a71ae696c..cd3561d53e 100644 --- a/scripts/test_validate_site_output.py +++ b/scripts/test_validate_site_output.py @@ -330,6 +330,71 @@ def test_security_only_mode_skips_aggregate_url_contracts(self) -> None: self.assertEqual(result.returncode, 1) self.assertIn("external CSS resource is forbidden", result.stdout) + def test_security_workers_preserve_cli_results(self) -> None: + with tempfile.TemporaryDirectory() as temp_name: + root = pathlib.Path(temp_name) + + def run(*options: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, str(VALIDATOR_PATH), str(root), + "https://hugegraph.apache.org/", *options], + check=False, capture_output=True, + ) + + def assert_same_results(expected_code: int) -> bytes: + default = run("--security-only") + self.assertEqual(default.returncode, expected_code, default.stderr) + for workers in ("1", "2"): + result = run("--security-only", "--workers", workers) + self.assertEqual( + (result.returncode, result.stdout, result.stderr), + (default.returncode, default.stdout, default.stderr), + ) + return default.stdout + + (root / "index.html").write_text("
safe
", encoding="utf-8") + (root / "404.html").write_text( + '', encoding="utf-8" + ) + (root / "site.css").write_text("body{color:black}", encoding="utf-8") + assert_same_results(0) + + (root / "a-invalid-url.html").write_text( + '
bad' + '
', + encoding="utf-8", + ) + (root / "b-invalid-utf8.html").write_bytes(b"
\xff
") + (root / "c-unsafe.html").write_text( + "
", encoding="utf-8" + ) + (root / "404.html").write_text( + '', encoding="utf-8" + ) + (root / "site.css").write_text( + 'body{background:url("missing.png")}', encoding="utf-8" + ) + output = assert_same_results(1) + for diagnostic in ( + b"malformed URL", b"cannot parse b-invalid-utf8.html", + b"c-unsafe.html: unsafe content markup: authored ' diff --git a/scripts/test_versioning.py b/scripts/test_versioning.py index 92e6d6d832..f310ccb0b4 100644 --- a/scripts/test_versioning.py +++ b/scripts/test_versioning.py @@ -1855,7 +1855,7 @@ def test_aggregate_parallel_copy_matches_serial_output(self) -> None: ): versioning.aggregate(args) routes.assert_called_once() - security.assert_called_once_with(args.output.resolve(), ORIGIN) + security.assert_called_once_with(args.output.resolve(), ORIGIN, workers=workers) outputs.append({ path.relative_to(args.output).as_posix(): path.read_bytes() for path in args.output.rglob("*") if path.is_file() @@ -1929,7 +1929,8 @@ def test_aggregate_security_scan_runs_after_metadata_is_written(self) -> None: asf_whoami="asf-staging-oink", ) - def assert_complete_aggregate(path: Path, origin: str) -> None: + def assert_complete_aggregate(path: Path, origin: str, *, workers: int) -> None: + self.assertEqual(workers, 1) self.assertEqual(path, output.resolve()) self.assertEqual(origin, ORIGIN) self.assertTrue((path / ".asf.yaml").is_file()) @@ -1976,7 +1977,7 @@ def assert_complete_aggregate(path: Path, origin: str) -> None: ): versioning.aggregate(args) - security_scan.assert_called_once_with(output.resolve(), ORIGIN) + security_scan.assert_called_once_with(output.resolve(), ORIGIN, workers=1) validate_routes.assert_called_once() validate_args = validate_artifact.call_args.args[0] self.assertEqual( diff --git a/scripts/versioning.py b/scripts/versioning.py index 536db7ed0c..2e37d3a435 100644 --- a/scripts/versioning.py +++ b/scripts/versioning.py @@ -4337,7 +4337,9 @@ def write_error_documents(output: pathlib.Path, seen: set[str]) -> int: return count -def validate_output_security(output: pathlib.Path, site_origin: str) -> None: +def validate_output_security( + output: pathlib.Path, site_origin: str, *, workers: int = 1 +) -> None: """Re-scan a complete output tree before it can be published.""" with timed_stage(f"security[{output.name}]"): subprocess.run( @@ -4347,6 +4349,7 @@ def validate_output_security(output: pathlib.Path, site_origin: str) -> None: str(output), site_origin, "--security-only", + "--workers", str(workers), ], cwd=ROOT, check=True, @@ -4499,7 +4502,7 @@ def aggregate(args: argparse.Namespace) -> None: with timed_stage("aggregate-routes"): validate_aggregate_version_routes(output, route_map, selected, manifest) with timed_stage("aggregate-security"): - validate_output_security(output, args.site_origin) + validate_output_security(output, args.site_origin, workers=workers) print( f"aggregated {len(resolved)} versions and {len(seen)} files " f"with {error_documents} error documents -> {output}" diff --git a/tests/e2e/workflow-contract.test.cjs b/tests/e2e/workflow-contract.test.cjs index 89552dcb9e..58f82ad5ea 100644 --- a/tests/e2e/workflow-contract.test.cjs +++ b/tests/e2e/workflow-contract.test.cjs @@ -110,7 +110,7 @@ test("version work shares a runner with bounded concurrency and full validation" const build = jobBody("build"); assert.doesNotMatch(build, /matrix:|strategy:/); assert.match(build, /scripts\/build_versions\.py/); - assert.match(build, /--workers 2/); + assert.match(build, /--workers 3/); assert.match(build, /name: Restore historical artifact bundle/); assert.match(build, /if: github.event_name != 'workflow_dispatch'/); assert.match(build, /scripts\/version_cache\.py plan/); From c0039080689a87333322c6c3182bc30e27d41b50 Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 2 Oct 2026 20:03:10 +0800 Subject: [PATCH 04/13] refactor: adapt build groups to cache availability - use one warm group and at most three cold groups - verify build fingerprints across prepared runners - save complete history bundles after validation - exclude stale rerun artifacts and tolerate cache outages --- .github/workflows/hugo.yml | 124 +++++++++++++++++++++------ contribution.md | 3 +- scripts/test_version_cache.py | 40 +++++++++ scripts/version_cache.py | 30 +++++++ tests/e2e/workflow-contract.test.cjs | 66 +++++++++++++- 5 files changed, 234 insertions(+), 29 deletions(-) diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index 15d938ce82..7c900fcc0f 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -52,6 +52,11 @@ jobs: publish_branch: ${{ steps.plan.outputs.publish_branch }} latest_sha: ${{ steps.plan.outputs.latest_sha }} source_sha: ${{ steps.plan.outputs.source_sha }} + python_version: ${{ steps.python.outputs.python-version }} + cache_key: ${{ steps.history-plan.outputs.key }} + history_hit: ${{ steps.history-lookup.outcome == 'success' && steps.history-lookup.outputs.cache-hit || 'false' }} + groups: ${{ steps.groups.outputs.groups }} + group_pattern: ${{ steps.groups.outputs.pattern }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -59,6 +64,7 @@ jobs: fetch-depth: 0 persist-credentials: false - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + id: python with: python-version: "3.13" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 @@ -174,12 +180,50 @@ jobs: python3 scripts/versioning.py prepare \ --latest-sha "$LATEST_SHA" --select "$SELECTION" \ --output resolved-versions.json + - name: Compute historical build fingerprints + id: history-plan + env: + SITE_ORIGIN: ${{ steps.plan.outputs.site_origin }} + HISTORICAL_ORIGIN: ${{ steps.plan.outputs.historical_origin }} + run: | + python3 scripts/version_cache.py plan \ + --resolved-manifest resolved-versions.json \ + --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ + --hugo-version "$HUGO_VERSION" --go-version "$(go version)" \ + --cache-dir "$RUNNER_TEMP/history-cache" --output cache-plan.json + echo "key=$(jq -r .cacheKey cache-plan.json)" >> "$GITHUB_OUTPUT" + - name: Look up the complete historical bundle + id: history-lookup + continue-on-error: true + if: github.event_name != 'workflow_dispatch' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ${{ runner.temp }}/history-cache + key: ${{ runner.os }}-${{ steps.history-plan.outputs.key }} + lookup-only: true + - name: Select bounded build groups + id: groups + env: + CACHE_HIT: ${{ steps.history-lookup.outcome == 'success' && steps.history-lookup.outputs.cache-hit || 'false' }} + run: | + python3 scripts/version_cache.py groups \ + --resolved-manifest resolved-versions.json --cache-hit "$CACHE_HIT" > build-groups.json + echo "groups=$(jq -c . build-groups.json)" >> "$GITHUB_OUTPUT" + # A full rerun can switch from three cold groups to one warm group. + # Match only that current group, never the older attempt's artifacts. + if [[ "$(jq length build-groups.json)" == 1 ]]; then + echo 'pattern=0' >> "$GITHUB_OUTPUT" + else + echo 'pattern=*' >> "$GITHUB_OUTPUT" + fi - name: Upload resolved version manifest uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: # Keep dependency artifact names stable across selective job reruns. name: resolved-versions-${{ github.run_id }} - path: resolved-versions.json + path: | + resolved-versions.json + cache-plan.json retention-days: 7 if-no-files-found: error overwrite: true @@ -191,7 +235,12 @@ jobs: permissions: { contents: read } env: HUGO_CACHEDIR: /tmp/hugo-resources - name: Build selected versions + strategy: + fail-fast: false + max-parallel: 3 + matrix: + group: ${{ fromJSON(needs.prepare.outputs.groups) }} + name: Build group ${{ matrix.group.id }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -200,7 +249,7 @@ jobs: persist-credentials: false - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: - python-version: "3.13" + python-version: ${{ needs.prepare.outputs.python_version }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: resolved-versions-${{ github.run_id }} @@ -218,33 +267,30 @@ jobs: uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: ${{ env.HUGO_CACHEDIR }} - key: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-all-${{ hashFiles('go.sum') }} - restore-keys: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-all- + key: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-group${{ matrix.group.id }}-${{ hashFiles('go.sum') }} + restore-keys: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-group${{ matrix.group.id }}- - name: Verify pinned OINK module run: | + lscpu | sed -n '/^CPU(s):/p; /^Model name:/p' test "$(go list -m -f '{{ .Path }}')" = "github.com/apache/hugegraph-doc" test "$(go list -m all | wc -l)" -eq 2 test -z "$(go list -m -f '{{ with .Replace }}{{ .Path }}{{ end }}' github.com/pgsty/oink)" python3 scripts/update_oink.py --check-baseline - - name: Compute historical build fingerprints - id: cache-plan + - name: Select this group's immutable versions env: - SITE_ORIGIN: ${{ needs.prepare.outputs.site_origin }} - HISTORICAL_ORIGIN: ${{ needs.prepare.outputs.historical_origin }} + VERSION_IDS: ${{ toJSON(matrix.group.versions) }} run: | - python3 scripts/version_cache.py plan \ - --resolved-manifest resolved/resolved-versions.json \ - --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ - --hugo-version "$HUGO_VERSION" --go-version "$(go version)" \ - --cache-dir "$RUNNER_TEMP/history-cache" --output "$RUNNER_TEMP/cache-plan.json" - echo "key=$(jq -r .cacheKey "$RUNNER_TEMP/cache-plan.json")" >> "$GITHUB_OUTPUT" + jq --argjson ids "$VERSION_IDS" \ + '.include |= map(select(.id as $id | $ids | index($id)))' \ + resolved/resolved-versions.json > selected-versions.json - name: Restore historical artifact bundle id: history-cache + continue-on-error: true if: github.event_name != 'workflow_dispatch' - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: ${{ runner.temp }}/history-cache - key: ${{ runner.os }}-${{ steps.cache-plan.outputs.key }} + key: ${{ runner.os }}-${{ needs.prepare.outputs.cache_key }} # A previous bundle is only transport: each version must pass its # exact input fingerprint and file digest before it can be reused. restore-keys: ${{ runner.os }}-history-v1- @@ -253,21 +299,26 @@ jobs: OINK_PYTHON: python3 SITE_ORIGIN: ${{ needs.prepare.outputs.site_origin }} HISTORICAL_ORIGIN: ${{ needs.prepare.outputs.historical_origin }} + EVENT_NAME: ${{ github.event_name }} run: | + expected=() + if [[ "$EVENT_NAME" != workflow_dispatch ]]; then + expected+=(--expected-plan resolved/cache-plan.json) + fi python3 scripts/version_cache.py plan \ - --resolved-manifest resolved/resolved-versions.json \ + --resolved-manifest selected-versions.json \ --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ --hugo-version "$HUGO_VERSION" --go-version "$(go version)" \ - --cache-dir "$RUNNER_TEMP/history-cache" --output "$RUNNER_TEMP/cache-plan.json" + --cache-dir "$RUNNER_TEMP/history-cache" --output "$RUNNER_TEMP/cache-plan.json" "${expected[@]}" python3 scripts/build_versions.py \ - --resolved-manifest resolved/resolved-versions.json \ + --resolved-manifest selected-versions.json \ --plan "$RUNNER_TEMP/cache-plan.json" --cache-dir "$RUNNER_TEMP/history-cache" \ --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ - --workers 3 --output "$RUNNER_TEMP/version-artifacts" + --workers 2 --output "$RUNNER_TEMP/version-artifacts" - name: Upload selected version artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: ${{ needs.prepare.outputs.artifact_prefix }}-versions-${{ github.run_id }} + name: ${{ needs.prepare.outputs.artifact_prefix }}-versions-${{ matrix.group.id }}-${{ github.run_id }} path: ${{ runner.temp }}/version-artifacts include-hidden-files: true retention-days: 1 @@ -289,14 +340,15 @@ jobs: persist-credentials: false - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: - python-version: "3.13" + python-version: ${{ needs.prepare.outputs.python_version }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: resolved-versions-${{ github.run_id }} path: resolved - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - name: ${{ needs.prepare.outputs.artifact_prefix }}-versions-${{ github.run_id }} + pattern: ${{ needs.prepare.outputs.artifact_prefix }}-versions-${{ needs.prepare.outputs.group_pattern }}-${{ github.run_id }} + merge-multiple: true path: version-artifacts - name: Assemble publishable site env: @@ -305,6 +357,7 @@ jobs: HISTORICAL_ORIGIN: ${{ needs.prepare.outputs.historical_origin }} SELECTION: ${{ needs.prepare.outputs.selection }} run: | + lscpu | sed -n '/^CPU(s):/p; /^Model name:/p' extra=() if [[ "$PREFIX" == staging ]]; then extra+=(--asf-profile oink --asf-whoami asf-staging-oink) @@ -319,6 +372,29 @@ jobs: env: DOWNLOAD_PUBLIC_DIR: ${{ runner.temp }}/public-site run: python3 -m unittest scripts.test_download_data.DownloadDataTest.test_rendered_download_pages_have_verified_rows -v + - name: Record validated historical artifacts + id: history-record + if: github.event_name != 'workflow_dispatch' && needs.prepare.outputs.history_hit != 'true' + run: | + complete=false + while IFS= read -r version; do + if python3 scripts/version_cache.py record \ + --plan resolved/cache-plan.json --version "$version" \ + --artifact "version-artifacts/$version" --cache-dir "$RUNNER_TEMP/history-cache"; then + complete=true + else + echo '::warning::Could not record history cache; validated site remains publishable.' + complete=false + break + fi + done < <(jq -r '.versions[] | select(.archived) | .id' resolved/cache-plan.json) + echo "complete=$complete" >> "$GITHUB_OUTPUT" + - name: Save complete historical bundle + if: steps.history-record.outputs.complete == 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ${{ runner.temp }}/history-cache + key: ${{ runner.os }}-${{ needs.prepare.outputs.cache_key }} - name: Upload publishable aggregate uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: diff --git a/contribution.md b/contribution.md index 1346c7137a..60ebe2b499 100644 --- a/contribution.md +++ b/contribution.md @@ -72,7 +72,8 @@ Cancelled runs skip report uploads and the final gate instead of holding the queue with `always()`. A queued job with no runner has not started testing; repeated reruns do not resolve runner capacity shortages. -Selected versions share one build runner with at most three builds in parallel. +A complete historical cache hit uses one build job. Cold runs use at most three +build groups, with at most two versions building concurrently within each runner. Historical artifacts are reused only when both the complete build-input fingerprint and artifact digest match. Missing or invalid cache entries rebuild automatically; manual dispatches start without the artifact cache. Cache hits still run the full diff --git a/scripts/test_version_cache.py b/scripts/test_version_cache.py index 041ee3f1dd..9fee4c4fb0 100644 --- a/scripts/test_version_cache.py +++ b/scripts/test_version_cache.py @@ -157,6 +157,46 @@ def test_unarchived_version_need_not_be_named_latest(self): self.assertEqual(before["cacheKey"], after["cacheKey"]) self.assertFalse(after["versions"][0]["reusable"]) + def test_cold_groups_cover_every_selected_version_once_with_bounded_jobs(self): + for size in range(1, 11): + with self.subTest(size=size): + entries = [{"id": str(index)} for index in range(size)] + groups = cache.build_groups({"versions": entries}, False) + self.assertEqual([g["id"] for g in groups], [str(i) for i in range(min(3, (size + 1) // 2))]) + flattened = [v for g in groups for v in g["versions"]] + self.assertCountEqual(flattened, [e["id"] for e in entries]) + if size <= 6: + self.assertTrue(all(len(g["versions"]) <= 2 for g in groups)) + groups = cache.build_groups({"versions": [{"id": str(i)} for i in range(5)]}, False) + self.assertEqual(groups, [{"id": "0", "versions": ["0", "3"]}, + {"id": "1", "versions": ["1", "4"]}, + {"id": "2", "versions": ["2"]}]) + + def test_warm_groups_and_single_selection_use_one_job(self): + self.assertEqual(cache.build_groups(self.manifest, True), [ + {"id": "0", "versions": ["latest", "1.7", "1.5"]}]) + self.manifest["include"] = [self.manifest["versions"][1]] + self.assertEqual(cache.build_groups(self.manifest, False), [{"id": "0", "versions": ["1.7"]}]) + + def test_groups_reject_empty_duplicate_or_unsafe_selection(self): + for selection in ([], [{"id": "1.7"}, {"id": "1.7"}], [{"id": "../outside"}]): + with self.subTest(selection=selection), self.assertRaises(ValueError): + cache.build_groups(dict(self.manifest, include=selection), False) + + def test_expected_plan_accepts_subset_and_rejects_input_mismatch(self): + expected = self.plan() + self.manifest["include"] = [self.manifest["versions"][1]] + cache.verify_expected_plan(self.plan(), expected) + self.options["go_version"] = "different" + with self.assertRaisesRegex(ValueError, "build inputs differ"): + cache.verify_expected_plan(self.plan(), expected) + actual = {"versions": [{"id": "unknown", "buildKey": "key"}]} + with self.assertRaises(ValueError): + cache.verify_expected_plan(actual, expected) + expected["versions"].append(expected["versions"][0]) + with self.assertRaisesRegex(ValueError, "duplicate"): + cache.verify_expected_plan(actual, expected) + def test_selection_is_honored(self): self.manifest["include"] = [copy.deepcopy(self.manifest["versions"][0])] self.assertEqual([e["id"] for e in self.plan()["versions"]], ["latest"]) diff --git a/scripts/version_cache.py b/scripts/version_cache.py index c7629cc511..6dc8406e2c 100644 --- a/scripts/version_cache.py +++ b/scripts/version_cache.py @@ -118,6 +118,27 @@ def make_plan(root, manifest, cache_dir, *, site_origin, historical_origin, "versions": entries, "include": [e for e in entries if not e["reusable"]]} +def build_groups(manifest, cache_hit): + """Bound cold-build jobs while keeping a warm bundle in one job.""" + selected = [entry["id"] for entry in manifest.get("include", manifest["versions"])] + for version in selected: + validate_version_id(version) + if not selected or len(set(selected)) != len(selected): + raise ValueError("selected versions must be nonempty and unique") + count = 1 if cache_hit else min(3, (len(selected) + 1) // 2) + return [{"id": str(index), "versions": selected[index::count]} for index in range(count)] + + +def verify_expected_plan(actual, expected): + """Builders may select a subset, but must reproduce every prepared key.""" + expected_keys = {entry["id"]: entry["buildKey"] for entry in expected["versions"]} + if len(expected_keys) != len(expected["versions"]): + raise ValueError("expected plan contains duplicate versions") + for entry in actual["versions"]: + if expected_keys.get(entry["id"]) != entry["buildKey"]: + raise ValueError(f"build inputs differ from prepared plan: {entry['id']}") + + def record(cache_dir, entry, artifact): if not entry["archived"]: raise ValueError("latest artifacts are not reusable") @@ -162,6 +183,10 @@ def main(): plan.add_argument("--go-version", required=True) plan.add_argument("--webp-version", default="unspecified") plan.add_argument("--output", type=pathlib.Path, required=True) + plan.add_argument("--expected-plan", type=pathlib.Path) + groups = commands.add_parser("groups") + groups.add_argument("--resolved-manifest", type=pathlib.Path, required=True) + groups.add_argument("--cache-hit", choices=("true", "false"), required=True) for name in ("restore", "record"): command = commands.add_parser(name) command.add_argument("--plan", type=pathlib.Path, required=True) @@ -175,7 +200,12 @@ def main(): result = make_plan(versioning.ROOT, manifest, args.cache_dir, site_origin=args.site_origin, historical_origin=args.historical_origin, hugo_version=args.hugo_version, go_version=args.go_version, webp_version=args.webp_version) + if args.expected_plan: + verify_expected_plan(result, json.loads(args.expected_plan.read_text())) args.output.write_text(json.dumps(result, indent=2) + "\n") + elif args.command == "groups": + manifest = versioning.load_resolved_manifest(args.resolved_manifest) + print(json.dumps(build_groups(manifest, args.cache_hit == "true"))) else: plan_data = json.loads(args.plan.read_text()) entry = next(e for e in plan_data["versions"] if e["id"] == args.version) diff --git a/tests/e2e/workflow-contract.test.cjs b/tests/e2e/workflow-contract.test.cjs index 58f82ad5ea..d411a4ee82 100644 --- a/tests/e2e/workflow-contract.test.cjs +++ b/tests/e2e/workflow-contract.test.cjs @@ -60,7 +60,7 @@ test("trusted workflow bounds the candidate module graph without pinning its ver test("dependency artifacts keep stable names across selective reruns", () => { assert.match(workflow, /name: resolved-versions-\$\{\{ github\.run_id \}\}/); - assert.match(workflow, /name: \$\{\{ needs\.prepare\.outputs\.artifact_prefix \}\}-versions-\$\{\{ github\.run_id \}\}/); + assert.match(workflow, /name: \$\{\{ needs\.prepare\.outputs\.artifact_prefix \}\}-versions-\$\{\{ matrix\.group\.id \}\}-\$\{\{ github\.run_id \}\}/); assert.match(workflow, /--artifacts version-artifacts --workers 2/); assert.match(workflow, /name: hugegraph-site-\$\{\{ needs\.prepare\.outputs\.artifact_prefix \}\}-\$\{\{ github\.run_id \}\}/); assert.doesNotMatch(workflow, /name: (?:resolved-versions|hugegraph-site-[^\n]+)-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}/); @@ -106,14 +106,72 @@ test("required gate rejects failed, skipped and cancelled prerequisites", () => }); -test("version work shares a runner with bounded concurrency and full validation", () => { +test("version work uses bounded groups and preserves full validation", () => { const build = jobBody("build"); - assert.doesNotMatch(build, /matrix:|strategy:/); + assert.match(build, /max-parallel: 3/); + assert.match(build, /fromJSON\(needs.prepare.outputs.groups\)/); assert.match(build, /scripts\/build_versions\.py/); - assert.match(build, /--workers 3/); + assert.match(build, /--workers 2/); assert.match(build, /name: Restore historical artifact bundle/); assert.match(build, /if: github.event_name != 'workflow_dispatch'/); assert.match(build, /scripts\/version_cache\.py plan/); assert.match(jobBody("aggregate"), /scripts\/versioning\.py aggregate/); assert.match(jobBody("publish"), /keep_files: false/); }); + + +test("warm reruns exclude stale cold-group artifacts and only complete caches are saved", () => { + const prepare = jobBody("prepare"); + assert.match(prepare, /lookup-only: true/); + assert.match(prepare, /echo 'pattern=0'/); + assert.match(jobBody("aggregate"), /versions-\$\{\{ needs\.prepare\.outputs\.group_pattern \}\}/); + assert.match(jobBody("aggregate"), /merge-multiple: true/); + assert.match(jobBody("build"), /--expected-plan resolved\/cache-plan\.json/); + assert.match(jobBody("aggregate"), /if: steps.history-record.outputs.complete == 'true'/); +}); + +test("cache recording failures cannot block an already validated site", () => { + const { spawnSync } = require("node:child_process"); + const os = require("node:os"); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "history-record-")); + const block = jobBody("aggregate").split(" - name: Record validated historical artifacts\n")[1] + .split(" - name: Save complete historical bundle\n")[0]; + const script = block.split(" run: |\n")[1].split("\n") + .filter(line => line.startsWith(" ")).map(line => line.slice(10)).join("\n"); + try { + fs.mkdirSync(path.join(directory, "resolved")); + fs.writeFileSync(path.join(directory, "resolved/cache-plan.json"), JSON.stringify({ + versions: [{id: "latest", archived: false}, {id: "1.7", archived: true}] + })); + for (const code of [0, 1]) { + fs.writeFileSync(path.join(directory, "python3"), `#!/bin/sh\nexit ${code}\n`, {mode: 0o755}); + const output = path.join(directory, `outputs-${code}`); + const result = spawnSync("bash", ["-e", "-c", script], { + cwd: directory, + env: {...process.env, PATH: `${directory}:${process.env.PATH}`, GITHUB_OUTPUT: output, RUNNER_TEMP: directory} + }); + assert.equal(result.status, 0, result.stderr.toString()); + assert.equal(fs.readFileSync(output, "utf8"), `complete=${code === 0}\n`); + } + } finally { + fs.rmSync(directory, {recursive: true, force: true}); + } +}); + + +test("cache lookup and restore outages fall back without bypassing tests", () => { + for (const [job, id] of [["prepare", "history-lookup"], ["build", "history-cache"]]) { + assert.match(jobBody(job), new RegExp(`id: ${id}\\n continue-on-error: true`)); + assert.equal((jobBody(job).match(/continue-on-error:/g) ?? []).length, 1); + } + assert.match(jobBody("prepare"), /history-lookup.outcome == 'success'/); + assert.doesNotMatch(jobBody("aggregate"), /continue-on-error:/); +}); + + +test("fingerprints use the same resolved Python patch on all version runners", () => { + assert.match(jobBody("prepare"), /python_version: \$\{\{ steps.python.outputs.python-version \}\}/); + for (const job of ["build", "aggregate"]) { + assert.match(jobBody(job), /python-version: \$\{\{ needs.prepare.outputs.python_version \}\}/); + } +}); From 07546a8374b995eff54f048a8048fde2aa5f0fc1 Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 2 Oct 2026 20:19:30 +0800 Subject: [PATCH 05/13] refactor: run required gate on aggregate runner - preserve deploy check with explicit predecessor guards - remove the separate gate job and its queue delay - overlap cached validation with three local workers - keep all validation and publication dependencies - test failed skipped and cancelled prerequisites --- .github/workflows/hugo.yml | 32 +++++++++++----------------- contribution.md | 13 ++++++----- tests/e2e/workflow-contract.test.cjs | 24 ++++++++++++++------- 3 files changed, 36 insertions(+), 33 deletions(-) diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index 7c900fcc0f..da18b999db 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -314,7 +314,7 @@ jobs: --resolved-manifest selected-versions.json \ --plan "$RUNNER_TEMP/cache-plan.json" --cache-dir "$RUNNER_TEMP/history-cache" \ --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ - --workers 2 --output "$RUNNER_TEMP/version-artifacts" + --workers "${{ needs.prepare.outputs.history_hit == 'true' && 3 || 2 }}" --output "$RUNNER_TEMP/version-artifacts" - name: Upload selected version artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -327,6 +327,9 @@ jobs: # Assembly and browser checks are serial; reuse one runner and its local output. aggregate: + # Preserve the required check name without scheduling a separate gate runner. + name: deploy + if: ${{ !cancelled() }} needs: [prepare, build] runs-on: ubuntu-latest timeout-minutes: 35 @@ -334,6 +337,13 @@ jobs: env: HUGO_CACHEDIR: /tmp/hugo-cache-${{ github.run_id }}-${{ github.run_attempt }}-aggregate steps: + - name: Require all blocking predecessors to succeed + env: + PREPARE_RESULT: ${{ needs.prepare.result }} + BUILD_RESULT: ${{ needs.build.result }} + run: | + test "$PREPARE_RESULT" = success + test "$BUILD_RESULT" = success - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ needs.prepare.outputs.source_sha }} @@ -502,27 +512,9 @@ jobs: retention-days: 7 if-no-files-found: warn - # Required check name in .asf.yaml. It gates artifacts without write access. - deploy: - if: ${{ !cancelled() }} - needs: [prepare, build, aggregate] - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: { contents: read } - steps: - - name: Require all blocking jobs to succeed - env: - PREPARE_RESULT: ${{ needs.prepare.result }} - BUILD_RESULT: ${{ needs.build.result }} - AGGREGATE_RESULT: ${{ needs.aggregate.result }} - run: | - test "$PREPARE_RESULT" = success - test "$BUILD_RESULT" = success - test "$AGGREGATE_RESULT" = success - publish: if: needs.prepare.outputs.publish_branch != '' - needs: [prepare, deploy] + needs: [prepare, aggregate] runs-on: ubuntu-latest timeout-minutes: 10 permissions: { contents: write } diff --git a/contribution.md b/contribution.md index 60ebe2b499..7bb6b326cf 100644 --- a/contribution.md +++ b/contribution.md @@ -72,8 +72,9 @@ Cancelled runs skip report uploads and the final gate instead of holding the queue with `always()`. A queued job with no runner has not started testing; repeated reruns do not resolve runner capacity shortages. -A complete historical cache hit uses one build job. Cold runs use at most three -build groups, with at most two versions building concurrently within each runner. +A complete historical cache hit uses one build job with up to three local workers, +so cached-version validation can overlap the latest build. Cold runs use at most +three build groups, with at most two versions building concurrently within each runner. Historical artifacts are reused only when both the complete build-input fingerprint and artifact digest match. Missing or invalid cache entries rebuild automatically; manual dispatches start without the artifact cache. Cache hits still run the full @@ -85,9 +86,11 @@ two processes; it starts after version validation finishes, so these pools do no nest. All checks remain enabled. Stage timings are printed as `timing : s`; nested security timings are included in their parent validation duration and must not be added twice. Site assembly and blocking browser -tests share one runner and the same local artifact. The required `deploy` check still -requires all blocking jobs to succeed; visual captures remain advisory and -publication alone receives write permission. For a test failure, rerun failed +tests share one runner and the same local artifact. The assembly job carries the +required `deploy` check name, avoiding a separate gate runner and its queue time. +Its first step requires successful preparation and builds, and all assembly and +browser checks remain blocking. Visual captures remain advisory and publication +alone receives write permission. For a test failure, rerun failed jobs after inspecting the cause; artifact names remain stable within the run. ## Repository structure diff --git a/tests/e2e/workflow-contract.test.cjs b/tests/e2e/workflow-contract.test.cjs index d411a4ee82..d13799b551 100644 --- a/tests/e2e/workflow-contract.test.cjs +++ b/tests/e2e/workflow-contract.test.cjs @@ -77,7 +77,7 @@ test("superseded runs can cancel reporting and the required gate", () => { assert.match(workflow, /group:.*format\('pr-\{0\}', github.event.pull_request.number\)/); assert.match(workflow, /cancel-in-progress: true/); assert.doesNotMatch(workflow, /if: always\(\)/); - assert.match(jobBody("deploy"), /if: \$\{\{ !cancelled\(\) \}\}/); + assert.match(jobBody("aggregate"), /if: \$\{\{ !cancelled\(\) \}\}/); }); test("assembly runs blocking browser tests on its local artifact", () => { @@ -86,17 +86,25 @@ test("assembly runs blocking browser tests on its local artifact", () => { assert.match(aggregate, /SITE_ROOT: \$\{\{ runner.temp \}\}\/public-site/); assert.match(aggregate, /run: npm run test:ci/); assert.doesNotMatch(aggregate, /continue-on-error:/); - assert.match(jobBody("deploy"), /needs: \[prepare, build, aggregate\]/); - assert.match(jobBody("publish"), /needs: \[prepare, deploy\]/); + assert.match(aggregate, /^ name: deploy$/m); + assert.match(aggregate, /needs: \[prepare, build\]/); + assert.doesNotMatch(workflow, /^ deploy:/m); + assert.match(jobBody("publish"), /needs: \[prepare, aggregate\]/); }); test("required gate rejects failed, skipped and cancelled prerequisites", () => { const { spawnSync } = require("node:child_process"); - const gate = jobBody("deploy").split(" run: |\n")[1] + const aggregate = jobBody("aggregate"); + assert.match(aggregate, /if: \$\{\{ !cancelled\(\) \}\}/); + assert.match(aggregate, /steps:\n - name: Require all blocking predecessors to succeed/); + assert.ok(aggregate.indexOf('test "$BUILD_RESULT" = success') < aggregate.indexOf("- uses: actions/checkout@")); + assert.match(aggregate, /PREPARE_RESULT: \$\{\{ needs.prepare.result \}\}/); + assert.match(aggregate, /BUILD_RESULT: \$\{\{ needs.build.result \}\}/); + const gate = aggregate.split(" run: |\n")[1].split(" - uses:")[0] .split("\n").filter(line => line.startsWith(" ")) .map(line => line.slice(10)).join("\n"); - const success = { PREPARE_RESULT: "success", BUILD_RESULT: "success", AGGREGATE_RESULT: "success" }; - const run = env => spawnSync("bash", ["-e", "-c", gate], {env: {...process.env, ...env}}).status; + const success = { PREPARE_RESULT: "success", BUILD_RESULT: "success" }; + const run = env => spawnSync("bash", ["-e", "-c", gate], {env: {...process.env, BASH_ENV: "", ...env}}).status; assert.equal(run(success), 0); for (const key of Object.keys(success)) { for (const result of ["failure", "skipped", "cancelled"]) { @@ -111,7 +119,7 @@ test("version work uses bounded groups and preserves full validation", () => { assert.match(build, /max-parallel: 3/); assert.match(build, /fromJSON\(needs.prepare.outputs.groups\)/); assert.match(build, /scripts\/build_versions\.py/); - assert.match(build, /--workers 2/); + assert.match(build, /history_hit == 'true' && 3 \|\| 2/); assert.match(build, /name: Restore historical artifact bundle/); assert.match(build, /if: github.event_name != 'workflow_dispatch'/); assert.match(build, /scripts\/version_cache\.py plan/); @@ -148,7 +156,7 @@ test("cache recording failures cannot block an already validated site", () => { const output = path.join(directory, `outputs-${code}`); const result = spawnSync("bash", ["-e", "-c", script], { cwd: directory, - env: {...process.env, PATH: `${directory}:${process.env.PATH}`, GITHUB_OUTPUT: output, RUNNER_TEMP: directory} + env: {...process.env, BASH_ENV: "", PATH: `${directory}:${process.env.PATH}`, GITHUB_OUTPUT: output, RUNNER_TEMP: directory} }); assert.equal(result.status, 0, result.stderr.toString()); assert.equal(fs.readFileSync(output, "utf8"), `complete=${code === 0}\n`); From bc4514d1e6e7f04699300fa514f7b21cf71aa825 Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 2 Oct 2026 20:38:10 +0800 Subject: [PATCH 06/13] refactor: avoid warm build CPU contention - restore two local workers after CI comparison - retain adaptive groups and the combined required gate - align workflow contracts and contributor guidance --- .github/workflows/hugo.yml | 2 +- contribution.md | 5 ++--- tests/e2e/workflow-contract.test.cjs | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index da18b999db..ffcc92cad4 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -314,7 +314,7 @@ jobs: --resolved-manifest selected-versions.json \ --plan "$RUNNER_TEMP/cache-plan.json" --cache-dir "$RUNNER_TEMP/history-cache" \ --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ - --workers "${{ needs.prepare.outputs.history_hit == 'true' && 3 || 2 }}" --output "$RUNNER_TEMP/version-artifacts" + --workers 2 --output "$RUNNER_TEMP/version-artifacts" - name: Upload selected version artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: diff --git a/contribution.md b/contribution.md index 7bb6b326cf..2d7f8dbf9f 100644 --- a/contribution.md +++ b/contribution.md @@ -72,9 +72,8 @@ Cancelled runs skip report uploads and the final gate instead of holding the queue with `always()`. A queued job with no runner has not started testing; repeated reruns do not resolve runner capacity shortages. -A complete historical cache hit uses one build job with up to three local workers, -so cached-version validation can overlap the latest build. Cold runs use at most -three build groups, with at most two versions building concurrently within each runner. +A complete historical cache hit uses one build job. Cold runs use at most three +build groups, with at most two versions building concurrently within each runner. Historical artifacts are reused only when both the complete build-input fingerprint and artifact digest match. Missing or invalid cache entries rebuild automatically; manual dispatches start without the artifact cache. Cache hits still run the full diff --git a/tests/e2e/workflow-contract.test.cjs b/tests/e2e/workflow-contract.test.cjs index d13799b551..1715b1b9e5 100644 --- a/tests/e2e/workflow-contract.test.cjs +++ b/tests/e2e/workflow-contract.test.cjs @@ -119,7 +119,7 @@ test("version work uses bounded groups and preserves full validation", () => { assert.match(build, /max-parallel: 3/); assert.match(build, /fromJSON\(needs.prepare.outputs.groups\)/); assert.match(build, /scripts\/build_versions\.py/); - assert.match(build, /history_hit == 'true' && 3 \|\| 2/); + assert.match(build, /--workers 2/); assert.match(build, /name: Restore historical artifact bundle/); assert.match(build, /if: github.event_name != 'workflow_dispatch'/); assert.match(build, /scripts\/version_cache\.py plan/); From 9b5db4811ee66fffc51423f84d53610a0a4bdac6 Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 2 Oct 2026 20:57:13 +0800 Subject: [PATCH 07/13] fix: preserve manual builds and update node action - keep manual candidates compatible with older build scripts - verify manual build arguments and failure propagation - move setup-node to the pinned Node 24 action runtime --- .github/workflows/hugo.yml | 43 ++++++++++++---- contribution.md | 3 +- tests/e2e/workflow-contract.test.cjs | 76 ++++++++++++++++++++++++++-- 3 files changed, 109 insertions(+), 13 deletions(-) diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index ffcc92cad4..05c010d6ad 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -295,26 +295,47 @@ jobs: # exact input fingerprint and file digest before it can be reused. restore-keys: ${{ runner.os }}-history-v1- - name: Build and validate selected versions + if: github.event_name != 'workflow_dispatch' env: OINK_PYTHON: python3 SITE_ORIGIN: ${{ needs.prepare.outputs.site_origin }} HISTORICAL_ORIGIN: ${{ needs.prepare.outputs.historical_origin }} - EVENT_NAME: ${{ github.event_name }} run: | - expected=() - if [[ "$EVENT_NAME" != workflow_dispatch ]]; then - expected+=(--expected-plan resolved/cache-plan.json) - fi python3 scripts/version_cache.py plan \ --resolved-manifest selected-versions.json \ --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ --hugo-version "$HUGO_VERSION" --go-version "$(go version)" \ - --cache-dir "$RUNNER_TEMP/history-cache" --output "$RUNNER_TEMP/cache-plan.json" "${expected[@]}" + --cache-dir "$RUNNER_TEMP/history-cache" --output "$RUNNER_TEMP/cache-plan.json" \ + --expected-plan resolved/cache-plan.json python3 scripts/build_versions.py \ --resolved-manifest selected-versions.json \ --plan "$RUNNER_TEMP/cache-plan.json" --cache-dir "$RUNNER_TEMP/history-cache" \ --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ --workers 2 --output "$RUNNER_TEMP/version-artifacts" + - name: Build and validate manual candidate versions + if: github.event_name == 'workflow_dispatch' + env: + OINK_PYTHON: python3 + SITE_ORIGIN: ${{ needs.prepare.outputs.site_origin }} + HISTORICAL_ORIGIN: ${{ needs.prepare.outputs.historical_origin }} + run: | + # Candidate branches may predate the cache and scheduling helpers. + jq -er '.include[] | [.id, .sha] | @tsv' selected-versions.json > selected-versions.tsv + cache_root="$HUGO_CACHEDIR" + while IFS=$'\t' read -r version sha; do + [[ "$version" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || exit 1 + [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || exit 1 + git cat-file -e "$sha^{commit}" || git fetch --no-tags origin "$sha" + export HUGO_CACHEDIR="$cache_root/$version" + python3 scripts/versioning.py build \ + --version "$version" --sha "$sha" \ + --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ + --output "$RUNNER_TEMP/version-artifacts/$version" + python3 scripts/versioning.py validate \ + --version "$version" --sha "$sha" \ + --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ + --artifact "$RUNNER_TEMP/version-artifacts/$version" + done < selected-versions.tsv - name: Upload selected version artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -362,6 +383,7 @@ jobs: path: version-artifacts - name: Assemble publishable site env: + EVENT_NAME: ${{ github.event_name }} PREFIX: ${{ needs.prepare.outputs.artifact_prefix }} SITE_ORIGIN: ${{ needs.prepare.outputs.site_origin }} HISTORICAL_ORIGIN: ${{ needs.prepare.outputs.historical_origin }} @@ -369,11 +391,14 @@ jobs: run: | lscpu | sed -n '/^CPU(s):/p; /^Model name:/p' extra=() + if [[ "$EVENT_NAME" != workflow_dispatch ]]; then + extra+=(--workers 2) + fi if [[ "$PREFIX" == staging ]]; then extra+=(--asf-profile oink --asf-whoami asf-staging-oink) fi python3 scripts/versioning.py aggregate \ - --artifacts version-artifacts --workers 2 \ + --artifacts version-artifacts \ --resolved-manifest resolved/resolved-versions.json \ --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ --select "$SELECTION" \ @@ -415,7 +440,7 @@ jobs: if-no-files-found: error overwrite: true - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: "24" cache: npm @@ -486,7 +511,7 @@ jobs: with: ref: ${{ needs.prepare.outputs.source_sha }} persist-credentials: false - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: "24" cache: npm diff --git a/contribution.md b/contribution.md index 2d7f8dbf9f..a0eac2e9ca 100644 --- a/contribution.md +++ b/contribution.md @@ -76,7 +76,8 @@ A complete historical cache hit uses one build job. Cold runs use at most three build groups, with at most two versions building concurrently within each runner. Historical artifacts are reused only when both the complete build-input fingerprint and artifact digest match. Missing or invalid cache entries rebuild automatically; -manual dispatches start without the artifact cache. Cache hits still run the full +manual dispatches start without the artifact cache and use the original build and +validate commands so older candidate branches do not need the new CI helpers. Cache hits still run the full version validator, and aggregation validates every selected version again. Assembly validates at most two versions concurrently, then writes the complete diff --git a/tests/e2e/workflow-contract.test.cjs b/tests/e2e/workflow-contract.test.cjs index 1715b1b9e5..058415dc30 100644 --- a/tests/e2e/workflow-contract.test.cjs +++ b/tests/e2e/workflow-contract.test.cjs @@ -61,7 +61,7 @@ test("trusted workflow bounds the candidate module graph without pinning its ver test("dependency artifacts keep stable names across selective reruns", () => { assert.match(workflow, /name: resolved-versions-\$\{\{ github\.run_id \}\}/); assert.match(workflow, /name: \$\{\{ needs\.prepare\.outputs\.artifact_prefix \}\}-versions-\$\{\{ matrix\.group\.id \}\}-\$\{\{ github\.run_id \}\}/); - assert.match(workflow, /--artifacts version-artifacts --workers 2/); + assert.match(workflow, /extra\+=\(--workers 2\)/); assert.match(workflow, /name: hugegraph-site-\$\{\{ needs\.prepare\.outputs\.artifact_prefix \}\}-\$\{\{ github\.run_id \}\}/); assert.doesNotMatch(workflow, /name: (?:resolved-versions|hugegraph-site-[^\n]+)-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}/); assert.equal((workflow.match(/\n\s+overwrite: true/g) ?? []).length, 3); @@ -104,7 +104,7 @@ test("required gate rejects failed, skipped and cancelled prerequisites", () => .split("\n").filter(line => line.startsWith(" ")) .map(line => line.slice(10)).join("\n"); const success = { PREPARE_RESULT: "success", BUILD_RESULT: "success" }; - const run = env => spawnSync("bash", ["-e", "-c", gate], {env: {...process.env, BASH_ENV: "", ...env}}).status; + const run = env => spawnSync("bash", ["--noprofile", "--norc", "-p", "-e", "-c", gate], {env: {...process.env, BASH_ENV: "", ...env}}).status; assert.equal(run(success), 0); for (const key of Object.keys(success)) { for (const result of ["failure", "skipped", "cancelled"]) { @@ -154,7 +154,7 @@ test("cache recording failures cannot block an already validated site", () => { for (const code of [0, 1]) { fs.writeFileSync(path.join(directory, "python3"), `#!/bin/sh\nexit ${code}\n`, {mode: 0o755}); const output = path.join(directory, `outputs-${code}`); - const result = spawnSync("bash", ["-e", "-c", script], { + const result = spawnSync("bash", ["--noprofile", "--norc", "-p", "-e", "-c", script], { cwd: directory, env: {...process.env, BASH_ENV: "", PATH: `${directory}:${process.env.PATH}`, GITHUB_OUTPUT: output, RUNNER_TEMP: directory} }); @@ -183,3 +183,73 @@ test("fingerprints use the same resolved Python patch on all version runners", ( assert.match(jobBody(job), /python-version: \$\{\{ needs.prepare.outputs.python_version \}\}/); } }); + +function stepBody(name) { + return workflow.split(` - name: ${name}\n`)[1].split(/\n - (?:name:|uses:)/)[0]; +} + +function stepScript(name) { + return stepBody(name).split(" run: |\n")[1].split("\n") + .filter(line => line.startsWith(" ")).map(line => line.slice(10)).join("\n"); +} + +test("manual candidates build and validate every immutable selected version using the original CLI", () => { + const { spawnSync } = require("node:child_process"); + const os = require("node:os"); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "manual-build-")); + const name = "Build and validate manual candidate versions"; + assert.match(stepBody(name), /if: github.event_name == 'workflow_dispatch'/); + assert.match(stepBody("Build and validate selected versions"), /if: github.event_name != 'workflow_dispatch'/); + assert.match(stepBody("Build and validate selected versions"), /--expected-plan resolved\/cache-plan.json/); + assert.doesNotMatch(stepScript(name), /version_cache|build_versions|--workers/); + const versions = [{id: "latest", sha: "a".repeat(40)}, {id: "1.7", sha: "b".repeat(40)}]; + try { + fs.writeFileSync(path.join(directory, "selected-versions.json"), JSON.stringify({include: versions})); + fs.writeFileSync(path.join(directory, "git"), '#!/bin/sh\nprintf "%s\\n" "$*" >> "$RUNNER_TEMP/git.log"\n[ "$1" != cat-file ]\n', {mode: 0o755}); + fs.writeFileSync(path.join(directory, "python3"), '#!/bin/sh\nprintf "%s\\t" "$HUGO_CACHEDIR" "$@" >> "$RUNNER_TEMP/python.log"\nprintf "\\n" >> "$RUNNER_TEMP/python.log"\n', {mode: 0o755}); + const env = {...process.env, BASH_ENV: "", PATH: `${directory}:${process.env.PATH}`, + RUNNER_TEMP: directory, HUGO_CACHEDIR: `${directory}/hugo`, + SITE_ORIGIN: "https://staging.example/", HISTORICAL_ORIGIN: "https://history.example/"}; + const run = () => spawnSync("bash", ["--noprofile", "--norc", "-p", "-e", "-c", stepScript(name)], {cwd: directory, env}); + const result = run(); + assert.equal(result.status, 0, result.stderr.toString()); + const calls = fs.readFileSync(path.join(directory, "python.log"), "utf8").trim().split("\n").map(line => line.trim().split("\t")); + assert.deepEqual(calls, versions.flatMap(({id, sha}) => ["build", "validate"].map(operation => [ + `${directory}/hugo/${id}`, "scripts/versioning.py", operation, "--version", id, "--sha", sha, + "--site-origin", env.SITE_ORIGIN, "--historical-origin", env.HISTORICAL_ORIGIN, + operation === "build" ? "--output" : "--artifact", `${directory}/version-artifacts/${id}` + ]))); + const gitCalls = fs.readFileSync(path.join(directory, "git.log"), "utf8"); + for (const {sha} of versions) assert.ok(gitCalls.includes(`fetch --no-tags origin ${sha}`)); + fs.writeFileSync(path.join(directory, "selected-versions.json"), JSON.stringify({include: [{id: "latest", sha: "HEAD"}]})); + assert.notEqual(run().status, 0); + assert.equal(fs.readFileSync(path.join(directory, "python.log"), "utf8").trim().split("\n").length, 4); + } finally { + fs.rmSync(directory, {recursive: true, force: true}); + } +}); + +test("manual aggregation retains staging flags without requiring new candidate CLI options", () => { + const { spawnSync } = require("node:child_process"); + const os = require("node:os"); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "manual-aggregate-")); + const script = stepScript("Assemble publishable site"); + assert.doesNotMatch(script, /version_cache|build_versions/); + try { + fs.writeFileSync(path.join(directory, "python3"), '#!/bin/sh\nprintf "%s\\n" "$@"\n', {mode: 0o755}); + fs.writeFileSync(path.join(directory, "lscpu"), '#!/bin/sh\nexit 0\n', {mode: 0o755}); + for (const event of ["workflow_dispatch", "pull_request"]) { + const result = spawnSync("bash", ["--noprofile", "--norc", "-p", "-e", "-c", script], {cwd: directory, + env: {...process.env, BASH_ENV: "", PATH: `${directory}:${process.env.PATH}`, EVENT_NAME: event, + PREFIX: "staging", RUNNER_TEMP: directory, SITE_ORIGIN: "https://staging.example/", + HISTORICAL_ORIGIN: "https://history.example/", SELECTION: "latest,1.7"}}); + assert.equal(result.status, 0, result.stderr.toString()); + const args = result.stdout.toString().trim().split("\n"); + assert.equal(args.includes("--workers"), event !== "workflow_dispatch"); + assert.deepEqual(args.slice(-4), ["--asf-profile", "oink", "--asf-whoami", "asf-staging-oink"]); + assert.deepEqual(args.slice(0, 4), ["scripts/versioning.py", "aggregate", "--artifacts", "version-artifacts"]); + } + } finally { + fs.rmSync(directory, {recursive: true, force: true}); + } +}); From f6569a2444fb3e1bc3f659ac92c35d30983cc757 Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 2 Oct 2026 20:58:20 +0800 Subject: [PATCH 08/13] fix: show AI consent only after activation - open first-use consent only from explicit Ask AI actions - remove the footer withdrawal control and unused labels - keep the search AI action visible on desktop and mobile - preserve consent and disabled-state browser coverage --- assets/js/kapa-adapter.js | 25 +---- assets/scss/_styles_project.scss | 19 +++- i18n/en.yaml | 2 - i18n/zh-CN.yaml | 2 - layouts/_partials/footer.html | 3 - layouts/_partials/hooks/body-end.html | 1 - scripts/kapa-rollout.md | 8 +- tests/e2e/ai.spec.js | 144 ++++++++++++++++++-------- tests/ui-ai/kapa-adapter.test.cjs | 82 ++++----------- 9 files changed, 145 insertions(+), 141 deletions(-) diff --git a/assets/js/kapa-adapter.js b/assets/js/kapa-adapter.js index 9c903e644a..993b1c1c94 100644 --- a/assets/js/kapa-adapter.js +++ b/assets/js/kapa-adapter.js @@ -141,8 +141,6 @@ // Storage may be disabled; consent still works for this page only. } var consented = choice === 'granted'; - var grantPersisted = consented; - var dismissed = choice === 'dismissed'; windowObject.addEventListener('storage', function (event) { if (event.key !== consentKey && event.key !== null) return; if ((consented && event.newValue !== 'granted') || @@ -155,14 +153,14 @@ var pending = null; var consent = documentObject.querySelector('[data-hg-ai-consent]'); var launcher = documentObject.querySelector('.hg-ask-ai-launcher'); - if (consent) consent.hidden = consented || dismissed; + if (consent) consent.hidden = true; function renderLauncher(hidden) { if (!launcher) return; launcher.hidden = hidden; if (consented) launcher.setAttribute('aria-haspopup', 'dialog'); else launcher.removeAttribute('aria-haspopup'); } - renderLauncher(!consented && !dismissed); + renderLauncher(false); function dismissConsent() { if (consent) consent.hidden = true; @@ -175,23 +173,6 @@ var activeQueue = null; var operation = null; var status = documentObject.querySelector('[data-hg-ai-status]'); - var revoke = documentObject.querySelector('[data-hg-ai-revoke]'); - if (revoke) { - revoke.hidden = !consented; - revoke.addEventListener('click', function () { - if (!consented) return; - if (grantPersisted) { - try { - windowObject.localStorage.removeItem(consentKey); - } catch (_) { - renderState(state, config.labels.revokeError); - return; - } - } - // Navigation terminates the vendor's loaded state and pending callbacks. - windowObject.location.reload(); - }); - } function renderState(next, message) { state = next; @@ -412,11 +393,9 @@ consented = true; try { windowObject.localStorage.setItem(consentKey, 'granted'); - grantPersisted = true; } catch (_) { // Never bypass initial consent when persistence is unavailable. } - if (revoke) revoke.hidden = false; dismissConsent(); load(request.query, request.submit); }); diff --git a/assets/scss/_styles_project.scss b/assets/scss/_styles_project.scss index f22f015a68..3925c78aae 100644 --- a/assets/scss/_styles_project.scss +++ b/assets/scss/_styles_project.scss @@ -865,12 +865,23 @@ } -// The first-use notice stays beside the AI entry without blocking the page. -.hg-ai-revoke { - font-size: inherit; - &[hidden] { display: none; } +// Keep the search action visible while documentation results scroll. +.td-shell-search__group[aria-labelledby="td-shell-search-group-actions"] { + position: sticky; + bottom: 0; + z-index: 1; + background: var(--bs-body-bg); + border-top: 1px solid var(--bs-border-color); + box-shadow: 0 -.25rem .75rem rgba(0, 0, 0, .06); + + .td-shell-search__item { + color: var(--bs-primary); + background: rgba(var(--bs-primary-rgb), .08); + } } +// The first-use notice opens beside the AI entry after explicit activation. + .hg-ai-consent { position: fixed; z-index: 1040; diff --git a/i18n/en.yaml b/i18n/en.yaml index 15ce3691b8..ea907fc6ed 100644 --- a/i18n/en.yaml +++ b/i18n/en.yaml @@ -24,6 +24,4 @@ ui_ai_consent_description: "Agree to send your questions to the third-party Kapa ui_ai_consent_continue: "Agree and open Ask AI" ui_ai_consent_cancel: "Not now" ui_ai_privacy: "Privacy policy" -ui_ai_revoke: "Withdraw AI consent" -ui_ai_revoke_error: "AI consent could not be removed and remains active. Clear this site's browser data to withdraw consent." ui_ai_community: "Ask the community on GitHub" diff --git a/i18n/zh-CN.yaml b/i18n/zh-CN.yaml index a194fb3f6b..3a043bf19c 100644 --- a/i18n/zh-CN.yaml +++ b/i18n/zh-CN.yaml @@ -25,6 +25,4 @@ ui_ai_consent_description: "同意后将把你的问题发送给第三方 Kapa ui_ai_consent_continue: "同意并打开 Ask AI" ui_ai_consent_cancel: "暂不使用" ui_ai_privacy: "隐私政策" -ui_ai_revoke: "撤回 AI 授权" -ui_ai_revoke_error: "未能撤回 AI 授权，授权仍有效。请清除此网站的浏览器数据以撤回授权。" ui_ai_community: "前往 GitHub 向社区提问" diff --git a/layouts/_partials/footer.html b/layouts/_partials/footer.html index 83eff243f7..94fe3c0e33 100644 --- a/layouts/_partials/footer.html +++ b/layouts/_partials/footer.html @@ -34,9 +34,6 @@

Copyright © {{ now.Year }} The Apache Software Foundation, Licensed under the Apache License Version 2.0

Apache, the names of Apache projects, and the feather logo are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries.

- {{- if (partial "ai/config.html" .).enabled }} - - {{- end }}
diff --git a/layouts/_partials/hooks/body-end.html b/layouts/_partials/hooks/body-end.html index d30e99e603..ff9b307d64 100644 --- a/layouts/_partials/hooks/body-end.html +++ b/layouts/_partials/hooks/body-end.html @@ -25,7 +25,6 @@ "description" (T "ui_ask_ai_description") "latest" (T "ui_ask_ai_latest") "error" (T "ui_ai_error") - "revokeError" (T "ui_ai_revoke_error") "community" (T "ui_ai_community") -}} {{- $questions := slice -}} diff --git a/scripts/kapa-rollout.md b/scripts/kapa-rollout.md index 45ee34d7af..fe677e77ef 100644 --- a/scripts/kapa-rollout.md +++ b/scripts/kapa-rollout.md @@ -36,16 +36,16 @@ HTML 抓取分别从 `/docs/` 与 `/cn/docs/` 开始，用锚定到站点和语 记录实际抓取站点和候选 SHA；引用会指向抓取站点，staging 来源不能直接当作生产引用配置。 [ASF FAQ](https://privacy.apache.org/faq/committers.html#can-i-use-kapaai-on-our-website-answer-machine) -确认 Kapa 已签署 DPA，并要求在加载前征得用户同意。本站在首次入口展示本地非模态授权说明， +确认 Kapa 已签署 DPA，并要求在加载前征得用户同意。本站在用户首次点击 Ask AI 时展示本地非模态授权说明， 用户同意后才加载第三方脚本；`data-consent-required=false` 用于避免 widget 再次弹窗， 不表示跳过本站授权。保留禁用分析 cookie、指纹及反馈的现有配置。 -首次入口显示本地非模态说明；用户点击“同意并打开 Ask AI”后，授权保存在当前浏览器中， +访问页面时只显示 Ask AI 入口；首次点击入口才显示本地非模态说明。用户点击“同意并打开 Ask AI”后，授权保存在当前浏览器中， 并由同源中英文及历史页面复用。清除网站数据或授权版本变化后会重新询问。 即使已同意，页面加载也不会请求 Kapa；再次点击 Ask AI 时才加载服务。 “暂不使用”也会在浏览器中记住，后续页面只显示入口；主动点击入口才再次显示授权说明。 -授权后可通过页脚的“撤回 AI 授权”清除授权并重新加载页面，卸载已加载的第三方资源。 -授权说明与撤回入口在 HTML 中默认隐藏，由本地适配器初始化后显示，避免闪现及失效按钮。 +清除网站浏览器数据并重新加载页面可重置授权；同源其他已授权页面会同步重新加载，卸载第三方资源。 +授权说明与 Ask AI 入口在 HTML 中默认隐藏；本地适配器初始化后仅显示入口，避免闪现及失效按钮。 存储不可用时仅对当前页面有效。不迁移此前仅限会话的授权，以免延长用户原先同意的范围。 Widget 从 `data/ai/questions.yaml` 的 36 组中英文常见问题中随机抽取 3 条，当前页面内保持不变，回答后显示低调的 GitHub Discussions 求助链接； 问题池按 Server 16、AI 10、Toolchain 7、Computing 3 分配，每条等概率抽取，因此展示比重由各类题数决定。 diff --git a/tests/e2e/ai.spec.js b/tests/e2e/ai.spec.js index 82555b08d3..6ad2e7114f 100644 --- a/tests/e2e/ai.spec.js +++ b/tests/e2e/ai.spec.js @@ -32,8 +32,8 @@ for (const [locale, route, source, language] of [ document.documentElement.setAttribute("data-bs-theme", "dark") ); const launcher = page.locator(".hg-ask-ai-launcher"); - await expect(launcher).toBeHidden(); - await expect(page.locator("[data-hg-ai-consent]")).toBeVisible(); + await expect(launcher).toBeVisible(); + await expect(page.locator("[data-hg-ai-consent]")).toBeHidden(); await page.locator("[data-td-shell-search-open]").first().click(); const input = page.locator(".td-shell-search__input"); @@ -93,6 +93,8 @@ test("AI consent cancel and Escape keep native search local", async ({ page }) = await page.goto(AI_ORIGIN + "/docs/"); const launcher = page.locator(".hg-ask-ai-launcher"); const consent = page.locator("[data-hg-ai-consent]"); + await expect(consent).toBeHidden(); + await launcher.click(); await expect(consent).toBeVisible(); await consent.locator("[data-hg-ai-cancel]").click(); await expect(consent).toBeHidden(); @@ -117,6 +119,7 @@ test("AI 500 remains non-blocking and retry issues one fresh request", async ({ }); await page.goto(AI_ORIGIN + "/docs/"); const launcher = page.locator(".hg-ask-ai-launcher"); + await launcher.click(); await expect(page.locator("[data-hg-ai-consent]")).toBeVisible(); expect(attempts).toBe(0); await page.locator("[data-hg-ai-consent] [data-hg-ai-continue]").click(); @@ -149,6 +152,7 @@ test("AI pending timeout discards stale state and retry waits for a fresh bundle }); await page.goto(AI_ORIGIN + "/docs/"); const launcher = page.locator(".hg-ask-ai-launcher"); + await launcher.click(); await expect(page.locator("[data-hg-ai-consent]")).toBeVisible(); expect(attempts).toBe(0); await page.locator("[data-hg-ai-consent] [data-hg-ai-continue]").click(); @@ -185,6 +189,8 @@ test("native AI row supports keyboard selection, IME, and consent focus handoff" await route.fulfill({ status: 200, contentType: "text/javascript", body: mockBundle }); }); await page.goto(AI_ORIGIN + "/docs/"); + await page.locator(".hg-ask-ai-launcher").click(); + await expect(page.locator("[data-hg-ai-consent]")).toBeVisible(); await page.locator("[data-hg-ai-cancel]").click(); await page.locator("[data-td-shell-search-open]").first().click(); const input = page.locator(".td-shell-search__input"); @@ -247,8 +253,9 @@ test("persistent consent survives language navigation, reload and a fresh tab wi await route.fulfill({ status: 200, contentType: "text/javascript", body: mockBundle }); }); await page.goto(AI_ORIGIN + "/docs/"); - await expect(page.locator("[data-hg-ai-revoke]")).toBeHidden(); - await expect(page.locator(".hg-ask-ai-launcher")).toBeHidden(); + await expect(page.locator("[data-hg-ai-revoke]")).toHaveCount(0); + await expect(page.locator(".hg-ask-ai-launcher")).toBeVisible(); + await page.locator(".hg-ask-ai-launcher").click(); await page.locator("[data-hg-ai-continue]").click(); await expect.poll(() => requests.length).toBe(1); @@ -291,22 +298,28 @@ test("blocked local storage requires fresh consent after navigation", async ({ p await route.fulfill({ status: 200, contentType: "text/javascript", body: mockBundle }); }); await page.goto(AI_ORIGIN + "/docs/"); + await expect(page.locator("[data-hg-ai-consent]")).toBeHidden(); + await page.locator(".hg-ask-ai-launcher").click(); await expect(page.locator("[data-hg-ai-consent]")).toBeVisible(); await page.locator("[data-hg-ai-cancel]").click(); await expect(page.locator("[data-hg-ai-consent]")).toBeHidden(); await page.reload(); + await expect(page.locator("[data-hg-ai-consent]")).toBeHidden(); + await page.locator(".hg-ask-ai-launcher").click(); await expect(page.locator("[data-hg-ai-consent]")).toBeVisible(); expect(requests).toHaveLength(0); await page.locator("[data-hg-ai-continue]").click(); await expect.poll(() => requests.length).toBe(1); await page.goto(AI_ORIGIN + "/cn/docs/"); await expect(page.locator("script[data-hg-kapa-widget]")).toHaveCount(0); + await expect(page.locator("[data-hg-ai-consent]")).toBeHidden(); + await page.locator(".hg-ask-ai-launcher").click(); await expect(page.locator("[data-hg-ai-consent]")).toBeVisible(); expect(requests).toHaveLength(1); }); -test("inline disclosure is nonmodal and first agreement opens AI with one click", async ({ page }) => { +test("explicit Ask AI activation shows a nonmodal disclosure before agreement", async ({ page }) => { const requests = []; await page.route("https://widget.kapa.ai/kapa-widget.bundle.js*", async route => { requests.push(route.request().url()); @@ -314,6 +327,8 @@ test("inline disclosure is nonmodal and first agreement opens AI with one click" }); await page.goto(AI_ORIGIN + "/docs/"); const notice = page.locator("[data-hg-ai-consent]"); + await expect(notice).toBeHidden(); + await page.locator(".hg-ask-ai-launcher").click(); await expect(notice).toBeVisible(); expect(await notice.evaluate(node => node.tagName)).toBe("SECTION"); await expect(notice).not.toHaveAttribute("aria-modal", "true"); @@ -342,12 +357,13 @@ for (const dismiss of ["button", "escape"]) { await route.fulfill({ status: 200, contentType: "text/javascript", body: mockBundle }); }); await page.goto(AI_ORIGIN + "/docs/"); + await page.locator(".hg-ask-ai-launcher").click(); if (dismiss === "button") await page.locator("[data-hg-ai-cancel]").click(); else await page.locator("[data-hg-ai-continue]").press("Escape"); await page.goto(AI_ORIGIN + "/cn/docs/"); await expect(page.locator("[data-hg-ai-consent]")).toBeHidden(); await expect(page.locator(".hg-ask-ai-launcher")).toBeVisible(); - await expect(page.locator("[data-hg-ai-revoke]")).toBeHidden(); + await expect(page.locator("[data-hg-ai-revoke]")).toHaveCount(0); await page.reload(); await expect(page.locator("[data-hg-ai-consent]")).toBeHidden(); const fresh = await page.context().newPage(); @@ -367,7 +383,7 @@ for (const dismiss of ["button", "escape"]) { } for (const loaded of [false, true]) { - test(`revocation resets grant and vendor state (${loaded ? "after" : "before"} loading)`, async ({ page }) => { + test(`clearing stored permission and reloading resets vendor state (${loaded ? "after" : "before"} loading)`, async ({ page }) => { const requests = []; await page.route("https://widget.kapa.ai/kapa-widget.bundle.js*", async route => { requests.push(route.request().url()); @@ -380,17 +396,21 @@ for (const loaded of [false, true]) { localStorage.setItem('unrelated', 'preserve'); }); await page.reload(); - await expect(page.locator("[data-hg-ai-revoke]")).toBeVisible(); + await expect(page.locator("[data-hg-ai-revoke]")).toHaveCount(0); expect(requests).toHaveLength(0); if (loaded) { await page.locator(".hg-ask-ai-launcher").click(); await expect.poll(() => requests.length).toBe(1); await expect(page.locator(".hg-ask-ai-launcher")).toHaveAttribute("data-hg-ai-state", "ready"); } - await page.locator("[data-hg-ai-revoke]").click(); - await expect(page.locator("[data-hg-ai-consent]")).toBeVisible(); - await expect(page.locator(".hg-ask-ai-launcher")).toBeHidden(); - await expect(page.locator("[data-hg-ai-revoke]")).toBeHidden(); + await page.evaluate(() => { + const config = JSON.parse(document.querySelector("#hg-ai-config").textContent); + localStorage.removeItem('hg-ai-consent:v2:' + config.websiteId); + }); + await page.reload(); + await expect(page.locator("[data-hg-ai-consent]")).toBeHidden(); + await expect(page.locator(".hg-ask-ai-launcher")).toBeVisible(); + await expect(page.locator("[data-hg-ai-revoke]")).toHaveCount(0); await expect(page.locator("script[data-hg-kapa-widget]")).toHaveCount(0); expect(await page.evaluate(() => window.__kapaCalls)).toBeUndefined(); expect(await page.evaluate(() => localStorage.getItem('unrelated'))).toBe('preserve'); @@ -408,38 +428,15 @@ test("blocked adapter leaves consent and launcher hidden and native search usabl await page.goto(AI_ORIGIN + "/docs/"); await expect(page.locator("[data-hg-ai-consent]")).toBeHidden(); await expect(page.locator(".hg-ask-ai-launcher")).toBeHidden(); - await expect(page.locator("[data-hg-ai-revoke]")).toBeHidden(); + await expect(page.locator("[data-hg-ai-revoke]")).toHaveCount(0); await page.locator("[data-td-shell-search-open]").first().click(); await expect(page.locator(".td-shell-search__input")).toBeFocused(); expect(requests).toHaveLength(0); }); -test("failed persistent grant removal announces error without pretending to revoke", async ({ page }) => { - await page.goto(AI_ORIGIN + "/docs/"); - await page.evaluate(() => { - const config = JSON.parse(document.querySelector("#hg-ai-config").textContent); - localStorage.setItem('hg-ai-consent:v2:' + config.websiteId, 'granted'); - }); - await page.reload(); - await page.evaluate(() => { - Storage.prototype.removeItem = function () { throw new DOMException('blocked', 'SecurityError'); }; - }); - await page.locator("[data-hg-ai-revoke]").click(); - const revokeError = await page.locator("#hg-ai-config").evaluate(node => - JSON.parse(node.textContent).labels.revokeError); - await expect(page.locator("[data-hg-ai-status]")).toHaveText(revokeError); - await expect(page.locator("[data-hg-ai-revoke]")).toBeVisible(); - await expect(page.locator("[data-hg-ai-consent]")).toBeHidden(); - expect(await page.evaluate(() => { - const config = JSON.parse(document.querySelector("#hg-ai-config").textContent); - return localStorage.getItem('hg-ai-consent:v2:' + config.websiteId); - })).toBe('granted'); -}); - - for (const loaded of [false, true]) { - test(`revocation synchronizes sibling tabs (${loaded ? "loaded vendor" : "before loading"})`, async ({ page }) => { + test(`clearing stored permission synchronizes sibling tabs (${loaded ? "loaded vendor" : "before loading"})`, async ({ page }) => { const requests = []; const fulfillBundle = async route => { requests.push(route.request().url()); @@ -462,8 +459,14 @@ for (const loaded of [false, true]) { await expect.poll(() => requests.length).toBe(2); await expect(sibling.locator(".hg-ask-ai-launcher")).toHaveAttribute("data-hg-ai-state", "ready"); } - await page.locator("[data-hg-ai-revoke]").click(); - await expect(page.locator("[data-hg-ai-consent]")).toBeVisible(); + const reloaded = sibling.waitForEvent("load"); + await page.evaluate(() => { + const config = JSON.parse(document.querySelector("#hg-ai-config").textContent); + localStorage.removeItem('hg-ai-consent:v2:' + config.websiteId); + }); + await reloaded; + await expect(sibling.locator("[data-hg-ai-consent]")).toBeHidden(); + await sibling.locator(".hg-ask-ai-launcher").click(); await expect(sibling.locator("[data-hg-ai-consent]")).toBeVisible(); await expect(sibling.locator("script[data-hg-kapa-widget]")).toHaveCount(0); expect(await sibling.evaluate(() => window.__kapaCalls)).toBeUndefined(); @@ -484,19 +487,22 @@ test("grant synchronizes stale disclosure tabs without autoloading AI", async ({ await page.goto(AI_ORIGIN + "/docs/"); const sibling = await page.context().newPage(); await sibling.goto(AI_ORIGIN + "/cn/docs/"); + await expect(sibling.locator("[data-hg-ai-consent]")).toBeHidden(); + await sibling.locator(".hg-ask-ai-launcher").click(); await expect(sibling.locator("[data-hg-ai-consent]")).toBeVisible(); + await page.locator(".hg-ask-ai-launcher").click(); await page.locator("[data-hg-ai-continue]").click(); await expect.poll(() => requests.length).toBe(1); await expect(sibling.locator("[data-hg-ai-consent]")).toBeHidden(); await expect(sibling.locator(".hg-ask-ai-launcher")).toBeVisible(); - await expect(sibling.locator("[data-hg-ai-revoke]")).toBeVisible(); + await expect(sibling.locator("[data-hg-ai-revoke]")).toHaveCount(0); await expect(sibling.locator("script[data-hg-kapa-widget]")).toHaveCount(0); expect(requests).toHaveLength(1); await sibling.close(); }); -test("blocked storage page-only consent can be revoked after loading AI", async ({ page }) => { +test("blocked storage page-only consent expires after reload", async ({ page }) => { const requests = []; await page.addInitScript(() => { Object.defineProperty(window, "localStorage", { @@ -508,13 +514,65 @@ test("blocked storage page-only consent can be revoked after loading AI", async await route.fulfill({ status: 200, contentType: "text/javascript", body: mockBundle }); }); await page.goto(AI_ORIGIN + "/docs/"); + await page.locator(".hg-ask-ai-launcher").click(); await page.locator("[data-hg-ai-continue]").click(); await expect.poll(() => requests.length).toBe(1); await expect(page.locator(".hg-ask-ai-launcher")).toHaveAttribute("data-hg-ai-state", "ready"); - await page.locator("[data-hg-ai-revoke]").click(); + await page.reload(); + await expect(page.locator("[data-hg-ai-consent]")).toBeHidden(); + await page.locator(".hg-ask-ai-launcher").click(); await expect(page.locator("[data-hg-ai-consent]")).toBeVisible(); - await expect(page.locator("[data-hg-ai-revoke]")).toBeHidden(); + await expect(page.locator("[data-hg-ai-revoke]")).toHaveCount(0); await expect(page.locator("script[data-hg-kapa-widget]")).toHaveCount(0); expect(await page.evaluate(() => window.__kapaCalls)).toBeUndefined(); expect(requests).toHaveLength(1); }); + +for (const route of ["/", "/cn/", "/docs/", "/cn/docs/"]) { + test(`visiting ${route} keeps consent hidden and AI offline until activation`, async ({ page }) => { + const requests = []; + await page.route(/https:\/\/[^/]*kapa\.ai\//, route => { + requests.push(route.request().url()); + return route.abort(); + }); + await page.goto(AI_ORIGIN + route); + const notice = page.locator("[data-hg-ai-consent]"); + await expect(page.locator(".hg-ask-ai-launcher")).toBeVisible(); + await expect(notice).toBeHidden(); + await expect(page.locator("[data-hg-ai-revoke]")).toHaveCount(0); + await expect(page.locator("script[data-hg-kapa-widget]")).toHaveCount(0); + expect(requests).toEqual([]); + await page.locator(".hg-ask-ai-launcher").click(); + await expect(notice).toBeVisible(); + await expect(page.locator("[data-hg-ai-continue]")).toBeFocused(); + expect(requests).toEqual([]); + }); +} + +for (const viewport of [{ width: 1440, height: 900 }, { width: 390, height: 844 }]) { + test(`Ask AI stays visible below broad search results at ${viewport.width}px`, async ({ page }) => { + await page.setViewportSize(viewport); + const requests = []; + await page.route(/https:\/\/[^/]*kapa\.ai\//, route => { + requests.push(route.request().url()); + return route.abort(); + }); + await page.goto(AI_ORIGIN + "/docs/"); + await page.locator("[data-td-shell-search-open]").first().click(); + await page.locator(".td-shell-search__input").fill("hugegraph"); + await expect.poll(() => page.getByRole("option").count()).toBeGreaterThan(5); + const action = page.getByRole("option").filter({ hasText: /Ask AI|询问 AI|问 AI/ }); + await expect(action).toBeVisible(); + // A click can scroll its target into view; measure before clicking so this + // catches an action hidden below the initial search results viewport. + await expect.poll(() => action.evaluate(node => { + const box = node.getBoundingClientRect(); + const dialog = node.closest('[role="dialog"], dialog').getBoundingClientRect(); + return box.top >= Math.max(0, dialog.top) && box.bottom <= Math.min(innerHeight, dialog.bottom) && + box.left >= Math.max(0, dialog.left) && box.right <= Math.min(innerWidth, dialog.right); + })).toBe(true); + await action.click(); + await expect(page.locator("[data-hg-ai-consent]")).toBeVisible(); + expect(requests).toEqual([]); + }); +} diff --git a/tests/ui-ai/kapa-adapter.test.cjs b/tests/ui-ai/kapa-adapter.test.cjs index 17710744d4..08262499f4 100644 --- a/tests/ui-ai/kapa-adapter.test.cjs +++ b/tests/ui-ai/kapa-adapter.test.cjs @@ -25,7 +25,6 @@ function harness(storage = new Map()) { const consentListeners = new Map(); const continueButton = { focus() { this.focused = true; }, addEventListener(name, callback) { consentListeners.set(`continue:${name}`, callback); } }; const cancelButton = { addEventListener(name, callback) { consentListeners.set(`cancel:${name}`, callback); } }; - const revoke = { hidden: true, addEventListener(name, cb) { consentListeners.set(`revoke:${name}`, cb); } }; const consent = { hidden: false, addEventListener(name, callback) { consentListeners.set(`dialog:${name}`, callback); }, @@ -40,7 +39,6 @@ function harness(storage = new Map()) { querySelector(selector) { if (selector === '[data-hg-ai-status]') return status; if (selector === '[data-hg-ai-consent]') return consent; - if (selector === '[data-hg-ai-revoke]') return revoke; if (selector === '.hg-ask-ai-launcher') return trigger; if (selector === 'script[data-hg-kapa-widget]') { return scripts.find((script) => !script.removed) || null; @@ -104,13 +102,11 @@ function harness(storage = new Map()) { themeColor: '#123456', exampleQuestions: ['How do I start HugeGraph?', 'How do I import data?'], communityURL: 'https://github.com/apache/hugegraph/discussions', - labels: { error: 'unavailable', revokeError: 'Unable to revoke. AI consent remains enabled.', community: 'Ask the community' }, + labels: { error: 'unavailable', community: 'Ask the community' }, }; return { calls, status, - revoke, - revokeConsent() { consentListeners.get('revoke:click')(); }, storageEvent(event) { windowListeners.get('storage')(event); }, consent, continueButton, @@ -575,12 +571,14 @@ test('a page retains its sampled examples when the widget load is retried', () = }); -test('inline consent opens AI in one click and restores the launcher on widget close', () => { +test('explicit activation opens consent and agreement restores the launcher on widget close', () => { const h = harness(); - adapter.createController(h.windowObject, h.documentObject, h.config); - assert.equal(h.consent.hidden, false); - assert.equal(h.trigger.hidden, true); + const controller = adapter.createController(h.windowObject, h.documentObject, h.config); + assert.equal(h.consent.hidden, true); + assert.equal(h.trigger.hidden, false); assert.equal(h.scripts.length, 0); + controller.activate('', false, h.trigger); + assert.equal(h.consent.hidden, false); h.continueConsent(); assert.equal(h.consent.hidden, true); assert.equal(h.trigger.hidden, false); @@ -595,9 +593,10 @@ test('inline consent opens AI in one click and restores the launcher on widget c assert.equal(h.scripts.length, 1); }); -test('initial inline dismissal remembers no-grant choice and pending activation focuses agreement', () => { +test('explicit consent dismissal remembers no-grant choice and pending activation focuses agreement', () => { const h = harness(); const controller = adapter.createController(h.windowObject, h.documentObject, h.config); + controller.activate('', false, h.trigger); h.cancelConsent(); assert.equal(h.trigger.hidden, false); assert.equal(h.trigger.focused, true); @@ -619,7 +618,7 @@ test('initial inline dismissal remembers no-grant choice and pending activation test('remembered dismissal stays local and explicit reopening still requires consent', () => { for (const dismiss of ['cancelConsent', 'escapeConsent']) { const first = harness(); - adapter.createController(first.windowObject, first.documentObject, first.config); + adapter.createController(first.windowObject, first.documentObject, first.config).activate('', false, first.trigger); first[dismiss](); const next = harness(first.storage); next.config.locale = 'zh'; @@ -627,7 +626,6 @@ test('remembered dismissal stays local and explicit reopening still requires con assert.equal(next.consent.hidden, true); assert.equal(next.trigger.hidden, false); assert.equal(next.trigger.attrs['aria-haspopup'], undefined); - assert.equal(next.revoke.hidden, true); assert.equal(next.scripts.length, 0); controller.activate('private query', true, next.trigger); assert.equal(controller.getState(), 'consent'); @@ -637,51 +635,18 @@ test('remembered dismissal stays local and explicit reopening still requires con } }); -test('revocation clears only the grant and reloads whether vendor is loaded or not', () => { - for (const loaded of [false, true]) { - const storage = new Map([['hg-ai-consent:v2:website', 'granted'], ['unrelated', 'preserve']]); - const h = harness(storage); - const controller = adapter.createController(h.windowObject, h.documentObject, h.config); - assert.equal(h.revoke.hidden, false); - if (loaded) { - controller.activate('', false, h.trigger); - h.scripts[0].fire('load'); - h.fireRender(); - } - h.revokeConsent(); - assert.equal(h.windowObject.reloaded, true); - assert.deepEqual(Array.from(storage), [['unrelated', 'preserve']]); - const fresh = harness(storage); - adapter.createController(fresh.windowObject, fresh.documentObject, fresh.config); - assert.equal(fresh.consent.hidden, false); - assert.equal(fresh.trigger.hidden, true); - assert.equal(fresh.revoke.hidden, true); - assert.equal(fresh.scripts.length, 0); - } -}); - -test('failed grant removal reports an error and keeps the active permission', () => { - const h = harness(new Map([['hg-ai-consent:v2:website', 'granted']])); - h.windowObject.localStorage.removeItem = () => { throw new Error('blocked'); }; - const controller = adapter.createController(h.windowObject, h.documentObject, h.config); - h.revokeConsent(); - assert.equal(h.windowObject.reloaded, undefined); - assert.equal(h.storage.get('hg-ai-consent:v2:website'), 'granted'); - assert.equal(h.revoke.hidden, false); - assert.equal(h.status.textContent, 'Unable to revoke. AI consent remains enabled.'); - controller.activate('', false, h.trigger); - assert.equal(controller.getState(), 'loading'); -}); - -test('blocked storage keeps dismissal page-only and exposes fresh disclosure next visit', () => { +test('blocked storage keeps dismissal page-only and requires explicit activation next visit', () => { const h = harness(); h.windowObject.localStorage.setItem = () => { throw new Error('blocked'); }; - adapter.createController(h.windowObject, h.documentObject, h.config); + adapter.createController(h.windowObject, h.documentObject, h.config).activate('', false, h.trigger); h.cancelConsent(); assert.equal(h.consent.hidden, true); assert.equal(h.storage.size, 0); const fresh = harness(h.storage); - adapter.createController(fresh.windowObject, fresh.documentObject, fresh.config); + const controller = adapter.createController(fresh.windowObject, fresh.documentObject, fresh.config); + assert.equal(fresh.consent.hidden, true); + assert.equal(fresh.scripts.length, 0); + controller.activate('', false, fresh.trigger); assert.equal(fresh.consent.hidden, false); assert.equal(fresh.scripts.length, 0); }); @@ -722,7 +687,7 @@ test('incoming permission synchronizes stale preconsent tabs without loading the test('load and vendor open failures return keyboard focus from hidden agreement to launcher', () => { for (const failure of ['bundle', 'open']) { const h = harness(); - adapter.createController(h.windowObject, h.documentObject, h.config); + adapter.createController(h.windowObject, h.documentObject, h.config).activate('', false, h.trigger); h.continueConsent(); assert.equal(h.consent.hidden, true); if (failure === 'bundle') h.scripts[0].fire('error'); @@ -741,22 +706,21 @@ test('load and vendor open failures return keyboard focus from hidden agreement }); -test('page-only permission can be revoked when storage is wholly blocked', () => { +test('page-only permission expires on navigation when storage is wholly blocked', () => { const h = harness(); Object.defineProperty(h.windowObject, 'localStorage', { get() { throw new Error('blocked'); }, }); - adapter.createController(h.windowObject, h.documentObject, h.config); + adapter.createController(h.windowObject, h.documentObject, h.config).activate('', false, h.trigger); h.continueConsent(); h.scripts[0].fire('load'); h.fireRender(); - assert.equal(h.revoke.hidden, false); - h.revokeConsent(); - assert.equal(h.windowObject.reloaded, true); - assert.equal(h.status.textContent, ''); assert.equal(h.storage.size, 0); const fresh = harness(h.storage); - adapter.createController(fresh.windowObject, fresh.documentObject, fresh.config); + const controller = adapter.createController(fresh.windowObject, fresh.documentObject, fresh.config); + assert.equal(fresh.consent.hidden, true); + assert.equal(fresh.scripts.length, 0); + controller.activate('', false, fresh.trigger); assert.equal(fresh.consent.hidden, false); assert.equal(fresh.scripts.length, 0); }); From d41402cd6661c50b0619f835cabbaebc3971792d Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 2 Oct 2026 21:01:15 +0800 Subject: [PATCH 09/13] fix: complete search and consent regressions - scroll document results independently of the AI action - verify keyboard selections remain visible on both viewports - update platform consent checks for explicit activation --- assets/scss/_styles_project.scss | 18 ++++++++++++++---- tests/e2e/ai.spec.js | 11 +++++++++++ tests/e2e/platform.spec.js | 4 +++- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/assets/scss/_styles_project.scss b/assets/scss/_styles_project.scss index 3925c78aae..b0286e7184 100644 --- a/assets/scss/_styles_project.scss +++ b/assets/scss/_styles_project.scss @@ -865,11 +865,21 @@ } -// Keep the search action visible while documentation results scroll. +// Let documentation results scroll without covering them with the AI action. +.td-shell-search__list:has(> [aria-labelledby="td-shell-search-group-actions"]) { + display: flex; + flex-direction: column; + overflow: hidden; + + > .td-shell-search__group { + min-height: 0; + overflow-y: auto; + overscroll-behavior: contain; + } +} + .td-shell-search__group[aria-labelledby="td-shell-search-group-actions"] { - position: sticky; - bottom: 0; - z-index: 1; + flex-shrink: 0; background: var(--bs-body-bg); border-top: 1px solid var(--bs-border-color); box-shadow: 0 -.25rem .75rem rgba(0, 0, 0, .06); diff --git a/tests/e2e/ai.spec.js b/tests/e2e/ai.spec.js index 6ad2e7114f..1faf108986 100644 --- a/tests/e2e/ai.spec.js +++ b/tests/e2e/ai.spec.js @@ -571,6 +571,17 @@ for (const viewport of [{ width: 1440, height: 900 }, { width: 390, height: 844 return box.top >= Math.max(0, dialog.top) && box.bottom <= Math.min(innerHeight, dialog.bottom) && box.left >= Math.max(0, dialog.left) && box.right <= Math.min(innerWidth, dialog.right); })).toBe(true); + // Keyboard scrolling must keep the last document above the action too. + const input = page.locator(".td-shell-search__input"); + await input.press("Control+End"); + await input.press("ArrowUp"); + const selected = page.locator('[role="option"][aria-selected="true"]'); + await expect(selected).not.toContainText("Ask AI:"); + expect(await selected.evaluate(node => { + const box = node.getBoundingClientRect(); + const actionGroup = document.querySelector('[aria-labelledby="td-shell-search-group-actions"]'); + return box.bottom <= actionGroup.getBoundingClientRect().top; + })).toBe(true); await action.click(); await expect(page.locator("[data-hg-ai-consent]")).toBeVisible(); expect(requests).toEqual([]); diff --git a/tests/e2e/platform.spec.js b/tests/e2e/platform.spec.js index 3af698353b..a1e36bcf03 100644 --- a/tests/e2e/platform.spec.js +++ b/tests/e2e/platform.spec.js @@ -214,7 +214,9 @@ test("enabled AI makes no third-party request before consent", async ({ page }) expect(kapaRequests).toEqual([]); await page.locator(".td-shell-search__input").press("Escape"); const launcher = page.locator("[data-hg-ask-ai]").first(); - await expect(launcher).toBeHidden(); + await expect(launcher).toBeVisible(); + await expect(page.locator("[data-hg-ai-consent]")).toBeHidden(); + await launcher.click(); await expect(page.locator("[data-hg-ai-consent]")).toBeVisible(); await page.locator("[data-hg-ai-cancel]").click(); await expect(page.locator("[data-hg-ai-consent]")).not.toBeVisible(); From c3498835f242506be2e0e95f833b3fd93c28275c Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 2 Oct 2026 21:11:50 +0800 Subject: [PATCH 10/13] fix: link AI community help to issues - send the answer community action to GitHub Issues - verify the destination in English and Chinese browser tests - align adapter contracts and rollout documentation --- layouts/_partials/hooks/body-end.html | 2 +- scripts/kapa-rollout.md | 2 +- tests/e2e/ai.spec.js | 2 +- tests/ui-ai/kapa-adapter.test.cjs | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/layouts/_partials/hooks/body-end.html b/layouts/_partials/hooks/body-end.html index ff9b307d64..82e672baa6 100644 --- a/layouts/_partials/hooks/body-end.html +++ b/layouts/_partials/hooks/body-end.html @@ -39,7 +39,7 @@ "historical" $historical "labels" $labels "exampleQuestions" $questions - "communityURL" "https://github.com/apache/hugegraph/discussions" + "communityURL" "https://github.com/apache/hugegraph/issues" -}}