diff --git a/.github/images/site-ci-overview.png b/.github/images/site-ci-overview.png new file mode 100644 index 0000000000..277669c690 Binary files /dev/null and b/.github/images/site-ci-overview.png differ diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index 50cf9928b0..05c010d6ad 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -45,7 +45,6 @@ jobs: env: HUGO_CACHEDIR: /tmp/hugo-cache-${{ github.run_id }}-${{ github.run_attempt }}-prepare outputs: - versions: ${{ steps.matrix.outputs.versions }} selection: ${{ steps.plan.outputs.selection }} site_origin: ${{ steps.plan.outputs.site_origin }} historical_origin: ${{ steps.plan.outputs.historical_origin }} @@ -53,6 +52,11 @@ jobs: publish_branch: ${{ steps.plan.outputs.publish_branch }} latest_sha: ${{ steps.plan.outputs.latest_sha }} source_sha: ${{ steps.plan.outputs.source_sha }} + python_version: ${{ steps.python.outputs.python-version }} + cache_key: ${{ steps.history-plan.outputs.key }} + history_hit: ${{ steps.history-lookup.outcome == 'success' && steps.history-lookup.outputs.cache-hit || 'false' }} + groups: ${{ steps.groups.outputs.groups }} + group_pattern: ${{ steps.groups.outputs.pattern }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -60,6 +64,7 @@ jobs: fetch-depth: 0 persist-credentials: false - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + id: python with: python-version: "3.13" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 @@ -166,7 +171,7 @@ jobs: go mod download PYTHONDONTWRITEBYTECODE=1 \ python3 -m unittest discover -s scripts -p 'test_*.py' -v - - name: Resolve immutable version matrix + - name: Resolve immutable version manifest id: matrix env: LATEST_SHA: ${{ steps.plan.outputs.latest_sha }} @@ -175,13 +180,50 @@ jobs: python3 scripts/versioning.py prepare \ --latest-sha "$LATEST_SHA" --select "$SELECTION" \ --output resolved-versions.json - echo "versions=$(jq -c '.include' resolved-versions.json)" >> "$GITHUB_OUTPUT" + - name: Compute historical build fingerprints + id: history-plan + env: + SITE_ORIGIN: ${{ steps.plan.outputs.site_origin }} + HISTORICAL_ORIGIN: ${{ steps.plan.outputs.historical_origin }} + run: | + python3 scripts/version_cache.py plan \ + --resolved-manifest resolved-versions.json \ + --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ + --hugo-version "$HUGO_VERSION" --go-version "$(go version)" \ + --cache-dir "$RUNNER_TEMP/history-cache" --output cache-plan.json + echo "key=$(jq -r .cacheKey cache-plan.json)" >> "$GITHUB_OUTPUT" + - name: Look up the complete historical bundle + id: history-lookup + continue-on-error: true + if: github.event_name != 'workflow_dispatch' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ${{ runner.temp }}/history-cache + key: ${{ runner.os }}-${{ steps.history-plan.outputs.key }} + lookup-only: true + - name: Select bounded build groups + id: groups + env: + CACHE_HIT: ${{ steps.history-lookup.outcome == 'success' && steps.history-lookup.outputs.cache-hit || 'false' }} + run: | + python3 scripts/version_cache.py groups \ + --resolved-manifest resolved-versions.json --cache-hit "$CACHE_HIT" > build-groups.json + echo "groups=$(jq -c . build-groups.json)" >> "$GITHUB_OUTPUT" + # A full rerun can switch from three cold groups to one warm group. + # Match only that current group, never the older attempt's artifacts. + if [[ "$(jq length build-groups.json)" == 1 ]]; then + echo 'pattern=0' >> "$GITHUB_OUTPUT" + else + echo 'pattern=*' >> "$GITHUB_OUTPUT" + fi - name: Upload resolved version manifest uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: # Keep dependency artifact names stable across selective job reruns. name: resolved-versions-${{ github.run_id }} - path: resolved-versions.json + path: | + resolved-versions.json + cache-plan.json retention-days: 7 if-no-files-found: error overwrite: true @@ -189,16 +231,16 @@ jobs: build: needs: prepare runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 25 permissions: { contents: read } env: - HUGO_CACHEDIR: /tmp/hugo-cache-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.version.id }} + HUGO_CACHEDIR: /tmp/hugo-resources strategy: fail-fast: false - max-parallel: 5 + max-parallel: 3 matrix: - version: ${{ fromJSON(needs.prepare.outputs.versions) }} - name: Build ${{ matrix.version.id }} + group: ${{ fromJSON(needs.prepare.outputs.groups) }} + name: Build group ${{ matrix.group.id }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -207,14 +249,11 @@ jobs: persist-credentials: false - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: - python-version: "3.13" - - name: Fetch immutable resolved source - env: - RESOLVED_SHA: ${{ matrix.version.sha }} - run: | - [[ "$RESOLVED_SHA" =~ ^[0-9a-f]{40}$ ]] - git fetch --no-tags origin "$RESOLVED_SHA" - git cat-file -e "$RESOLVED_SHA^{commit}" + python-version: ${{ needs.prepare.outputs.python_version }} + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: resolved-versions-${{ github.run_id }} + path: resolved - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: go.mod @@ -228,33 +267,80 @@ jobs: uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: ${{ env.HUGO_CACHEDIR }} - key: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-${{ matrix.version.id }}-${{ hashFiles('go.sum') }} - restore-keys: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-${{ matrix.version.id }}- + key: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-group${{ matrix.group.id }}-${{ hashFiles('go.sum') }} + restore-keys: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-group${{ matrix.group.id }}- - name: Verify pinned OINK module run: | + lscpu | sed -n '/^CPU(s):/p; /^Model name:/p' test "$(go list -m -f '{{ .Path }}')" = "github.com/apache/hugegraph-doc" test "$(go list -m all | wc -l)" -eq 2 test -z "$(go list -m -f '{{ with .Replace }}{{ .Path }}{{ end }}' github.com/pgsty/oink)" python3 scripts/update_oink.py --check-baseline - - name: Build isolated version artifact + - name: Select this group's immutable versions + env: + VERSION_IDS: ${{ toJSON(matrix.group.versions) }} + run: | + jq --argjson ids "$VERSION_IDS" \ + '.include |= map(select(.id as $id | $ids | index($id)))' \ + resolved/resolved-versions.json > selected-versions.json + - name: Restore historical artifact bundle + id: history-cache + continue-on-error: true + if: github.event_name != 'workflow_dispatch' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ${{ runner.temp }}/history-cache + key: ${{ runner.os }}-${{ needs.prepare.outputs.cache_key }} + # A previous bundle is only transport: each version must pass its + # exact input fingerprint and file digest before it can be reused. + restore-keys: ${{ runner.os }}-history-v1- + - name: Build and validate selected versions + if: github.event_name != 'workflow_dispatch' env: OINK_PYTHON: python3 SITE_ORIGIN: ${{ needs.prepare.outputs.site_origin }} HISTORICAL_ORIGIN: ${{ needs.prepare.outputs.historical_origin }} run: | - python3 scripts/versioning.py build \ - --version "${{ matrix.version.id }}" --sha "${{ matrix.version.sha }}" \ + python3 scripts/version_cache.py plan \ + --resolved-manifest selected-versions.json \ --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ - --output "${RUNNER_TEMP}/version-public" - python3 scripts/versioning.py validate \ - --version "${{ matrix.version.id }}" --sha "${{ matrix.version.sha }}" \ + --hugo-version "$HUGO_VERSION" --go-version "$(go version)" \ + --cache-dir "$RUNNER_TEMP/history-cache" --output "$RUNNER_TEMP/cache-plan.json" \ + --expected-plan resolved/cache-plan.json + python3 scripts/build_versions.py \ + --resolved-manifest selected-versions.json \ + --plan "$RUNNER_TEMP/cache-plan.json" --cache-dir "$RUNNER_TEMP/history-cache" \ --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ - --artifact "${RUNNER_TEMP}/version-public" - - name: Upload isolated version artifact + --workers 2 --output "$RUNNER_TEMP/version-artifacts" + - name: Build and validate manual candidate versions + if: github.event_name == 'workflow_dispatch' + env: + OINK_PYTHON: python3 + SITE_ORIGIN: ${{ needs.prepare.outputs.site_origin }} + HISTORICAL_ORIGIN: ${{ needs.prepare.outputs.historical_origin }} + run: | + # Candidate branches may predate the cache and scheduling helpers. + jq -er '.include[] | [.id, .sha] | @tsv' selected-versions.json > selected-versions.tsv + cache_root="$HUGO_CACHEDIR" + while IFS=$'\t' read -r version sha; do + [[ "$version" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || exit 1 + [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || exit 1 + git cat-file -e "$sha^{commit}" || git fetch --no-tags origin "$sha" + export HUGO_CACHEDIR="$cache_root/$version" + python3 scripts/versioning.py build \ + --version "$version" --sha "$sha" \ + --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ + --output "$RUNNER_TEMP/version-artifacts/$version" + python3 scripts/versioning.py validate \ + --version "$version" --sha "$sha" \ + --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ + --artifact "$RUNNER_TEMP/version-artifacts/$version" + done < selected-versions.tsv + - name: Upload selected version artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: ${{ needs.prepare.outputs.artifact_prefix }}-${{ matrix.version.id }}-${{ github.run_id }} - path: ${{ runner.temp }}/version-public + name: ${{ needs.prepare.outputs.artifact_prefix }}-versions-${{ matrix.group.id }}-${{ github.run_id }} + path: ${{ runner.temp }}/version-artifacts include-hidden-files: true retention-days: 1 if-no-files-found: error @@ -262,6 +348,9 @@ jobs: # Assembly and browser checks are serial; reuse one runner and its local output. aggregate: + # Preserve the required check name without scheduling a separate gate runner. + name: deploy + if: ${{ !cancelled() }} needs: [prepare, build] runs-on: ubuntu-latest timeout-minutes: 35 @@ -269,35 +358,47 @@ jobs: env: HUGO_CACHEDIR: /tmp/hugo-cache-${{ github.run_id }}-${{ github.run_attempt }}-aggregate steps: + - name: Require all blocking predecessors to succeed + env: + PREPARE_RESULT: ${{ needs.prepare.result }} + BUILD_RESULT: ${{ needs.build.result }} + run: | + test "$PREPARE_RESULT" = success + test "$BUILD_RESULT" = success - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ needs.prepare.outputs.source_sha }} persist-credentials: false - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: - python-version: "3.13" + python-version: ${{ needs.prepare.outputs.python_version }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: resolved-versions-${{ github.run_id }} path: resolved - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - pattern: ${{ needs.prepare.outputs.artifact_prefix }}-*-${{ github.run_id }} + pattern: ${{ needs.prepare.outputs.artifact_prefix }}-versions-${{ needs.prepare.outputs.group_pattern }}-${{ github.run_id }} + merge-multiple: true path: version-artifacts - name: Assemble publishable site env: + EVENT_NAME: ${{ github.event_name }} PREFIX: ${{ needs.prepare.outputs.artifact_prefix }} SITE_ORIGIN: ${{ needs.prepare.outputs.site_origin }} HISTORICAL_ORIGIN: ${{ needs.prepare.outputs.historical_origin }} SELECTION: ${{ needs.prepare.outputs.selection }} run: | + lscpu | sed -n '/^CPU(s):/p; /^Model name:/p' extra=() + if [[ "$EVENT_NAME" != workflow_dispatch ]]; then + extra+=(--workers 2) + fi if [[ "$PREFIX" == staging ]]; then extra+=(--asf-profile oink --asf-whoami asf-staging-oink) fi python3 scripts/versioning.py aggregate \ - --artifacts version-artifacts --artifact-prefix "$PREFIX-" \ - --artifact-suffix="-${GITHUB_RUN_ID}" \ + --artifacts version-artifacts \ --resolved-manifest resolved/resolved-versions.json \ --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ --select "$SELECTION" \ @@ -306,6 +407,29 @@ jobs: env: DOWNLOAD_PUBLIC_DIR: ${{ runner.temp }}/public-site run: python3 -m unittest scripts.test_download_data.DownloadDataTest.test_rendered_download_pages_have_verified_rows -v + - name: Record validated historical artifacts + id: history-record + if: github.event_name != 'workflow_dispatch' && needs.prepare.outputs.history_hit != 'true' + run: | + complete=false + while IFS= read -r version; do + if python3 scripts/version_cache.py record \ + --plan resolved/cache-plan.json --version "$version" \ + --artifact "version-artifacts/$version" --cache-dir "$RUNNER_TEMP/history-cache"; then + complete=true + else + echo '::warning::Could not record history cache; validated site remains publishable.' + complete=false + break + fi + done < <(jq -r '.versions[] | select(.archived) | .id' resolved/cache-plan.json) + echo "complete=$complete" >> "$GITHUB_OUTPUT" + - name: Save complete historical bundle + if: steps.history-record.outputs.complete == 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ${{ runner.temp }}/history-cache + key: ${{ runner.os }}-${{ needs.prepare.outputs.cache_key }} - name: Upload publishable aggregate uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -316,7 +440,7 @@ jobs: if-no-files-found: error overwrite: true - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: "24" cache: npm @@ -387,7 +511,7 @@ jobs: with: ref: ${{ needs.prepare.outputs.source_sha }} persist-credentials: false - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: "24" cache: npm @@ -413,27 +537,9 @@ jobs: retention-days: 7 if-no-files-found: warn - # Required check name in .asf.yaml. It gates artifacts without write access. - deploy: - if: ${{ !cancelled() }} - needs: [prepare, build, aggregate] - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: { contents: read } - steps: - - name: Require all blocking jobs to succeed - env: - PREPARE_RESULT: ${{ needs.prepare.result }} - BUILD_RESULT: ${{ needs.build.result }} - AGGREGATE_RESULT: ${{ needs.aggregate.result }} - run: | - test "$PREPARE_RESULT" = success - test "$BUILD_RESULT" = success - test "$AGGREGATE_RESULT" = success - publish: if: needs.prepare.outputs.publish_branch != '' - needs: [prepare, deploy] + needs: [prepare, aggregate] runs-on: ubuntu-latest timeout-minutes: 10 permissions: { contents: write } diff --git a/assets/js/kapa-adapter.js b/assets/js/kapa-adapter.js index 9c903e644a..993b1c1c94 100644 --- a/assets/js/kapa-adapter.js +++ b/assets/js/kapa-adapter.js @@ -141,8 +141,6 @@ // Storage may be disabled; consent still works for this page only. } var consented = choice === 'granted'; - var grantPersisted = consented; - var dismissed = choice === 'dismissed'; windowObject.addEventListener('storage', function (event) { if (event.key !== consentKey && event.key !== null) return; if ((consented && event.newValue !== 'granted') || @@ -155,14 +153,14 @@ var pending = null; var consent = documentObject.querySelector('[data-hg-ai-consent]'); var launcher = documentObject.querySelector('.hg-ask-ai-launcher'); - if (consent) consent.hidden = consented || dismissed; + if (consent) consent.hidden = true; function renderLauncher(hidden) { if (!launcher) return; launcher.hidden = hidden; if (consented) launcher.setAttribute('aria-haspopup', 'dialog'); else launcher.removeAttribute('aria-haspopup'); } - renderLauncher(!consented && !dismissed); + renderLauncher(false); function dismissConsent() { if (consent) consent.hidden = true; @@ -175,23 +173,6 @@ var activeQueue = null; var operation = null; var status = documentObject.querySelector('[data-hg-ai-status]'); - var revoke = documentObject.querySelector('[data-hg-ai-revoke]'); - if (revoke) { - revoke.hidden = !consented; - revoke.addEventListener('click', function () { - if (!consented) return; - if (grantPersisted) { - try { - windowObject.localStorage.removeItem(consentKey); - } catch (_) { - renderState(state, config.labels.revokeError); - return; - } - } - // Navigation terminates the vendor's loaded state and pending callbacks. - windowObject.location.reload(); - }); - } function renderState(next, message) { state = next; @@ -412,11 +393,9 @@ consented = true; try { windowObject.localStorage.setItem(consentKey, 'granted'); - grantPersisted = true; } catch (_) { // Never bypass initial consent when persistence is unavailable. } - if (revoke) revoke.hidden = false; dismissConsent(); load(request.query, request.submit); }); diff --git a/assets/scss/_styles_project.scss b/assets/scss/_styles_project.scss index f22f015a68..b0286e7184 100644 --- a/assets/scss/_styles_project.scss +++ b/assets/scss/_styles_project.scss @@ -865,12 +865,33 @@ } -// The first-use notice stays beside the AI entry without blocking the page. -.hg-ai-revoke { - font-size: inherit; - &[hidden] { display: none; } +// Let documentation results scroll without covering them with the AI action. +.td-shell-search__list:has(> [aria-labelledby="td-shell-search-group-actions"]) { + display: flex; + flex-direction: column; + overflow: hidden; + + > .td-shell-search__group { + min-height: 0; + overflow-y: auto; + overscroll-behavior: contain; + } } +.td-shell-search__group[aria-labelledby="td-shell-search-group-actions"] { + flex-shrink: 0; + background: var(--bs-body-bg); + border-top: 1px solid var(--bs-border-color); + box-shadow: 0 -.25rem .75rem rgba(0, 0, 0, .06); + + .td-shell-search__item { + color: var(--bs-primary); + background: rgba(var(--bs-primary-rgb), .08); + } +} + +// The first-use notice opens beside the AI entry after explicit activation. + .hg-ai-consent { position: fixed; z-index: 1040; diff --git a/content/cn/docs/quickstart/toolchain/export-migration/hugegraph-seatunnel-source.md b/content/cn/docs/quickstart/toolchain/export-migration/hugegraph-seatunnel-source.md index 7bff2e2c69..b528728072 100644 --- a/content/cn/docs/quickstart/toolchain/export-migration/hugegraph-seatunnel-source.md +++ b/content/cn/docs/quickstart/toolchain/export-migration/hugegraph-seatunnel-source.md @@ -6,7 +6,10 @@ weight: 2 如果你需要把数据从一张 HugeGraph 图复制到另一张图，使用 `graph2graph`：HugeGraph Source 从源图（A 图）读取顶点和边，数据经过可选的 Transform 后，由 HugeGraph Sink 写入目标图（B 图）。数据方向是 `A 图 → HugeGraph Source →（可选 Transform）→ HugeGraph Sink → B 图`。如果要把图数据导出到文件、JDBC、Kafka 等其他系统，则使用 `graph2any`，由下游 Sink 接收 Source 读取的数据。本文介绍这两类任务。 -> **版本要求：本文面向 SeaTunnel [3.0+](https://github.com/apache/seatunnel/tree/3.0.0-release)** +> **版本要求：本文面向 SeaTunnel [3.0+](https://seatunnel.apache.org/download/)** + +SeaTunnel 3.0+ 提供 HugeGraph Source，支持图模型自动发现和多标签读取。 +并行扫描的后端要求和配置限制见[官方 Source 文档](https://seatunnel.apache.org/docs/connectors/source/HugeGraph/)。 开始前请先完成[导入页中的通用环境准备和配置](/cn/docs/quickstart/toolchain/import/hugegraph-seatunnel-connector/#2-准备环境)，其中包含 JDK、HOCON、插件安装和图模型说明。 @@ -138,7 +141,7 @@ sink { > **为什么保留 ID？** HugeGraph 的 `PRIMARY_KEY` ID 包含顶点标签的内部 ID，两张图可能不同。例如源图顶点是 `1:marko`，目标图重新按主键生成的可能是 `2:marko`。如果重新生成顶点 ID 后仍复用源图的边端点，边就会连错。本例将原 ID 保存为字符串，因此会改变目标图的 ID 策略 -若要一次读取全部标签，省略 Source 的 `label` 后会读取 `label_type`（默认 `VERTEX`）下的全部 label，每个 label 输出一张表。这时需用 `sourceTable` 将各 Sink 映射绑定到对应表，例如 `sourceTable = "default.person"`；具体值以 Writer 日志中的完整表名为准。不能直接套用本节的单标签配置。其他限制见 [HugeGraph Source 文档](https://github.com/apache/seatunnel/blob/3.0.0-release/docs/zh/connectors/source/HugeGraph.md)。 +若要一次读取全部标签，省略 Source 的 `label` 后会读取 `label_type`（默认 `VERTEX`）下的全部 label，每个 label 输出一张表。这时需用 `sourceTable` 将各 Sink 映射绑定到对应表，例如 `sourceTable = "default.person"`；具体值以 Writer 日志中的完整表名为准。不能直接套用本节的单标签配置。其他限制见 [HugeGraph Source 文档](https://seatunnel.apache.org/docs/connectors/source/HugeGraph/)。 ## 2 导出到其他系统（graph2any） @@ -188,8 +191,8 @@ sink { **连接器** -

[1] HugeGraph Source
-[2] LocalFile Sink

+

[1] HugeGraph Source
+[2] LocalFile Sink

**关联文档** diff --git a/content/cn/docs/quickstart/toolchain/import/hugegraph-seatunnel-connector.md b/content/cn/docs/quickstart/toolchain/import/hugegraph-seatunnel-connector.md index 430f55ca18..83da9368e2 100644 --- a/content/cn/docs/quickstart/toolchain/import/hugegraph-seatunnel-connector.md +++ b/content/cn/docs/quickstart/toolchain/import/hugegraph-seatunnel-connector.md @@ -8,7 +8,7 @@ aliases: SeaTunnel 可以把数据库、Kafka 等数据源接入 HugeGraph。连接器分为两部分：**Source 负责读取，Sink 负责写入**[1][2]，中间可以接 SeaTunnel 的数据转换组件。需要从 HugeGraph 导出或迁移数据时，请查看[SeaTunnel Source 导出与迁移文档](/cn/docs/quickstart/toolchain/export-migration/hugegraph-seatunnel-source/)。 -> **版本要求：本文面向 SeaTunnel [3.0+](https://github.com/apache/seatunnel/tree/3.0.0-release)。** 所有示例使用 `mappings` +> **版本要求：本文面向 SeaTunnel [3.0+](https://seatunnel.apache.org/download/)。** 所有示例使用 SeaTunnel 3.0+ 提供的 `mappings` 配置。 [![Loader 与 SeaTunnel 的工作流对比：直接导入图数据与复用 Source、Transform、Sink 管道](/cn/docs/images/seatunnel/seatunnel-vs-loader-zh.png)](/cn/docs/images/seatunnel/seatunnel-vs-loader-zh.png) @@ -38,25 +38,45 @@ SeaTunnel 覆盖 Loader 的图导入和 Tools 的导出、迁移场景，可以 > **已有 Spark/Flink 每日任务** > -> HugeGraph Source 和 Sink 在 SeaTunnel 3.0.0-release 中都支持 SeaTunnel Engine（Zeta）、Spark 和 Flink。若把每日任务改成 SeaTunnel 作业，并用对应引擎提交，数据可以在 Source → Transform → Sink 之间直接传递，不需要先落盘再交给 Loader。若保留现有 Spark/Flink DAG，SeaTunnel 不会自动接管内存中的 DataFrame 或 Stream，需要改造成 SeaTunnel 作业，或让 Source 读取已有系统中的数据 +> HugeGraph Source 和 Sink 在 SeaTunnel 3.0+ 中都支持 SeaTunnel Engine（Zeta）、Spark 和 Flink。若把每日任务改成 SeaTunnel 作业，并用对应引擎提交，数据可以在 Source → Transform → Sink 之间直接传递，不需要先落盘再交给 Loader。若保留现有 Spark/Flink DAG，SeaTunnel 不会自动接管内存中的 DataFrame 或 Stream，需要改造成 SeaTunnel 作业，或让 Source 读取已有系统中的数据 -Loader 和 Tools 的优势是专注、直接、上手快。需要直接导入图数据时可先用 Loader；需要备份、恢复、导出或日常运维时可用 Tools。如果已经有 SeaTunnel 作业，通常在原管道中接入 HugeGraph 更方便。需要更高导入吞吐时，Loader 的 bypass-server 和其他导入优化更合适；Loader 在特定后端、数据规模和硬件条件下实测峰值可达 100～200 万条/秒，不能直接当作通用性能承诺，仍需单独压测。新建 SeaTunnel 任务使用 [3.0+](https://github.com/apache/seatunnel/tree/3.0.0-release) 和 `mappings`。使用其他版本时，请重新核对连接器配置。 +Loader 和 Tools 的优势是专注、直接、上手快。需要直接导入图数据时可先用 Loader；需要备份、恢复、导出或日常运维时可用 Tools。如果已经有 SeaTunnel 作业，通常在原管道中接入 HugeGraph 更方便。需要更高导入吞吐时，Loader 的 bypass-server 和其他导入优化更合适；Loader 在特定后端、数据规模和硬件条件下实测峰值可达 100～200 万条/秒，不能直接当作通用性能承诺，仍需单独压测。新建 SeaTunnel 任务使用 [3.0+](https://seatunnel.apache.org/download/) 和 `mappings`。使用其他版本时，请重新核对连接器配置。 ## 2 准备环境 ### 2.1 获取 SeaTunnel 3.0+ -SeaTunnel 3.0+ 官方开发文档列出 JDK 8 和 JDK 11；本文统一使用 JDK 11，并设置 `JAVA_HOME`。从 [SeaTunnel 3.0+](https://github.com/apache/seatunnel/tree/3.0.0-release)[4] 获取源码，按上游[开发环境文档](https://github.com/apache/seatunnel/blob/3.0.0-release/docs/zh/developer/setup.md)[5]构建发行包： +使用 JDK 11 并设置 `JAVA_HOME`；官方[部署文档](https://seatunnel.apache.org/docs/getting-started/locally/deployment/) +列出的前置要求为 Java 8 或 11。从[官方下载页](https://seatunnel.apache.org/download/)[4] +选择 3.0+ 二进制发行包，并使用页面提供的校验文件或签名验证下载包。 +下面以 3.0.0 为例，请将 `SEATUNNEL_VERSION` 改为实际下载的发行版本。 ```bash -git clone --branch 3.0.0-release https://github.com/apache/seatunnel.git -cd seatunnel -./mvnw clean package -pl seatunnel-dist -am -Dmaven.test.skip=true +export SEATUNNEL_VERSION=3.0.0 +tar -xzf "apache-seatunnel-${SEATUNNEL_VERSION}-bin.tar.gz" +cd "apache-seatunnel-${SEATUNNEL_VERSION}" ``` -解压 `seatunnel-dist/target/` 中生成的二进制包，后续命令都在解压后的 SeaTunnel 安装目录执行。需要更新功能时，可以切换到其他版本；引擎与连接器插件应来自同一次构建，避免混用不同版本的 JAR。 +二进制发行包不包含连接器插件。在 `config/plugin_config` 中选择示例需要的连接器： -本文使用 SeaTunnel 自带的 **Zeta 引擎和 local 模式**[6][7]。确认安装目录的 `connectors/` 中包含 HugeGraph，以及所需的 JDBC 或 Kafka 连接器[11][12]；如果自定义构建没有包含它们，需补齐同一次构建产出的插件。JDBC 示例还需要将 MySQL 驱动 JAR 放入 `lib/`，驱动类为 `com.mysql.cj.jdbc.Driver`。 +```text +--seatunnel-connectors-- +connector-hugegraph +connector-jdbc +connector-kafka +--end-- +``` + +按官方[插件安装说明](https://seatunnel.apache.org/docs/getting-started/locally/deployment/#download-the-connector-plugins)[5] +安装同版本的已发布插件： + +```bash +sh bin/install-plugin.sh "${SEATUNNEL_VERSION}" +``` + +后续命令均在该安装目录执行，引擎和连接器插件保持相同版本。 +本文使用自带的 **Zeta 引擎和 local 模式**[6][7]。确认 `connectors/` 中有 HugeGraph，以及所需的 JDBC 或 Kafka 插件[11][12]。 +JDBC 示例还需将 MySQL 驱动 JAR 放入 `lib/`，驱动类为 `com.mysql.cj.jdbc.Driver`。 ### 2.2 准备 HugeGraph 和数据源 @@ -294,7 +314,7 @@ HugeGraph Sink 是 **at-least-once（至少一次）** 写入，故障恢复可 | `batch_size` | 单批记录数，默认 500 | | `env.sink.flush.interval` | Zeta 定时刷新间隔，单位毫秒 | | `check_vertex` | 写边时检查端点，本文的边任务设为 `true` | -| [`batch_failure_fallback`](https://github.com/apache/seatunnel/blob/3.0.0-release/docs/zh/connectors/sink/HugeGraph.md)[2] | 默认 `true`，批量失败后逐条重试，最多跳过 `max_insert_errors` 条失败记录；本文示例显式设为 `false`，让批量失败直接终止任务 | +| [`batch_failure_fallback`](https://seatunnel.apache.org/docs/connectors/sink/HugeGraph/)[2] | 默认 `true`，批量失败后逐条重试，最多跳过 `max_insert_errors` 条失败记录；本文示例显式设为 `false`，让批量失败直接终止任务 | | `max_insert_errors` | 逐条回退时允许跳过的失败记录数；默认 `500`，`-1` 表示不限制，仅在开启 `batch_failure_fallback` 时生效 | 遇到问题时可按下面检查： @@ -314,33 +334,25 @@ HugeGraph Sink 是 **at-least-once（至少一次）** 写入，故障恢复可 **HugeGraph 连接器** -

[1] HugeGraph Source
-[2] HugeGraph Sink

+

[1] HugeGraph Source
+[2] HugeGraph Sink

**配置与部署**

[3] HOCON 作业文件配置说明
-[4] SeaTunnel 3.0.0-release 分支
-[5] SeaTunnel 开发环境文档
+[4] SeaTunnel 3.0+ 正式版下载
+[5] 连接器插件安装说明
[6] SeaTunnel 本地部署

**执行引擎** -

[7] SeaTunnel 引擎概览
-[8] SeaTunnel Spark 引擎
-[9] SeaTunnel Flink 引擎
-[10] Connector V2 多引擎说明

+

[7] SeaTunnel 引擎概览
+[8] SeaTunnel Spark 引擎
+[9] SeaTunnel Flink 引擎
+[10] Connector V2 多引擎说明

**数据源连接器** -

[11] JDBC Source
-[12] Kafka Source
+

[11] JDBC Source
+[12] Kafka Source
[13] MySQL CDC Source

- -**旧版本兼容** - -

[14] SeaTunnel 2.3.13 HugeGraph Sink

- -> **旧版本说明** -> -> 本文面向 SeaTunnel 3.0+，文中的 Source、`mappings` 和图迁移示例不适用于 2.3.13。2.3.13 已过时，仅提供 HugeGraph Sink，配置使用 `schema_config`，并且需要提前创建图模型。如必须使用 2.3.13，请参考[官方 Sink 文档](https://github.com/apache/seatunnel/blob/2.3.13/docs/zh/connectors/sink/HugeGraph.md)[14]，不要套用本文配置 diff --git a/content/en/docs/quickstart/toolchain/export-migration/hugegraph-seatunnel-source.md b/content/en/docs/quickstart/toolchain/export-migration/hugegraph-seatunnel-source.md index 29e1260a0f..7b5437ac89 100644 --- a/content/en/docs/quickstart/toolchain/export-migration/hugegraph-seatunnel-source.md +++ b/content/en/docs/quickstart/toolchain/export-migration/hugegraph-seatunnel-source.md @@ -6,7 +6,11 @@ weight: 2 If you need to copy data from one HugeGraph graph to another, use `graph2graph`: HugeGraph Source reads vertices and edges from the source graph (graph A), and HugeGraph Sink writes them to the target graph (graph B), with an optional Transform in between. The data path is `graph A → HugeGraph Source → (optional Transform) → HugeGraph Sink → graph B`. If you need to export graph data to a file, JDBC, Kafka, or another system, use `graph2any`, where a downstream Sink receives the records read by HugeGraph Source. This page covers both job types. -> **Version requirement: This guide targets SeaTunnel [3.0+](https://github.com/apache/seatunnel/tree/3.0.0-release)** +> **Version requirement: This guide targets SeaTunnel [3.0+](https://seatunnel.apache.org/download/)** + +SeaTunnel 3.0+ provides HugeGraph Source, with schema auto-discovery and multi-label reads. +See the [official Source documentation](https://seatunnel.apache.org/docs/connectors/source/HugeGraph/) +for parallel-scan backend requirements and configuration limits. Before starting, complete the [shared environment and configuration steps on the import page](/docs/quickstart/toolchain/import/hugegraph-seatunnel-connector/#2-prepare-the-environment). They cover JDK, HOCON, plugin installation, and the sample graph model. @@ -138,7 +142,7 @@ This example checks endpoints and makes write errors fail the job. The default ` > **Why preserve IDs?** A HugeGraph `PRIMARY_KEY` ID contains the internal ID of the vertex label, and that internal ID can differ between graphs. For example, a source vertex can be `1:marko`, while regenerating the primary key in the target graph can produce `2:marko`. Reusing the source edge endpoints after regenerating vertex IDs can connect edges to the wrong vertices. This example stores the original ID as a string, which changes the target graph's ID strategy -When Source reads every label, omit `label` to read all labels of `label_type` (default `VERTEX`). It produces one output table per label. Bind each Sink mapping to its table with `sourceTable`, for example `sourceTable = "default.person"`; use the full table name shown in the Writer log for the exact value. Do not reuse the single-label configuration from this section. See the [HugeGraph Source documentation](https://github.com/apache/seatunnel/blob/3.0.0-release/docs/en/connectors/source/HugeGraph.md) for other limitations. +When Source reads every label, omit `label` to read all labels of `label_type` (default `VERTEX`). It produces one output table per label. Bind each Sink mapping to its table with `sourceTable`, for example `sourceTable = "default.person"`; use the full table name shown in the Writer log for the exact value. Do not reuse the single-label configuration from this section. See the [HugeGraph Source documentation](https://seatunnel.apache.org/docs/connectors/source/HugeGraph/) for other limitations. ## 2 Export to another system (graph2any) @@ -188,8 +192,8 @@ This page covers row reads and writes. It does not copy source indexes, TTLs, or **Connectors** -

[1] HugeGraph Source
-[2] LocalFile Sink

+

[1] HugeGraph Source
+[2] LocalFile Sink

**Related guide** diff --git a/content/en/docs/quickstart/toolchain/import/hugegraph-seatunnel-connector.md b/content/en/docs/quickstart/toolchain/import/hugegraph-seatunnel-connector.md index 04ef99700c..2c66bd08c8 100644 --- a/content/en/docs/quickstart/toolchain/import/hugegraph-seatunnel-connector.md +++ b/content/en/docs/quickstart/toolchain/import/hugegraph-seatunnel-connector.md @@ -8,7 +8,8 @@ aliases: SeaTunnel connects data sources such as databases and Kafka to HugeGraph. The connector has two parts: **Source reads data and Sink writes data**[1][2], with SeaTunnel transform components available between them. To export or migrate data from HugeGraph, see the [SeaTunnel Source export and migration guide](/docs/quickstart/toolchain/export-migration/hugegraph-seatunnel-source/). -> **Version requirement: This guide targets SeaTunnel [3.0+](https://github.com/apache/seatunnel/tree/3.0.0-release).** All examples use `mappings` +> **Version requirement: This guide targets SeaTunnel [3.0+](https://seatunnel.apache.org/download/).** +> All examples use the `mappings` configuration available in SeaTunnel 3.0+. [![Loader imports data directly with graph mappings; SeaTunnel 3.0+ combines Source, Transform, and Sink, and both support JDBC, Kafka, and graph data](/docs/images/seatunnel/seatunnel-vs-loader-en.png)](/docs/images/seatunnel/seatunnel-vs-loader-en.png) @@ -38,25 +39,44 @@ SeaTunnel covers Loader's graph-import and Tools' export and migration scenarios > **Existing Spark/Flink daily jobs** > -> Both HugeGraph Source and Sink list SeaTunnel Engine (Zeta), Spark, and Flink as supported engines in SeaTunnel 3.0.0-release. If you express the daily job as a SeaTunnel job and submit it to that engine, records can move directly from Source to Transform to Sink without an intermediate file. If you keep the existing Spark/Flink DAG, SeaTunnel does not automatically take over its in-memory DataFrame or stream. Adapt it into a SeaTunnel job or expose the data through a Source connector +> Both HugeGraph Source and Sink list SeaTunnel Engine (Zeta), Spark, and Flink as supported engines in SeaTunnel 3.0+. If you express the daily job as a SeaTunnel job and submit it to that engine, records can move directly from Source to Transform to Sink without an intermediate file. If you keep the existing Spark/Flink DAG, SeaTunnel does not automatically take over its in-memory DataFrame or stream. Adapt it into a SeaTunnel job or expose the data through a Source connector -Loader and Tools are focused, direct, and quick to start. Use Loader for a direct graph import; use Tools for backup, restore, export, or daily operations. If a SeaTunnel job already exists, adding HugeGraph to that pipeline is usually simpler. For higher import throughput, Loader's bypass-server path and other import optimizations are a better fit; measured peaks of 1-2 million records/s require a specific backend, data set, and hardware configuration and are not a general performance guarantee. For new SeaTunnel jobs, use [3.0+](https://github.com/apache/seatunnel/tree/3.0.0-release) and `mappings`. Recheck the connector configuration when using another version. +Loader and Tools are focused, direct, and quick to start. Use Loader for a direct graph import; use Tools for backup, restore, export, or daily operations. If a SeaTunnel job already exists, adding HugeGraph to that pipeline is usually simpler. For higher import throughput, Loader's bypass-server path and other import optimizations are a better fit; measured peaks of 1-2 million records/s require a specific backend, data set, and hardware configuration and are not a general performance guarantee. For new SeaTunnel jobs, use [3.0+](https://seatunnel.apache.org/download/) and `mappings`. Recheck the connector configuration when using another version. ## 2 Prepare the environment ### 2.1 Get SeaTunnel 3.0+ -The SeaTunnel 3.0+ setup guide lists JDK 8 and JDK 11 as supported. This guide uses JDK 11 and sets `JAVA_HOME`. Clone the [SeaTunnel 3.0+](https://github.com/apache/seatunnel/tree/3.0.0-release)[4] branch and build a distribution by following the upstream [development setup guide](https://github.com/apache/seatunnel/blob/3.0.0-release/docs/en/developer/setup.md)[5]: +Use JDK 11 and set `JAVA_HOME`; the official [deployment guide](https://seatunnel.apache.org/docs/getting-started/locally/deployment/) +lists Java 8 and 11 as prerequisites. Choose a 3.0+ binary distribution from the +[official release page](https://seatunnel.apache.org/download/)[4] and verify its checksum or signature using the files linked there. +The commands below use 3.0.0 as an example; set `SEATUNNEL_VERSION` to the release you downloaded. ```bash -git clone --branch 3.0.0-release https://github.com/apache/seatunnel.git -cd seatunnel -./mvnw clean package -pl seatunnel-dist -am -Dmaven.test.skip=true +export SEATUNNEL_VERSION=3.0.0 +tar -xzf "apache-seatunnel-${SEATUNNEL_VERSION}-bin.tar.gz" +cd "apache-seatunnel-${SEATUNNEL_VERSION}" ``` -Extract the binary package from `seatunnel-dist/target/`. Run the remaining commands from the extracted SeaTunnel installation directory. When updating the feature set, switch to another version as needed. Keep the engine and connector plugins from the same build, and do not mix different plugin versions. +The binary distribution does not include connector plugins. In `config/plugin_config`, select the connectors needed by the examples: -This guide uses the bundled **Zeta engine in local mode**[6][7]. Check that `connectors/` contains HugeGraph and the JDBC or Kafka connector required by each example[11][12]. If a custom build does not include them, add the plugins produced by that same build. The JDBC examples also require the MySQL driver JAR in `lib/`, with driver class `com.mysql.cj.jdbc.Driver`. +```text +--seatunnel-connectors-- +connector-hugegraph +connector-jdbc +connector-kafka +--end-- +``` + +Install the matching released plugins using the upstream plugin installation instructions[5]: + +```bash +sh bin/install-plugin.sh "${SEATUNNEL_VERSION}" +``` + +Run the remaining commands from this installation directory and keep the engine and connector plugins at the same version. +This guide uses the bundled **Zeta engine in local mode**[6][7]. Check that `connectors/` contains HugeGraph, JDBC, and Kafka +as needed[11][12]. The JDBC examples also require the MySQL driver JAR in `lib/`, with driver class `com.mysql.cj.jdbc.Driver`. ### 2.2 Prepare HugeGraph and data sources @@ -294,7 +314,7 @@ The following table applies to the SeaTunnel 3.0+ version used by this guide: | `batch_size` | Number of records per batch; default 500 | | `env.sink.flush.interval` | Zeta scheduled flush interval in milliseconds | | `check_vertex` | Check edge endpoints; the edge job in this guide sets it to `true` | -| [`batch_failure_fallback`](https://github.com/apache/seatunnel/blob/3.0.0-release/docs/en/connectors/sink/HugeGraph.md)[2] | Defaults to `true`, so a failed batch falls back to record-by-record retries, capped by `max_insert_errors`; the examples explicitly set `false` so a batch failure stops the job | +| [`batch_failure_fallback`](https://seatunnel.apache.org/docs/connectors/sink/HugeGraph/)[2] | Defaults to `true`, so a failed batch falls back to record-by-record retries, capped by `max_insert_errors`; the examples explicitly set `false` so a batch failure stops the job | | `max_insert_errors` | Number of failed records that record-by-record fallback may skip; default `500`, `-1` for unlimited, and only applies when `batch_failure_fallback` is enabled | Use these checks when a job fails: @@ -314,33 +334,25 @@ Choose a tool based on the work to complete. Use [Tools](/docs/quickstart/toolch **HugeGraph connectors** -

[1] HugeGraph Source
-[2] HugeGraph Sink

+

[1] HugeGraph Source
+[2] HugeGraph Sink

**Configuration and deployment**

[3] HOCON job configuration
-[4] SeaTunnel 3.0.0-release branch
-[5] SeaTunnel development setup
+[4] SeaTunnel 3.0+ release download
+[5] Connector plugin installation
[6] SeaTunnel local deployment

**Execution engines** -

[7] SeaTunnel Engine Overview
-[8] SeaTunnel Spark Engine
-[9] SeaTunnel Flink Engine
-[10] Connector V2 multi-engine support

+

[7] SeaTunnel Engine Overview
+[8] SeaTunnel Spark Engine
+[9] SeaTunnel Flink Engine
+[10] Connector V2 multi-engine support

**Data source connectors** -

[11] JDBC Source
-[12] Kafka Source
+

[11] JDBC Source
+[12] Kafka Source
[13] MySQL CDC Source

- -**Legacy compatibility** - -

[14] SeaTunnel 2.3.13 HugeGraph Sink

- -> **Legacy version note** -> -> This guide targets SeaTunnel 3.0+. Its Source, `mappings`, and graph migration examples do not apply to 2.3.13. That legacy version provides only the HugeGraph Sink, uses `schema_config`, and requires the graph schema to be created in advance. If you must use 2.3.13, follow the [official Sink documentation](https://github.com/apache/seatunnel/blob/2.3.13/docs/en/connectors/sink/HugeGraph.md)[14] instead of copying this guide's configuration diff --git a/contribution.md b/contribution.md index 3b239726c4..5162910882 100644 --- a/contribution.md +++ b/contribution.md @@ -72,12 +72,38 @@ Cancelled runs skip report uploads and the final gate instead of holding the queue with `always()`. A queued job with no runner has not started testing; repeated reruns do not resolve runner capacity shortages. -Version builds remain parallel. Site assembly and blocking browser tests share -one runner and the same local artifact. The required `deploy` check still -requires all blocking jobs to succeed; visual captures remain advisory and -publication alone receives write permission. For a test failure, rerun failed +A complete historical cache hit uses one build job. Cold runs use at most three +build groups, with at most two versions building concurrently within each runner. +Historical artifacts are reused only when both the complete build-input fingerprint +and artifact digest match. Missing or invalid cache entries rebuild automatically; +manual dispatches start without the artifact cache and use the original build and +validate commands so older candidate branches do not need the new CI helpers. Cache hits still run the full +version validator, and aggregation validates every selected version again. + +Assembly validates at most two versions concurrently, then writes the complete +site and runs the cross-version checks. The final security scan also uses at most +two processes; it starts after version validation finishes, so these pools do not +nest. All checks remain enabled. Stage timings are printed as +`timing : s`; nested security timings are included in their parent +validation duration and must not be added twice. Site assembly and blocking browser +tests share one runner and the same local artifact. The assembly job carries the +required `deploy` check name, avoiding a separate gate runner and its queue time. +Its first step requires successful preparation and builds, and all assembly and +browser checks remain blocking. Visual captures remain advisory and publication +alone receives write permission. For a test failure, rerun failed jobs after inspecting the cause; artifact names remain stable within the run. +## SeaTunnel documentation versions + +The `latest` SeaTunnel guides use the 3.0+ minimum-version label and link to +upstream latest/Next documentation. Concrete versions in installation commands are +examples, with the engine and plugins kept at the same version. When preparing the +HugeGraph 1.8.0 documentation snapshot, pin its SeaTunnel references and installation +version to 3.0.0 in the release branch; keep the `latest` guides tracking upstream. + +latest 文档使用 SeaTunnel 3.0+ 等最低版本说明，并引用上游 latest/Next 文档；安装命令中的具体版本仅作为示例，引擎和插件保持同版。 +准备 HugeGraph 1.8.0 文档快照时，在发布分支中将 SeaTunnel 文档引用和安装版本固定为 3.0.0，latest 文档继续跟随上游。 + ## Repository structure - `content/en/` and `content/cn/` contain the bilingual source pages. diff --git a/dist/validate-site-output.py b/dist/validate-site-output.py index 4b9894037c..66f3852e55 100644 --- a/dist/validate-site-output.py +++ b/dist/validate-site-output.py @@ -19,6 +19,8 @@ from __future__ import annotations import argparse +from concurrent.futures import ProcessPoolExecutor +from functools import partial import html.parser import json import pathlib @@ -1091,6 +1093,37 @@ def document_url_shape_errors( return errors +def parse_and_check_page_security( + page: pathlib.Path, + root: pathlib.Path, + base_parts: urllib.parse.SplitResult, + error_paths: set[str], +) -> tuple[DocumentParser | None, list[str]]: + parser = DocumentParser() + try: + parser.feed(page.read_text(encoding="utf-8")) + except (OSError, UnicodeError) as exc: + return None, [f"cannot parse {page.relative_to(root)}: {exc}"] + + page_name = page.relative_to(root).as_posix() + errors = document_url_shape_errors(parser, page_name) + if errors: + return None, errors + errors.extend(document_security_errors(parser, page_name, base_parts)) + errors.extend(error_document_seo_errors(parser, page_name, error_paths)) + return parser, errors + + +def page_security_errors( + page: pathlib.Path, + root: pathlib.Path, + base_parts: urllib.parse.SplitResult, + error_paths: set[str], +) -> list[str]: + # Keep the parsed document local to the worker instead of serializing it. + return parse_and_check_page_security(page, root, base_parts, error_paths)[1] + + def main() -> int: argument_parser = argparse.ArgumentParser( description="Validate a generated HugeGraph documentation artifact." @@ -1105,6 +1138,13 @@ def main() -> int: "boundaries only; versioning.py validates isolated URL contracts." ), ) + argument_parser.add_argument( + "--workers", + type=int, + choices=(1, 2), + default=1, + help="Number of HTML security-check processes in --security-only mode.", + ) args = argument_parser.parse_args() root = args.public_dir.resolve() @@ -1148,23 +1188,28 @@ def main() -> int: f"route leaked into search: {ref}" ) - for page in html_files: - parser = DocumentParser() - try: - parser.feed(page.read_text(encoding="utf-8")) - except (OSError, UnicodeError) as exc: - errors.append(f"cannot parse {page.relative_to(root)}: {exc}") + if args.security_only: + check_page = partial( + page_security_errors, root=root, base_parts=base_parts, + error_paths=error_paths, + ) + if args.workers == 2 and html_files: + with ProcessPoolExecutor(max_workers=args.workers) as executor: + # map preserves sorted page order, including diagnostics. + for page_errors in executor.map(check_page, html_files): + errors.extend(page_errors) + else: + for page_errors in map(check_page, html_files): + errors.extend(page_errors) + + for page in [] if args.security_only else html_files: + parser, page_errors = parse_and_check_page_security( + page, root, base_parts, error_paths + ) + errors.extend(page_errors) + if parser is None: continue - page_name = page.relative_to(root).as_posix() - shape_errors = document_url_shape_errors(parser, page_name) - errors.extend(shape_errors) - if shape_errors: - continue - errors.extend(document_security_errors(parser, page_name, base_parts)) - errors.extend(error_document_seo_errors(parser, page_name, error_paths)) - if args.security_only: - continue errors.extend(toc_accessibility_errors(parser, page_name)) try: alias_target = refresh_target(parser) diff --git a/i18n/en.yaml b/i18n/en.yaml index 15ce3691b8..ea907fc6ed 100644 --- a/i18n/en.yaml +++ b/i18n/en.yaml @@ -24,6 +24,4 @@ ui_ai_consent_description: "Agree to send your questions to the third-party Kapa ui_ai_consent_continue: "Agree and open Ask AI" ui_ai_consent_cancel: "Not now" ui_ai_privacy: "Privacy policy" -ui_ai_revoke: "Withdraw AI consent" -ui_ai_revoke_error: "AI consent could not be removed and remains active. Clear this site's browser data to withdraw consent." ui_ai_community: "Ask the community on GitHub" diff --git a/i18n/zh-CN.yaml b/i18n/zh-CN.yaml index a194fb3f6b..3a043bf19c 100644 --- a/i18n/zh-CN.yaml +++ b/i18n/zh-CN.yaml @@ -25,6 +25,4 @@ ui_ai_consent_description: "同意后将把你的问题发送给第三方 Kapa ui_ai_consent_continue: "同意并打开 Ask AI" ui_ai_consent_cancel: "暂不使用" ui_ai_privacy: "隐私政策" -ui_ai_revoke: "撤回 AI 授权" -ui_ai_revoke_error: "未能撤回 AI 授权，授权仍有效。请清除此网站的浏览器数据以撤回授权。" ui_ai_community: "前往 GitHub 向社区提问" diff --git a/layouts/_partials/footer.html b/layouts/_partials/footer.html index 83eff243f7..94fe3c0e33 100644 --- a/layouts/_partials/footer.html +++ b/layouts/_partials/footer.html @@ -34,9 +34,6 @@

Copyright © {{ now.Year }} The Apache Software Foundation, Licensed under the Apache License Version 2.0

Apache, the names of Apache projects, and the feather logo are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries.

- {{- if (partial "ai/config.html" .).enabled }} - - {{- end }}
diff --git a/layouts/_partials/hooks/body-end.html b/layouts/_partials/hooks/body-end.html index d30e99e603..82e672baa6 100644 --- a/layouts/_partials/hooks/body-end.html +++ b/layouts/_partials/hooks/body-end.html @@ -25,7 +25,6 @@ "description" (T "ui_ask_ai_description") "latest" (T "ui_ask_ai_latest") "error" (T "ui_ai_error") - "revokeError" (T "ui_ai_revoke_error") "community" (T "ui_ai_community") -}} {{- $questions := slice -}} @@ -40,7 +39,7 @@ "historical" $historical "labels" $labels "exampleQuestions" $questions - "communityURL" "https://github.com/apache/hugegraph/discussions" + "communityURL" "https://github.com/apache/hugegraph/issues" -}}