From 48eb0053d60c9802dc9fc558607504d7606f2d8d Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Fri, 2 Oct 2026 08:02:53 -0700 Subject: [PATCH 1/6] Scan the pages a PR changes with axe in the preview workflow --- .github/workflows/preview.yml | 118 +++++++++++++++++++++++++++++++++- accessibility/site-report.qmd | 4 ++ 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index 52a5cc24a7..b5b7903fcd 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -56,6 +56,11 @@ jobs: contents: read statuses: write # for the Netlify deploy status pull-requests: write # so it can post the preview comment + outputs: + # Consumed by the `axe` job below, which runs after the deploy. + quarto_version: ${{ steps.github-release.outputs.version }} + axe_pages: ${{ steps.pr-comment.outputs.axe_pages }} + axe_full: ${{ steps.axe-full.outputs.any_changed }} steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -138,6 +143,18 @@ jobs: overwrites-pull-request-comment: false timeout-minutes: 1 + # The `axe` job scans this render instead of rendering again. One tar + # file uploads much faster than the thousands of files in _site. + - name: Pack the rendered site for the axe scan + run: tar -cf site.tar _site + + - name: Upload the rendered site + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: site + path: site.tar + retention-days: 1 + - name: Get changed files id: changed-files uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 @@ -158,7 +175,26 @@ jobs: docs/_*/** json: true escape_json: false - + + # A change to any of these can reach every page, so the axe job scans + # the whole site instead of the changed pages. + - name: Check for site-wide changes + id: axe-full + uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 + with: + base_sha: ${{ github.event.pull_request.base.sha || steps.pr-info.outputs.base_sha }} + files: | + _quarto*.yml + _brand.yml + _axe-baseline.json + **/*.scss + **/*.css + _extensions/** + docs/**/_metadata.yml + **/_*.md + **/_*.qmd + docs/**/_*/** + - name: Map changed images to doc pages id: image-pages env: @@ -183,6 +219,7 @@ jobs: shell: bash - name: Create custom PR comment + id: pr-comment if: github.event.pull_request || github.event.issue.pull_request uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: @@ -195,6 +232,8 @@ jobs: const prNumber = context.payload.pull_request?.number || context.payload.issue.number; const deployUrl = process.env.DEPLOY_URL; const changedFiles = JSON.parse(process.env.CHANGED_FILES || '[]'); + // Every page this PR touches, for the `axe` job's --pages + const axePages = new Set(); let commentBody = `## πŸ“ Preview Deployment\n\n`; commentBody += `πŸ” Full site preview: [${deployUrl}](${deployUrl})\n\n`; @@ -265,6 +304,7 @@ jobs: if (pageToFiles.size > 0) { commentBody += `### πŸ”„ Modified Documents\n\n`; for (const [page, files] of pageToFiles) { + axePages.add(page); const fileUrl = `${deployUrl}/${page}`; if (files.length === 1) { commentBody += `- [${page}](${fileUrl})\n`; @@ -286,12 +326,15 @@ jobs: if (pages.length > 0) { commentBody += `\n### πŸ–ΌοΈ Pages with Updated Screenshots\n\n`; for (const page of pages) { + axePages.add(page); const fileUrl = `${deployUrl}/${page}`; commentBody += `- [${page}](${fileUrl})\n`; } } } + core.setOutput('axe_pages', [...axePages].sort().join(',')); + await github.rest.issues.createComment({ issue_number: prNumber, owner: context.repo.owner, @@ -299,3 +342,76 @@ jobs: body: commentBody }); + axe: + # Scan the preview render with `quarto call axe`. Runs after the deploy, + # so the preview is never slower because of it. Scans only the pages the + # PR touches, or the whole site when a change can reach every page. + # See accessibility/site-report.qmd and .github/workflows/axe-report.yml. + name: Axe accessibility scan + needs: [build-deploy-preview] + if: > + needs.build-deploy-preview.outputs.axe_full == 'true' || + needs.build-deploy-preview.outputs.axe_pages != '' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + # The scanner reads _axe-baseline.json from the project root. + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: refs/pull/${{ github.event.pull_request.number || github.event.issue.number }}/merge + persist-credentials: false # nothing here pushes + + - name: Set up Quarto + uses: quarto-dev/quarto-actions/setup@v2 + with: + version: ${{ needs.build-deploy-preview.outputs.quarto_version }} + + - name: Download the rendered site + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: site + + - name: Unpack the rendered site + run: tar -xf site.tar + + - name: Scan with axe + id: scan + env: + AXE_FULL: ${{ needs.build-deploy-preview.outputs.axe_full }} + AXE_PAGES: ${{ needs.build-deploy-preview.outputs.axe_pages }} + # .github/axe-exclude.txt lists the pages to skip and why. The + # excludes apply after --pages, so a PR that edits one of them is + # not blocked. + run: | + AXE_EXCLUDE=$(grep -v -e '^#' -e '^[[:space:]]*$' .github/axe-exclude.txt | paste -sd, -) + set +e + if [ "$AXE_FULL" = "true" ]; then + echo "Site-wide change: scanning the whole site" + quarto call axe _site --exclude "$AXE_EXCLUDE" --fail-on serious + else + echo "Scanning changed pages: $AXE_PAGES" + quarto call axe _site --pages "$AXE_PAGES" --exclude "$AXE_EXCLUDE" --fail-on serious + fi + code=$? + set -e + echo "exit_code=$code" >> "$GITHUB_OUTPUT" + # 0: complete, nothing new at or above serious. + # 1: new serious or critical findings. Fix them, or accept them in + # _axe-baseline.json with a note that says why. + # 2: the scan did not complete, so it cannot pass. + exit "$code" + + - name: Write the report to the job summary + if: always() && steps.scan.outputs.exit_code != '' + run: cat _axe-checks/report.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload scan artifacts + if: always() && steps.scan.outputs.exit_code != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: axe-checks + path: _axe-checks/ + retention-days: 14 + diff --git a/accessibility/site-report.qmd b/accessibility/site-report.qmd index 8fa3bdcc38..47e6ea8e72 100644 --- a/accessibility/site-report.qmd +++ b/accessibility/site-report.qmd @@ -15,6 +15,10 @@ As one part of that commitment, this page shows the results of the [latest autom Every week, we run [`quarto call axe`](https://{{< prerelease-docs-url 1.11 >}}quarto.org/docs/websites/website-accessibility.html), which checks every page with [axe-core](https://github.com/dequelabs/axe-core). Each page is checked at a desktop and a phone viewport, and in light and dark mode. +Findings we know about and cannot fix in this site are recorded in [`_axe-baseline.json`](https://github.com/quarto-dev/quarto-web/blob/main/_axe-baseline.json). Each entry says why it is accepted, who owns the fix, and when it can be removed. Most are defects in third-party libraries that the documentation demonstrates, or defects in Quarto itself that a linked quarto-cli issue tracks. + +Pull requests to quarto.org run a smaller scan of the pages they change. The check fails if a pull request adds a new serious or critical finding. + ## Reporting a problem If you find an accessibility problem on [quarto.org](https://quarto.org), please [open an issue](https://github.com/quarto-dev/quarto-cli/issues/new). From 2d30ab6884ab6b39cdd15b22f6125bb5c75fe4d2 Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Fri, 2 Oct 2026 12:28:45 -0700 Subject: [PATCH 2/6] Address review of the axe PR check - Scan only the changed pages that rendered in _site - Read the exclude list from _axe-exclude.txt - Upload the site artifact after the PR comment, with overwrite: true - Skip the job summary if the scan wrote no report - Do not start a full-site scan for _axe-report.md --- .github/workflows/preview.yml | 53 ++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index b5b7903fcd..6f66e70a1b 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -143,18 +143,6 @@ jobs: overwrites-pull-request-comment: false timeout-minutes: 1 - # The `axe` job scans this render instead of rendering again. One tar - # file uploads much faster than the thousands of files in _site. - - name: Pack the rendered site for the axe scan - run: tar -cf site.tar _site - - - name: Upload the rendered site - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: site - path: site.tar - retention-days: 1 - - name: Get changed files id: changed-files uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 @@ -194,6 +182,9 @@ jobs: **/_*.md **/_*.qmd docs/**/_*/** + # The weekly report workflow commits this file, and release merges + # carry it. It changes one page only, so it needs no full scan. + files_ignore: _axe-report.md - name: Map changed images to doc pages id: image-pages @@ -342,9 +333,24 @@ jobs: body: commentBody }); + # The `axe` job scans this render instead of rendering again. One tar + # file uploads much faster than the thousands of files in _site. + # These steps come after the PR comment, so the preview links are not + # delayed. + - name: Pack the rendered site for the axe scan + run: tar -cf site.tar _site + + - name: Upload the rendered site + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: site + path: site.tar + retention-days: 1 + overwrite: true # a re-run uploads again + axe: - # Scan the preview render with `quarto call axe`. Runs after the deploy, - # so the preview is never slower because of it. Scans only the pages the + # Scan the preview render with `quarto call axe`. Runs after the deploy + # and the PR comment, so the preview is not delayed by it. Scans only the pages the # PR touches, or the whole site when a change can reach every page. # See accessibility/site-report.qmd and .github/workflows/axe-report.yml. name: Axe accessibility scan @@ -381,11 +387,22 @@ jobs: env: AXE_FULL: ${{ needs.build-deploy-preview.outputs.axe_full }} AXE_PAGES: ${{ needs.build-deploy-preview.outputs.axe_pages }} - # .github/axe-exclude.txt lists the pages to skip and why. The + # _axe-exclude.txt lists the pages to skip and why. The # excludes apply after --pages, so a PR that edits one of them is # not blocked. run: | - AXE_EXCLUDE=$(grep -v -e '^#' -e '^[[:space:]]*$' .github/axe-exclude.txt | paste -sd, -) + AXE_EXCLUDE=$(grep -v -e '^#' -e '^[[:space:]]*$' _axe-exclude.txt | paste -sd, -) + if [ "$AXE_FULL" != "true" ]; then + # Keep only the pages that rendered. A changed file can be a + # README, a file in an example sub-project, or a deleted page. + AXE_PAGES=$(tr ',' '\n' <<< "$AXE_PAGES" | while read -r page; do + if [ -f "_site/$page" ]; then echo "$page"; fi + done | paste -sd, -) + if [ -z "$AXE_PAGES" ]; then + echo "No changed page rendered: nothing to scan" + exit 0 + fi + fi set +e if [ "$AXE_FULL" = "true" ]; then echo "Site-wide change: scanning the whole site" @@ -405,7 +422,9 @@ jobs: - name: Write the report to the job summary if: always() && steps.scan.outputs.exit_code != '' - run: cat _axe-checks/report.md >> "$GITHUB_STEP_SUMMARY" + # A scan that exits 2 early may not write the report. + run: | + [ -f _axe-checks/report.md ] && cat _axe-checks/report.md >> "$GITHUB_STEP_SUMMARY" || true - name: Upload scan artifacts if: always() && steps.scan.outputs.exit_code != '' From c5bacf50af15008997a76429758ac916344b308c Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Fri, 2 Oct 2026 12:50:23 -0700 Subject: [PATCH 3/6] Leave the site report page unchanged --- accessibility/site-report.qmd | 4 ---- 1 file changed, 4 deletions(-) diff --git a/accessibility/site-report.qmd b/accessibility/site-report.qmd index 47e6ea8e72..8fa3bdcc38 100644 --- a/accessibility/site-report.qmd +++ b/accessibility/site-report.qmd @@ -15,10 +15,6 @@ As one part of that commitment, this page shows the results of the [latest autom Every week, we run [`quarto call axe`](https://{{< prerelease-docs-url 1.11 >}}quarto.org/docs/websites/website-accessibility.html), which checks every page with [axe-core](https://github.com/dequelabs/axe-core). Each page is checked at a desktop and a phone viewport, and in light and dark mode. -Findings we know about and cannot fix in this site are recorded in [`_axe-baseline.json`](https://github.com/quarto-dev/quarto-web/blob/main/_axe-baseline.json). Each entry says why it is accepted, who owns the fix, and when it can be removed. Most are defects in third-party libraries that the documentation demonstrates, or defects in Quarto itself that a linked quarto-cli issue tracks. - -Pull requests to quarto.org run a smaller scan of the pages they change. The check fails if a pull request adds a new serious or critical finding. - ## Reporting a problem If you find an accessibility problem on [quarto.org](https://quarto.org), please [open an issue](https://github.com/quarto-dev/quarto-cli/issues/new). From face9a4837932ce56067de3e0b1c439896222728 Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Fri, 2 Oct 2026 12:58:50 -0700 Subject: [PATCH 4/6] Test the axe PR check with an SCSS change --- theme.scss | 2 ++ 1 file changed, 2 insertions(+) diff --git a/theme.scss b/theme.scss index 8643ca64ea..57e9c6eae4 100644 --- a/theme.scss +++ b/theme.scss @@ -87,3 +87,5 @@ img.ui-icon { .quarto-title .quarto-categories .quarto-category { opacity: 1; } + +// test for axe PR check (#2264); do not merge From c8306320f722c9f0f29a57039fe59d9721e18b38 Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Mon, 5 Oct 2026 14:31:42 -0700 Subject: [PATCH 5/6] Comment on the PR with the axe scan scope and result --- .github/workflows/preview.yml | 70 +++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index 6f66e70a1b..dfa0e5c229 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -361,6 +361,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + pull-requests: write # so it can post the scan comment steps: # The scanner reads _axe-baseline.json from the project root. - name: Check out repository @@ -400,6 +401,7 @@ jobs: done | paste -sd, -) if [ -z "$AXE_PAGES" ]; then echo "No changed page rendered: nothing to scan" + echo "skipped=true" >> "$GITHUB_OUTPUT" exit 0 fi fi @@ -434,3 +436,71 @@ jobs: path: _axe-checks/ retention-days: 14 + # Like the preview comment, this posts a new comment on every run. + - name: Comment with the scan result + if: always() && (steps.scan.outputs.exit_code != '' || steps.scan.outputs.skipped == 'true') + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + EXIT_CODE: ${{ steps.scan.outputs.exit_code }} + SKIPPED: ${{ steps.scan.outputs.skipped }} + AXE_FULL: ${{ needs.build-deploy-preview.outputs.axe_full }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const prNumber = context.payload.pull_request?.number || context.payload.issue.number; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const full = process.env.AXE_FULL === 'true'; + const code = process.env.EXIT_CODE; + + let findings = null; + if (fs.existsSync('_axe-checks/findings.json')) { + findings = JSON.parse(fs.readFileSync('_axe-checks/findings.json', 'utf8')); + } + + let body = `## β™Ώ Accessibility Scan (axe)\n\n`; + + if (process.env.SKIPPED === 'true') { + body += `⏭️ No changed page rendered, so there was nothing to scan.\n`; + } else { + if (code === '0') { + body += `βœ… No new serious or critical findings.\n\n`; + } else if (code === '1') { + body += `❌ New serious or critical findings. Fix them, or accept them in \`_axe-baseline.json\` with a note that says why.\n\n`; + } else { + body += `⚠️ The scan did not complete (exit code ${code}). See the [run log](${runUrl}).\n\n`; + } + + if (findings) { + const pages = findings.pages.map(p => p.output); + const newFindings = findings.findings.filter(f => !f.baselined); + const baselined = findings.findings.length - newFindings.length; + const cells = findings.cells; + body += full + ? `**Scope:** whole site (site-wide change), ${pages.length} pages\n\n` + : `**Scope:** ${pages.length} changed page${pages.length === 1 ? '' : 's'}\n\n` + + pages.map(p => `- ${p}`).join('\n') + `\n\n`; + body += `**Result:** ${newFindings.length} new finding${newFindings.length === 1 ? '' : 's'} (any impact), ` + + `${baselined} baselined, ${cells.ok}/${cells.total} page Γ— mode Γ— viewport checks completed\n\n`; + + if (newFindings.length > 0) { + const max = 20; + body += `| rule | impact | pages | instances |\n|---|---|---:|---:|\n`; + for (const f of newFindings.slice(0, max)) { + body += `| [\`${f.rule}\`](${f.helpUrl}) | ${f.impact} | ${f.pages.length} | ${f.instances} |\n`; + } + if (newFindings.length > max) { + body += `\n…and ${newFindings.length - max} more.\n`; + } + body += `\n`; + } + } + body += `πŸ“„ [Full report](${runUrl}) in the job summary, and \`findings.json\` in the \`axe-checks\` artifact.\n`; + } + + await github.rest.issues.createComment({ + issue_number: prNumber, + owner: context.repo.owner, + repo: context.repo.repo, + body + }); From 0c95d8341bffd28f5d0bbd65e8315bbd89d100ad Mon Sep 17 00:00:00 2001 From: Charlotte Wickham Date: Mon, 5 Oct 2026 15:04:00 -0700 Subject: [PATCH 6/6] Add deliberate axe violations (test, do not merge) --- docs/authoring/callouts.qmd | 3 +++ theme.scss | 3 +++ 2 files changed, 6 insertions(+) diff --git a/docs/authoring/callouts.qmd b/docs/authoring/callouts.qmd index 3b0d790491..a838cc83c3 100644 --- a/docs/authoring/callouts.qmd +++ b/docs/authoring/callouts.qmd @@ -162,3 +162,6 @@ When the target format doesn't support callouts, they are rendered as a simple b ## Cross-References {{< include _cross-references-callouts.qmd >}} + + + diff --git a/theme.scss b/theme.scss index 57e9c6eae4..84c481dd21 100644 --- a/theme.scss +++ b/theme.scss @@ -89,3 +89,6 @@ img.ui-icon { } // test for axe PR check (#2264); do not merge + +// Deliberate low contrast for the axe PR check (#2264); do not merge +h1.title { color: #c8c8c8; }