From 7dc848ec28dc14b775e24097a7c219b3913d02d2 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 16 Sep 2026 15:23:28 +0200 Subject: [PATCH 1/6] Add --agent flag to test runners for low-noise output A green run of the deno pretty reporter scales with test count and becomes expensive to drive from an AI coding session - 165 passing tests cost roughly 52KB and 14000 input tokens for a run whose only signal is the final tally line. --reporter=dot collapses that to roughly two bytes per test while preserving what a failure needs: assertion message, source frame, stack, and the harness-assembled rerun command all survive, since they're folded into the thrown failure message rather than captured stdout. Captured console output (progress prints outside the assertion message) is dropped for every test; the few places that was the only diagnostic are tracked separately and don't block this. The flag is explicit opt-in only - no environment sniffing, no TTY detection - so a human and an agent running the same command see the same output unless --agent is passed. --- tests/run-tests-agent-flag.test.sh | 282 +++++++++++++++++++++++++++++ tests/run-tests.ps1 | 8 +- tests/run-tests.sh | 17 +- 3 files changed, 302 insertions(+), 5 deletions(-) create mode 100644 tests/run-tests-agent-flag.test.sh diff --git a/tests/run-tests-agent-flag.test.sh b/tests/run-tests-agent-flag.test.sh new file mode 100644 index 00000000000..3f44d287547 --- /dev/null +++ b/tests/run-tests-agent-flag.test.sh @@ -0,0 +1,282 @@ +#!/usr/bin/env bash +# Regression test for the --agent flag on run-tests.sh: verifies the +# --reporter=dot token is inserted exactly once, in the correct position, +# composes with --keep-outputs and smoke-all routing, is stripped before +# the file-type classifier, participates correctly in the (bash-only) +# reporter-collision case with QUARTO_DENO_EXTRA_OPTIONS, and that +# --keep-outputs artifact gating is unaffected by --agent. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." >/dev/null 2>&1 && pwd)" + +# run-tests.sh derives its own architecture directory from $DENO_DIR (see +# "Architecture detection" in run-tests.sh) instead of computing it itself, +# so any REAL (non-stub) invocation below needs DENO_DIR pointed at the arch +# dir name ahead of time, or DENO_ARCH_DIR is empty there and it resolves an +# empty "tools//deno" path. Source the same utils.sh run-tests.sh sources, +# purely to obtain DENO_ARCH_DIR for that - this is not a cache setting, +# don't delete it. +# shellcheck source=/dev/null +source "$REPO_ROOT/package/scripts/common/utils.sh" # provides DENO_ARCH_DIR + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +fail() { + echo "FAIL: $1" + exit 1 +} + +# --- Isolated fake repo skeleton, just enough for run-tests.sh to run --- +mkdir -p "$TMP/tests/docs/smoke-all" +mkdir -p "$TMP/package/scripts/common" +mkdir -p "$TMP/package/dist/bin/tools/stub" +mkdir -p "$TMP/src/resources" + +tr -d '\r' < "$REPO_ROOT/tests/run-tests.sh" > "$TMP/tests/run-tests.sh" +tr -d '\r' < "$REPO_ROOT/package/scripts/common/utils.sh" > "$TMP/package/scripts/common/utils.sh" +chmod +x "$TMP/tests/run-tests.sh" + +RECORD_FILE="$TMP/recorded-argv.txt" +cat > "$TMP/package/dist/bin/tools/stub/deno" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' "$@" > "$RECORD_FILE" +exit 0 +EOF +chmod +x "$TMP/package/dist/bin/tools/stub/deno" + +# docs/smoke-all/ lives INSIDE tests/, not as a sibling - run-tests.sh's +# timing mode does `find docs/smoke-all/` relative to its own cwd (tests/). +: > "$TMP/tests/docs/smoke-all/fixture.qmd" + +run_stub() { + # Runs the fake run-tests.sh with the given test-runner args (NOT + # including "./run-tests.sh" itself), populates $RECORD_FILE. + ( cd "$TMP/tests" && \ + QUARTO_TESTS_NO_CONFIG=true \ + QUARTO_TESTS_FORCE_NO_VENV=true \ + DENO_DIR=stub \ + RECORD_FILE="$RECORD_FILE" \ + ./run-tests.sh "$@" ) +} + +# --- Expectation 4: no flag -> no reporter token at all --- +rm -f "$RECORD_FILE" +run_stub "foo.test.ts" >/dev/null 2>&1 +[ -s "$RECORD_FILE" ] || fail "no-flag invocation never reached the stub deno (RECORD_FILE missing or empty)" +mapfile -t recorded < "$RECORD_FILE" +for tok in "${recorded[@]}"; do + [[ "$tok" == --reporter=* ]] && fail "no-flag run recorded a reporter token: ${recorded[*]}" +done + +# --- Expectation 3 + 5: --agent -> exactly one --reporter=dot, no -q, no --quiet, no --agent leak --- +rm -f "$RECORD_FILE" +run_stub "--agent" "foo.test.ts" >/dev/null 2>&1 +[ -s "$RECORD_FILE" ] || fail "--agent foo.test.ts invocation never reached the stub deno (RECORD_FILE missing or empty)" +mapfile -t recorded < "$RECORD_FILE" +reporter_count=0 +import_map_index=-1 +reporter_index=-1 +foo_index=-1 +for i in "${!recorded[@]}"; do + tok="${recorded[$i]}" + [[ "$tok" == --importmap=* ]] && import_map_index=$i + [[ "$tok" == "--reporter=dot" ]] && { reporter_count=$((reporter_count + 1)); reporter_index=$i; } + [[ "$tok" == "-q" || "$tok" == "--quiet" ]] && fail "-q/--quiet leaked into argv: ${recorded[*]}" + [[ "$tok" == "--agent" ]] && fail "--agent itself leaked into deno argv: ${recorded[*]}" + [[ "$tok" == "foo.test.ts" ]] && foo_index=$i +done +[ "$reporter_count" -eq 1 ] || fail "expected exactly one --reporter=dot token, got $reporter_count (argv: ${recorded[*]})" +[ "$import_map_index" -ge 0 ] || fail "import-map token not found in argv" +[ "$reporter_index" -eq $((import_map_index + 1)) ] || fail "--reporter=dot not immediately after import-map arg (import-map at $import_map_index, reporter at $reporter_index)" +[ "$foo_index" -gt "$reporter_index" ] || fail "target foo.test.ts did not come after --reporter=dot" +echo "PASS: --agent composes to exactly one --reporter=dot, positioned after import-map, before target; no -q/--quiet/--agent leak" + +# --agent first, last, middle (two targets), and combined with --keep-outputs +for arg_order in "--agent foo.test.ts" "foo.test.ts --agent" "foo.test.ts --agent bar.test.ts" "--agent --keep-outputs foo.test.ts" "foo.test.ts --keep-outputs --agent"; do + rm -f "$RECORD_FILE" + # shellcheck disable=SC2086 + run_stub $arg_order >/dev/null 2>&1 + [ -s "$RECORD_FILE" ] || fail "order '$arg_order' never reached the stub deno (RECORD_FILE missing or empty)" + mapfile -t recorded < "$RECORD_FILE" + found_reporter=0 + found_foo=0 + for tok in "${recorded[@]}"; do + [ "$tok" = "--reporter=dot" ] && found_reporter=1 + [ "$tok" = "--keep-outputs" ] && fail "--keep-outputs reached the deno classifier for order '$arg_order' (argv: ${recorded[*]})" + [ "$tok" = "--agent" ] && fail "--agent reached the deno classifier for order '$arg_order' (argv: ${recorded[*]})" + [ "$tok" = "foo.test.ts" ] && found_foo=1 + done + [ "$found_reporter" -eq 1 ] || fail "--reporter=dot missing for order '$arg_order'" + [ "$found_foo" -eq 1 ] || fail "foo.test.ts missing for order '$arg_order'" + if [[ "$arg_order" == *bar.test.ts* ]]; then + found_bar=0 + for tok in "${recorded[@]}"; do [ "$tok" = "bar.test.ts" ] && found_bar=1; done + [ "$found_bar" -eq 1 ] || fail "bar.test.ts missing for two-target middle-position order '$arg_order' (argv: ${recorded[*]})" + fi +done +echo "PASS: --agent works in first/middle/last position, with a second target, and composes with --keep-outputs without either reaching the classifier" + +# smoke-all qmd routing with --agent: reporter token must land before the "--" +# separator, and the fixture document itself must land after it +rm -f "$RECORD_FILE" +run_stub "--agent" "docs/smoke-all/fixture.qmd" >/dev/null 2>&1 +[ -s "$RECORD_FILE" ] || fail "smoke-all qmd routing invocation never reached the stub deno (RECORD_FILE missing or empty)" +mapfile -t recorded < "$RECORD_FILE" +dashdash_index=-1 +reporter_index=-1 +smokeall_index=-1 +fixture_index=-1 +for i in "${!recorded[@]}"; do + [ "${recorded[$i]}" = "--" ] && dashdash_index=$i + [ "${recorded[$i]}" = "--reporter=dot" ] && reporter_index=$i + [[ "${recorded[$i]}" == *smoke-all.test.ts ]] && smokeall_index=$i + [[ "${recorded[$i]}" == *fixture.qmd ]] && fixture_index=$i +done +[ "$dashdash_index" -ge 0 ] || fail "smoke-all routing: no '--' separator found (argv: ${recorded[*]})" +[ "$smokeall_index" -ge 0 ] || fail "smoke-all routing: smoke-all.test.ts not routed in (argv: ${recorded[*]})" +[ "$reporter_index" -ge 0 ] || fail "smoke-all routing: --reporter=dot missing (argv: ${recorded[*]})" +[ "$fixture_index" -ge 0 ] || fail "smoke-all routing: fixture.qmd missing from argv entirely (argv: ${recorded[*]})" +[ "$reporter_index" -lt "$dashdash_index" ] || fail "smoke-all routing: --reporter=dot landed after '--' separator (argv: ${recorded[*]})" +[ "$fixture_index" -gt "$dashdash_index" ] || fail "smoke-all routing: fixture.qmd did not land after '--' separator (argv: ${recorded[*]})" +echo "PASS: --agent + qmd routes to smoke-all.test.ts with '--' and the fixture document intact after the separator, reporter token before it" + +# --- Expectation 7: timing mode carries the reporter in both bash timing +# invocations. /usr/bin/time does not exist in Git Bash on Windows, and +# run-tests.sh's timing branches invoke it directly (not through a +# resolvable PATH lookup we could stub), so bash fails to exec before ever +# reaching the stub deno on that host. Guard on its presence rather than +# weakening the assertions: a declared skip is honest, a vacuous pass from a +# missing record file is not. +if [ -x /usr/bin/time ]; then + # Timing, per-file. Delete RECORD_FILE first and require it be recreated - + # reusing a stale file from an earlier step would let this pass even if the + # timing branch never reached the stub deno at all. Assert token order, not + # just presence, matching the rigor of the generic-path check above. + rm -f "$RECORD_FILE" + ( cd "$TMP/tests" && QUARTO_TESTS_NO_CONFIG=true QUARTO_TESTS_FORCE_NO_VENV=true DENO_DIR=stub QUARTO_TEST_TIMING=true RECORD_FILE="$RECORD_FILE" ./run-tests.sh "--agent" "foo.test.ts" >/dev/null 2>&1 ) + [ -s "$RECORD_FILE" ] || fail "timing per-file invocation never reached the stub deno (RECORD_FILE missing or empty)" + mapfile -t recorded < "$RECORD_FILE" + import_map_index=-1 + reporter_index=-1 + foo_index=-1 + for i in "${!recorded[@]}"; do + tok="${recorded[$i]}" + [[ "$tok" == --importmap=* ]] && import_map_index=$i + [ "$tok" = "--reporter=dot" ] && reporter_index=$i + [ "$tok" = "foo.test.ts" ] && foo_index=$i + done + [ "$import_map_index" -ge 0 ] || fail "timing per-file: import-map token not found (argv: ${recorded[*]})" + [ "$reporter_index" -eq $((import_map_index + 1)) ] || fail "timing per-file: --reporter=dot not immediately after import-map arg (argv: ${recorded[*]})" + [ "$foo_index" -gt "$reporter_index" ] || fail "timing per-file: target did not come after --reporter=dot (argv: ${recorded[*]})" + + # Timing, smoke-all per-document. Same freshness + position rigor, and check + # the fixture document lands after the "--" separator, not just that a + # reporter token exists somewhere in the recorded argv. + rm -f "$RECORD_FILE" + ( cd "$TMP/tests" && QUARTO_TESTS_NO_CONFIG=true QUARTO_TESTS_FORCE_NO_VENV=true DENO_DIR=stub QUARTO_TEST_TIMING=true RECORD_FILE="$RECORD_FILE" ./run-tests.sh "--agent" "./smoke/smoke-all.test.ts" >/dev/null 2>&1 ) + [ -s "$RECORD_FILE" ] || fail "timing smoke-all invocation never reached the stub deno (RECORD_FILE missing or empty)" + mapfile -t recorded < "$RECORD_FILE" + import_map_index=-1 + reporter_index=-1 + dashdash_index=-1 + fixture_index=-1 + for i in "${!recorded[@]}"; do + tok="${recorded[$i]}" + [[ "$tok" == --importmap=* ]] && import_map_index=$i + [ "$tok" = "--reporter=dot" ] && reporter_index=$i + [ "$tok" = "--" ] && dashdash_index=$i + [[ "$tok" == *fixture.qmd ]] && fixture_index=$i + done + [ "$import_map_index" -ge 0 ] || fail "timing smoke-all: import-map token not found (argv: ${recorded[*]})" + [ "$reporter_index" -eq $((import_map_index + 1)) ] || fail "timing smoke-all: --reporter=dot not immediately after import-map arg (argv: ${recorded[*]})" + [ "$dashdash_index" -gt "$reporter_index" ] || fail "timing smoke-all: '--' separator missing or not after --reporter=dot (argv: ${recorded[*]})" + [ "$fixture_index" -gt "$dashdash_index" ] || fail "timing smoke-all: fixture.qmd did not land after '--' separator (argv: ${recorded[*]})" + echo "PASS: both bash timing invocations freshly reached the stub deno with --reporter=dot in the correct position" +else + echo "SKIP: timing-mode expectations - /usr/bin/time not available on this host (run-tests.sh timing branches invoke it directly)" +fi + +# --- Expectation 8 (argv half): collision records both reporter tokens --- +rm -f "$RECORD_FILE" +( cd "$TMP/tests" && QUARTO_TESTS_NO_CONFIG=true QUARTO_TESTS_FORCE_NO_VENV=true DENO_DIR=stub QUARTO_DENO_EXTRA_OPTIONS="--reporter=tap" RECORD_FILE="$RECORD_FILE" ./run-tests.sh "--agent" "foo.test.ts" >/dev/null 2>&1 ) +[ -s "$RECORD_FILE" ] || fail "collision (stub) invocation never reached the stub deno (RECORD_FILE missing or empty)" +mapfile -t recorded < "$RECORD_FILE" +found_dot=0 +found_tap=0 +for tok in "${recorded[@]}"; do + [ "$tok" = "--reporter=dot" ] && found_dot=1 + [ "$tok" = "--reporter=tap" ] && found_tap=1 +done +[ "$found_dot" -eq 1 ] && [ "$found_tap" -eq 1 ] || fail "collision case did not record both reporter tokens (argv: ${recorded[*]})" +echo "PASS: --agent + QUARTO_DENO_EXTRA_OPTIONS reporter records both tokens in argv (stub, construction proof)" + +# --- Expectation 8 (behavioral half): real deno rejects the duplicate +# reporter. Bypasses the wrapper entirely, per the design's split (a stub +# can prove construction but not deno's own exit code/diagnostic). +DENO_BIN="" +for candidate in "$REPO_ROOT"/package/dist/bin/tools/*/deno "$REPO_ROOT"/package/dist/bin/tools/*/deno.exe; do + [ -f "$candidate" ] && { DENO_BIN="$candidate"; break; } +done +[ -n "$DENO_BIN" ] || fail "could not find a built deno binary under package/dist/bin/tools/*/deno[.exe] - run quarto-bld configure first" + +cat > "$TMP/throwaway-collision.test.ts" <<'EOF' +Deno.test("throwaway", () => {}); +EOF + +collision_output="$("$DENO_BIN" test --reporter=dot --reporter=tap "$TMP/throwaway-collision.test.ts" 2>&1)" +collision_exit=$? +[ "$collision_exit" -ne 0 ] || fail "expected non-zero exit from duplicate --reporter, got 0" +# Two substrings rather than the full sentence: tolerant of minor wording +# drift across deno versions, but specific enough that an unrelated stderr +# message merely mentioning "reporter" (e.g. a deprecation notice) can't +# satisfy it. +echo "$collision_output" | grep -qi "reporter" || fail "expected a reporter-related diagnostic, got: $collision_output" +echo "$collision_output" | grep -qi "multiple times" || fail "expected the duplicate-reporter diagnostic to mention 'multiple times', got: $collision_output" +echo "PASS: real deno 2.7.14 exits non-zero with the duplicate-reporter diagnostic on duplicate --reporter tokens" + +# The two checks above prove deno's own parser rejects a duplicate --reporter, +# and (the argv half above) prove the wrapper constructs both tokens into +# argv - but neither proves the WRAPPER itself surfaces the diagnostic and +# propagates deno's non-zero exit end-to-end. Confirm that directly, through +# the real (non-stub) wrapper, since exit-code passthrough on a +# CLI-usage-error path is a different code path than the "test assertion +# failed" case already checked elsewhere. DENO_DIR must be set to the real +# arch dir here (see the top-of-script comment) or the wrapper resolves an +# empty tools//deno path. +collision_wrapper_output="$( cd "$REPO_ROOT/tests" && QUARTO_TESTS_NO_CONFIG=true QUARTO_TESTS_FORCE_NO_VENV=true DENO_DIR="$DENO_ARCH_DIR" QUARTO_DENO_EXTRA_OPTIONS="--reporter=tap" ./run-tests.sh --agent unit/filter-paths.test.ts 2>&1 )" +collision_wrapper_exit=$? +[ "$collision_wrapper_exit" -ne 0 ] || fail "expected run-tests.sh to exit non-zero when --agent collides with QUARTO_DENO_EXTRA_OPTIONS, got 0" +echo "$collision_wrapper_output" | grep -qi "multiple times" || fail "expected the wrapper's own output to surface deno's duplicate-reporter diagnostic, got: $collision_wrapper_output" +echo "PASS: the real (non-stub) wrapper surfaces the duplicate-reporter diagnostic and propagates deno's non-zero exit" + +# --- Expectation 9: keep-outputs paired runs, real render, identical with +# and without --agent. Runs against the real repo's tests/run-tests.sh (not +# the fake skeleton) and the tests/smoke/render/render-minimal.test.ts +# fixture, whose artifact is tests/docs/minimal.html (single file, no +# support dir - render-minimal.test.ts asserts noSupporting: true). +# Requires a built distribution. +ARTIFACT="$REPO_ROOT/tests/docs/minimal.html" +trap 'rm -f "$ARTIFACT"; rm -rf "$TMP"' EXIT # replaces the earlier bare TMP-only trap - also covers Ctrl-C mid-render + +run_and_check() { + local expect_exists="$1"; shift + rm -f "$ARTIFACT" + local status + ( cd "$REPO_ROOT/tests" && QUARTO_TESTS_NO_CONFIG=true QUARTO_TESTS_FORCE_NO_VENV=true DENO_DIR="$DENO_ARCH_DIR" ./run-tests.sh "$@" smoke/render/render-minimal.test.ts >/dev/null 2>&1 ) + status=$? + [ "$status" -eq 0 ] || fail "run-tests.sh exited $status for args: $* (a render failure would make either outcome meaningless)" + if [ "$expect_exists" = "yes" ]; then + [ -f "$ARTIFACT" ] || fail "expected $ARTIFACT to survive (args: $*)" + else + [ ! -f "$ARTIFACT" ] || fail "expected $ARTIFACT to be removed (args: $*)" + fi + rm -f "$ARTIFACT" +} + +run_and_check no # no flags: removed +run_and_check yes --keep-outputs # keep-outputs alone: survives +run_and_check no --agent # agent alone: removed (same as no flags) +run_and_check yes --agent --keep-outputs # both: survives, identically to keep-outputs alone +echo "PASS: keep-outputs artifact gating is removed/retained identically with and without --agent" diff --git a/tests/run-tests.ps1 b/tests/run-tests.ps1 index f0f3ef77f7c..b9a4f9dac24 100644 --- a/tests/run-tests.ps1 +++ b/tests/run-tests.ps1 @@ -106,12 +106,15 @@ if ( $MyInvocation.Line -eq "" ) { $customArgs = $argList ? @(Invoke-Expression "Write-Output -- $argList") : @() } -# Check if keep-outputs mode is enabled and filter it from arguments +# Check if keep-outputs mode or agent mode is enabled and filter it from arguments $KEEP_OUTPUTS = $false +$AGENT_MODE = $false $FILTERED_CUSTOM_ARGS = @() foreach ($arg in $customArgs) { if ($arg -eq "--keep-outputs" -or $arg -eq "-k") { $KEEP_OUTPUTS = $true + } elseif ($arg -eq "--agent") { + $AGENT_MODE = $true } else { $FILTERED_CUSTOM_ARGS += $arg } @@ -171,6 +174,9 @@ If ($QUARTO_DENO_EXTRA_OPTIONS -ne $null) { $DENO_ARGS += -split $QUARTO_DENO_EXTRA_OPTIONS } $DENO_ARGS += -split $QUARTO_IMPORT_MAP_ARG +If ($AGENT_MODE) { + $DENO_ARGS += "--reporter=dot" +} $DENO_ARGS += $TESTS_TO_RUN # Activate python virtualenv diff --git a/tests/run-tests.sh b/tests/run-tests.sh index a2c134fd392..9967d98b2f8 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -15,14 +15,18 @@ if [[ "$RUNNER_DEBUG" == "1" ]] || [[ "$QUARTO_TEST_VERBOSE" == "true" ]]; then VERBOSE_MODE=true fi -# Check if keep-outputs mode is enabled +# Check if keep-outputs mode or agent mode is enabled KEEP_OUTPUTS=false +AGENT_MODE=false FILTERED_ARGS=() for arg in "$@"; do case $arg in --keep-outputs|-k) KEEP_OUTPUTS=true ;; + --agent) + AGENT_MODE=true + ;; *) FILTERED_ARGS+=("$arg") ;; @@ -35,6 +39,11 @@ if [[ "$KEEP_OUTPUTS" == "true" ]]; then echo "> Keep outputs mode enabled - test artifacts will not be deleted" fi +AGENT_REPORTER_ARGS=() +if [[ "$AGENT_MODE" == "true" ]]; then + AGENT_REPORTER_ARGS=(--reporter=dot) +fi + source $SCRIPT_PATH/../package/scripts/common/utils.sh export QUARTO_ROOT="$(cd "$SCRIPT_PATH/.." > /dev/null 2>&1 && pwd)" @@ -114,13 +123,13 @@ if [ "$QUARTO_TEST_TIMING" != "" ] && [ "$QUARTO_TEST_TIMING" != "false" ]; then SMOKE_ALL_FILES=`find docs/smoke-all/ -type f -regextype "posix-extended" -regex ".*/[^_][^/]*[.]qmd" -o -regex ".*/[^_][^/]*[.]md" -o -regex ".*/[^_][^/]*[.]ipynb"` for j in $SMOKE_ALL_FILES; do echo "${SMOKE_ALL_TEST_FILE} -- ${j}" >> "$QUARTO_TEST_TIMING" - /usr/bin/time -f " %e real %U user %S sys" -a -o ${QUARTO_TEST_TIMING} "${QUARTO_BIN_PATH}/tools/${DENO_ARCH_DIR}/deno" test ${QUARTO_DENO_OPTIONS} --no-check ${QUARTO_DENO_EXTRA_OPTIONS} "${QUARTO_IMPORT_MAP_ARG}" ${SMOKE_ALL_TEST_FILE} -- ${j} + /usr/bin/time -f " %e real %U user %S sys" -a -o ${QUARTO_TEST_TIMING} "${QUARTO_BIN_PATH}/tools/${DENO_ARCH_DIR}/deno" test ${QUARTO_DENO_OPTIONS} --no-check ${QUARTO_DENO_EXTRA_OPTIONS} "${QUARTO_IMPORT_MAP_ARG}" "${AGENT_REPORTER_ARGS[@]}" ${SMOKE_ALL_TEST_FILE} -- ${j} done continue fi # Otherwise we time the individual test.ts test echo $i >> "$QUARTO_TEST_TIMING" - /usr/bin/time -f " %e real %U user %S sys" -a -o "$QUARTO_TEST_TIMING" "${QUARTO_BIN_PATH}/tools/${DENO_ARCH_DIR}/deno" test ${QUARTO_DENO_OPTIONS} --no-check ${QUARTO_DENO_EXTRA_OPTIONS} "${QUARTO_IMPORT_MAP_ARG}" $i + /usr/bin/time -f " %e real %U user %S sys" -a -o "$QUARTO_TEST_TIMING" "${QUARTO_BIN_PATH}/tools/${DENO_ARCH_DIR}/deno" test ${QUARTO_DENO_OPTIONS} --no-check ${QUARTO_DENO_EXTRA_OPTIONS} "${QUARTO_IMPORT_MAP_ARG}" "${AGENT_REPORTER_ARGS[@]}" $i done # exit the script with an error code if the timing file shows error grep -q 'Command exited with non-zero status' $QUARTO_TEST_TIMING && SUCCESS=1 || SUCCESS=0 @@ -166,7 +175,7 @@ else # and smoke-all.test.ts expands it itself via expandGlobSync. Expanding it # here instead would depend on bash's own (non-recursive by default) glob # semantics and could silently drop deeply nested matches. - "${QUARTO_BIN_PATH}/tools/${DENO_ARCH_DIR}/deno" test ${QUARTO_DENO_OPTIONS} --check ${QUARTO_DENO_EXTRA_OPTIONS} "${QUARTO_IMPORT_MAP_ARG}" "${TESTS_TO_RUN[@]}" + "${QUARTO_BIN_PATH}/tools/${DENO_ARCH_DIR}/deno" test ${QUARTO_DENO_OPTIONS} --check ${QUARTO_DENO_EXTRA_OPTIONS} "${QUARTO_IMPORT_MAP_ARG}" "${AGENT_REPORTER_ARGS[@]}" "${TESTS_TO_RUN[@]}" SUCCESS=$? fi From 1298c2a9e37b98137027f1d7e9f99144e6d05bb8 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 16 Sep 2026 15:28:26 +0200 Subject: [PATCH 2/6] Document the --agent test runner flag Covers the flag itself, the failure-recovery fallback chain (harness- printed rerun command first, smoke-all document-name caveat, direct- registration files, then original-command-minus-flag as the universal fallback), and the bash-only reporter-collision caveat with QUARTO_DENO_EXTRA_OPTIONS. --- .claude/rules/testing/overview.md | 7 +++++++ tests/README.md | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/.claude/rules/testing/overview.md b/.claude/rules/testing/overview.md index bec82f921b2..f68ec115263 100644 --- a/.claude/rules/testing/overview.md +++ b/.claude/rules/testing/overview.md @@ -28,6 +28,13 @@ QUARTO_TESTS_NO_CONFIG="true" ./run-tests.sh test.ts # Linux/macOS $env:QUARTO_TESTS_NO_CONFIG=$true; .\run-tests.ps1 # Windows ``` +**Low-noise output for agent sessions:** +```bash +./run-tests.sh --agent unit/my-test.test.ts # Linux/macOS +.\run-tests.ps1 --agent unit/my-test.test.ts # Windows +``` +Collapses a green run to a dot per test plus a tally line; failures keep their assertion message, source frame, stack, and exit code. See `tests/README.md` for the fallback rerun workflow and the bash-only reporter-collision caveat. + ## Test Types | Type | Location | File Pattern | Details | diff --git a/tests/README.md b/tests/README.md index ef401eddaad..6aba5e16b74 100644 --- a/tests/README.md +++ b/tests/README.md @@ -224,6 +224,28 @@ $env:QUARTO_TEST_KEEP_OUTPUTS="true" ./run-tests.ps1 ``` +**--agent flag** +- Switches deno's reporter to `--reporter=dot`, collapsing a green run's output to roughly two bytes per test plus deno's tally line +- Failures keep what identifies them: assertion message, source frame, stack, exit code, and the harness-assembled rerun command (see below) all survive. What's dropped is captured console output (progress prints outside the assertion message) for every test, passing or failing - a handful of places relied on that as their only diagnostic (a snapshot mismatch, a few test files that print then assert generically); those are tracked as follow-up work, not blockers for this flag +- Explicit opt-in only - no environment or TTY detection, so a human and an agent running the same command see the same output unless this flag is passed +- Not a general "quiet" flag: deno's own `-q`/`--quiet` is never forwarded (it's a no-op under `--reporter=dot` and its semantics caused a prior log-marker suppression incident) + +```bash +./run-tests.sh --agent unit/my-test.test.ts +``` + +```powershell +./run-tests.ps1 --agent unit/my-test.test.ts +``` + +*Recovering from a failure under `--agent`:* the failure message the harness prints already contains a ready-to-run rerun command for most test failures (tests registered through `unitTest`/`testQuartoCmd`, failing inside the render/verify step). A few cases don't get that assembled command and need a fallback: +- A **smoke-all** document failure: the printed command reruns the whole `smoke-all.test.ts` corpus rather than just the failing document - use the document path shown in the test name instead. +- A few files register directly with `Deno.test` rather than through the harness (`smoke/create/create.test.ts`, `smoke/logging/log-level-and-formats.test.ts`, `integration/playwright-tests.test.ts`) - for these the reported source location is the actual test file, but deno prints it as `path/to/file.test.ts:line:column`; strip the `:line:column` suffix before passing it back to the runner, since the runner's file-type check only accepts a path ending in `.ts`/`.qmd`/`.md`/`.ipynb` and rejects the location as printed. +- For anything else (a setup/teardown failure, or any failure that isn't Error-shaped), rerun the original command with `--agent` removed - this always works and needs no output parsing. +- Do **not** use the location in the `FAILURES` summary section as a rerun target for harness-registered tests - it resolves to `test.ts`'s own `Deno.test` call site, which registers no tests of its own and would rerun nothing. + +*Reporter collision (bash only):* passing `--agent` together with a reporter already set via `QUARTO_DENO_EXTRA_OPTIONS` is unsupported - deno rejects duplicate `--reporter` arguments and exits non-zero. This combination is not detected or blocked by the wrapper; it surfaces as a deno error. (On PowerShell this combination cannot occur: `QUARTO_DENO_EXTRA_OPTIONS` isn't currently honored there at all.) + **Other environment variables** - `QUARTO_TEST_VERBOSE` - Enable verbose test output - `QUARTO_TESTS_NO_CHECK` - Not currently used (legacy variable) From 62fe9e8927c5534b3ae8c34dac465ab325ff7be2 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 16 Sep 2026 16:11:10 +0200 Subject: [PATCH 3/6] Trim unverifiable claims from --agent flag documentation Two clauses pointed readers at things they cannot reach: a prior log-marker suppression incident that has no artifact anywhere in this repository, and "tracked as follow-up work" naming a tracker other contributors cannot read. A rationale nobody can check is worse than none, because it reads as load-bearing while being unfollowable, so it tends to get preserved rather than questioned. The dropped-output bullet now names a concrete consequence instead - a snapshot mismatch loses its printed unified diff, while the .diff file is still written to disk - which a reader can confirm in verify.ts. That bullet was also carrying four sentences, so it is split to match the surrounding style. --- tests/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/README.md b/tests/README.md index 6aba5e16b74..938cebc7b2d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -226,9 +226,10 @@ $env:QUARTO_TEST_KEEP_OUTPUTS="true" **--agent flag** - Switches deno's reporter to `--reporter=dot`, collapsing a green run's output to roughly two bytes per test plus deno's tally line -- Failures keep what identifies them: assertion message, source frame, stack, exit code, and the harness-assembled rerun command (see below) all survive. What's dropped is captured console output (progress prints outside the assertion message) for every test, passing or failing - a handful of places relied on that as their only diagnostic (a snapshot mismatch, a few test files that print then assert generically); those are tracked as follow-up work, not blockers for this flag +- Failures keep what identifies them: assertion message, source frame, stack, exit code, and the harness-assembled rerun command (see below) all survive +- What's dropped is captured console output (progress prints outside the assertion message), for every test, passing or failing - a few places rely on that as their only diagnostic, notably a snapshot mismatch, whose unified diff is printed rather than thrown (the `.diff` file is still written to disk) - Explicit opt-in only - no environment or TTY detection, so a human and an agent running the same command see the same output unless this flag is passed -- Not a general "quiet" flag: deno's own `-q`/`--quiet` is never forwarded (it's a no-op under `--reporter=dot` and its semantics caused a prior log-marker suppression incident) +- Not a general "quiet" flag: deno's own `-q`/`--quiet` is never forwarded (it's a no-op under `--reporter=dot`) ```bash ./run-tests.sh --agent unit/my-test.test.ts From f503863257f6e3113a18269ffc882db3cb139d92 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 16 Sep 2026 16:11:39 +0200 Subject: [PATCH 4/6] Derive the deno version in the --agent regression script output The duplicate-reporter check printed a literal version string in its PASS line while asserting nothing at all about the version, so the next bundled deno bump would have left it confidently reporting a version that was never exercised. Read it from the binary under test instead, and fall back to a placeholder rather than an empty string if the probe fails. --- tests/run-tests-agent-flag.test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/run-tests-agent-flag.test.sh b/tests/run-tests-agent-flag.test.sh index 3f44d287547..23f828f617e 100644 --- a/tests/run-tests-agent-flag.test.sh +++ b/tests/run-tests-agent-flag.test.sh @@ -234,7 +234,10 @@ collision_exit=$? # satisfy it. echo "$collision_output" | grep -qi "reporter" || fail "expected a reporter-related diagnostic, got: $collision_output" echo "$collision_output" | grep -qi "multiple times" || fail "expected the duplicate-reporter diagnostic to mention 'multiple times', got: $collision_output" -echo "PASS: real deno 2.7.14 exits non-zero with the duplicate-reporter diagnostic on duplicate --reporter tokens" +# Report the version actually exercised. Nothing here asserts a version, so a +# literal one would silently misreport the moment the bundled deno is bumped. +deno_version="$("$DENO_BIN" --version 2>/dev/null | awk 'NR==1{print $2}')" +echo "PASS: real deno ${deno_version:-(version unknown)} exits non-zero with the duplicate-reporter diagnostic on duplicate --reporter tokens" # The two checks above prove deno's own parser rejects a duplicate --reporter, # and (the argv half above) prove the wrapper constructs both tokens into From 8b66310fe45c30b428c027943f883c5a2b0d743c Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 16 Sep 2026 16:31:18 +0200 Subject: [PATCH 5/6] Point testing rule docs at the --agent flag typescript-tests.md and smoke-all-tests.md predate the --agent flag and still showed plain run-tests examples; agents following those docs would miss the low-noise option documented in overview.md. --- .claude/rules/testing/smoke-all-tests.md | 2 ++ .claude/rules/testing/typescript-tests.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.claude/rules/testing/smoke-all-tests.md b/.claude/rules/testing/smoke-all-tests.md index 6863ae55c09..f6821a4dbc8 100644 --- a/.claude/rules/testing/smoke-all-tests.md +++ b/.claude/rules/testing/smoke-all-tests.md @@ -22,6 +22,8 @@ Document-based tests using YAML metadata for verification. Tests live in `tests/ .\run-tests.ps1 docs/smoke-all/path/to/test.qmd ``` +Add `--agent` for low-noise output — see `.claude/rules/testing/overview.md` § Running Tests. + ## Test Structure Tests are defined in `_quarto.tests` YAML metadata: diff --git a/.claude/rules/testing/typescript-tests.md b/.claude/rules/testing/typescript-tests.md index ea49883ba3d..29cf5945340 100644 --- a/.claude/rules/testing/typescript-tests.md +++ b/.claude/rules/testing/typescript-tests.md @@ -20,6 +20,8 @@ TypeScript-based tests using Deno. Smoke tests render documents; unit tests veri .\run-tests.ps1 smoke/render/render.test.ts ``` +Add `--agent` for low-noise output — see `.claude/rules/testing/overview.md` § Running Tests. + ## Core Infrastructure Core test files (`test.ts`, `verify.ts`, `utils.ts`) are described in `.claude/rules/testing/overview.md` § Core Files. From 8d86ab733fee56a3839d35ab6cf316d9ce7826e8 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 16 Sep 2026 18:30:48 +0200 Subject: [PATCH 6/6] Reorder testing rule to lead with --agent flag Rule doc is Claude's every-session lookup; --agent is the default invocation now, plain form and full flag list pushed to tests/README.md. --- .claude/rules/testing/overview.md | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/.claude/rules/testing/overview.md b/.claude/rules/testing/overview.md index f68ec115263..e8d3b182abe 100644 --- a/.claude/rules/testing/overview.md +++ b/.claude/rules/testing/overview.md @@ -9,31 +9,19 @@ Quarto's test suite lives in `tests/`. For comprehensive documentation, see `tes ## Running Tests +Use `--agent` — collapses a green run to a dot per test plus a tally line; failures keep assertion message, source frame, stack, exit code. + ```bash cd tests # Linux/macOS -./run-tests.sh # All tests -./run-tests.sh smoke/render/render.test.ts # Specific test -./run-tests.sh docs/smoke-all/path/test.qmd # Smoke-all document +QUARTO_TESTS_NO_CONFIG="true" ./run-tests.sh --agent unit/my-test.test.ts # Windows (PowerShell 7+) -.\run-tests.ps1 -.\run-tests.ps1 smoke/render/render.test.ts -``` - -**Skip dependency configuration:** -```bash -QUARTO_TESTS_NO_CONFIG="true" ./run-tests.sh test.ts # Linux/macOS -$env:QUARTO_TESTS_NO_CONFIG=$true; .\run-tests.ps1 # Windows +$env:QUARTO_TESTS_NO_CONFIG="true"; .\run-tests.ps1 --agent unit/my-test.test.ts ``` -**Low-noise output for agent sessions:** -```bash -./run-tests.sh --agent unit/my-test.test.ts # Linux/macOS -.\run-tests.ps1 --agent unit/my-test.test.ts # Windows -``` -Collapses a green run to a dot per test plus a tally line; failures keep their assertion message, source frame, stack, and exit code. See `tests/README.md` for the fallback rerun workflow and the bash-only reporter-collision caveat. +Plain form (no `--agent`), full flag list, rerun-on-failure workflow, bash-only reporter-collision caveat: `tests/README.md`. ## Test Types