From 4862cd063a4d94b0d6e49c7a83940a2a01f42361 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 6 Oct 2026 02:33:18 -0500 Subject: [PATCH 1/3] Include the fatal_error lines when the test step does not run The checker already greps those lines and only records that the pattern matched. They are written into the result file, not echoed, so a second pass does not match the checker itself. --- bot/check-test.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bot/check-test.sh b/bot/check-test.sh index 2731e754..d3a9d6db 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -219,6 +219,13 @@ comment_details_list=${comment_details_list}$(add_detail ${SLURM_OUTPUT_FOUND} 1 success_msg="no message matching ${GP_error}" failure_msg="found message matching ${GP_error}" +# The matched lines stay in the result file. Echoing them would write the +# pattern back into the Slurm output, and a second pass would match it again. +if [[ ${ERROR} -eq 1 && -n ${grep_out} ]]; then + error_quote=$(printf '%s\n' "${grep_out}" | head -n 20 | sed 's/&/\&/g; s//\>/g') + error_quote=${error_quote//$'\n'/
} + failure_msg="${failure_msg}
${error_quote}
" +fi comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") # Add an escape character to every *, for it to be printed correctly in the comment on GitHub From bbe828e3c7f0c850e7d129df144cffdff024da18 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 6 Oct 2026 02:52:03 -0500 Subject: [PATCH 2/3] Escape percent signs in the quoted test-step lines The ReFrame summary in this script doubles percent signs because the bot treats the comment as a format string. The quoted lines get the same treatment. --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index d3a9d6db..927f9b3e 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -222,7 +222,7 @@ failure_msg="found message matching ${GP_error}" # The matched lines stay in the result file. Echoing them would write the # pattern back into the Slurm output, and a second pass would match it again. if [[ ${ERROR} -eq 1 && -n ${grep_out} ]]; then - error_quote=$(printf '%s\n' "${grep_out}" | head -n 20 | sed 's/&/\&/g; s//\>/g') + error_quote=$(printf '%s\n' "${grep_out}" | head -n 20 | sed -e 's/&/\&/g' -e 's//\>/g' -e 's/\%/\%\%/g') error_quote=${error_quote//$'\n'/
} failure_msg="${failure_msg}
${error_quote}
" fi From 34aad50a70ec3e2b0a9dcace311570525770c4e9 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 6 Oct 2026 03:26:06 -0500 Subject: [PATCH 3/3] Keep the test step alive when ReFrame has no default module A missing default ReFrame module loads the newest versioned one. Import, version, list, and run failures include their own output. A failed test-suite clone exits with that output instead of continuing. --- run_tests.sh | 8 +++++++- test_suite.sh | 42 +++++++++++++++++++++++++----------------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 76558f72..6b24180a 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -24,7 +24,13 @@ LATEST_VERSION="VERSION=\$(git tag | grep '^v[0-9]\+\.[0-9]\+\.[0-9]\+$' | sort CHECKOUT_LATEST="git checkout \${VERSION}" # Git clone has to be run in compat layer, to make the git command available -./run_in_compat_layer_env.sh "${TEST_CLONE} && ${LATEST_VERSION} && ${CHECKOUT_LATEST}" +clone_err=$(./run_in_compat_layer_env.sh "${TEST_CLONE} && ${LATEST_VERSION} && ${CHECKOUT_LATEST}" 2>&1) +clone_rc=$? +printf '%s\n' "${clone_err}" +if [[ ${clone_rc} -ne 0 ]]; then + echo "ERROR: Failed to clone the EESSI test suite: ${clone_err}" >&2 + exit "${clone_rc}" +fi # Run the test suite ./test_suite.sh "$@" diff --git a/test_suite.sh b/test_suite.sh index be66eaa3..78a810e6 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -90,19 +90,24 @@ source $TOPDIR/init/bash # Reason is that the LMOD cache is normally only updated on the Stratum 0, once everything is ingested export LMOD_IGNORE_CACHE=1 -# Load the ReFrame module -# Currently, we load the default version. Maybe we should somehow make this configurable in the future? -module load ReFrame -if [[ $? -eq 0 ]]; then +# Load the ReFrame module. The default is preferred. A generation that +# ships ReFrame without a default still has a versioned module. +if module load ReFrame; then echo_green ">> Loaded ReFrame module" else - fatal_error "Failed to load the ReFrame module" + reframe_mod=$(module -t avail ReFrame 2>&1 | grep -E '^ReFrame/' | tail -n 1) + if [[ -n ${reframe_mod} ]] && module load "${reframe_mod}"; then + echo_green ">> Loaded ${reframe_mod}" + else + module avail ReFrame >&2 || true + fatal_error "Failed to load the ReFrame module" + fi fi # Check that a python3 executable is available python3_found=$(command -v python3) if [ -z ${python3_found} ]; then - fatal_error "No python3 executable found" + fatal_error "No python3 executable found in PATH=${PATH}" else echo_green "Executable python3 found:" python3 -V @@ -110,11 +115,11 @@ fi # Check that ReFrame can be imported reframe_import="reframe" -python3 -c "import ${reframe_import}" +import_err=$(python3 -c "import ${reframe_import}" 2>&1) if [[ $? -eq 0 ]]; then echo_green "Succesfully found and imported ${reframe_import}" else - fatal_error "Failed to import ${reframe_import}" + fatal_error "Failed to import ${reframe_import}: ${import_err}" fi # Cloning should already be done in run_tests.sh before test_suite.sh is invoked @@ -129,11 +134,11 @@ export PYTHONPATH=$TESTSUITEPREFIX:$PYTHONPATH # Check that we can import from the testsuite testsuite_import="eessi.testsuite" -python3 -c "import ${testsuite_import}" +suite_err=$(python3 -c "import ${testsuite_import}" 2>&1) if [[ $? -eq 0 ]]; then echo_green "Succesfully found and imported ${testsuite_import}" else - fatal_error "Failed to import ${testsuite_import}" + fatal_error "Failed to import ${testsuite_import}: ${suite_err}" fi # Configure ReFrame, see https://www.eessi.io/docs/test-suite/installation-configuration @@ -176,11 +181,11 @@ cat "${RFM_CONFIG_FILES}" export FI_PROVIDER="^psm3" # Check we can run reframe -reframe --version +version_err=$(reframe --version 2>&1) if [[ $? -eq 0 ]]; then echo_green "Succesfully ran 'reframe --version'" else - fatal_error "Failed to run 'reframe --version'" + fatal_error "Failed to run 'reframe --version': ${version_err}" fi # Check if the partition specified by RFM_SYSTEM is in the config file @@ -242,21 +247,24 @@ export REFRAME_ARGS="${REFRAME_CI_TAG} ${REFRAME_SCALE_TAG} ${REFRAME_ADDITIONAL # List the tests we want to run echo "Listing tests: reframe ${REFRAME_ARGS} --list" -reframe ${REFRAME_ARGS} --list -if [[ $? -eq 0 ]]; then +list_err=$(reframe ${REFRAME_ARGS} --list 2>&1) +list_rc=$? +printf '%s\n' "${list_err}" +if [[ ${list_rc} -eq 0 ]]; then echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list" else - fatal_error "Failed to list ReFrame tests with command: reframe ${REFRAME_ARGS} --list" + fatal_error "Failed to list ReFrame tests with command: reframe ${REFRAME_ARGS} --list: ${list_err}" fi # Run all tests echo "Running tests: reframe ${REFRAME_ARGS} --run" -reframe ${REFRAME_ARGS} --run +run_err=$(reframe ${REFRAME_ARGS} --run 2>&1) reframe_exit_code=$? +printf '%s\n' "${run_err}" if [[ ${reframe_exit_code} -eq 0 ]]; then echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run." else - fatal_error "ReFrame runtime failed to run with command: reframe ${REFRAME_ARGS} --run." + fatal_error "ReFrame runtime failed to run with command: reframe ${REFRAME_ARGS} --run: ${run_err}" fi echo ">> Cleaning up ${TMPDIR}..."