Audit the instructions in each build tarball against its CPU target

.github/workflows/test-eb-hooks-toolchains.yml

@@ -0,0 +1,33 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Unit tests for supported toolchains in EasyBuild hooks
+on:
+  push:
+    paths:
+      - eb_hooks.py
+      - eessi_supported_toolchains.toml
+      - tests/eb_hooks/**
+      - .github/workflows/test-eb-hooks-toolchains.yml
+  pull_request:
+    paths:
+      - eb_hooks.py
+      - eessi_supported_toolchains.toml
+      - tests/eb_hooks/**
+      - .github/workflows/test-eb-hooks-toolchains.yml
+  workflow_dispatch:
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  test_supported_toolchains:
+    runs-on: ubuntu-24.04
+    steps:
+        - name: Check out software-layer repository
+          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+        - name: Install EasyBuild and pytest
+          run: |
+              python3 -m venv "$RUNNER_TEMP/venv"
+              "$RUNNER_TEMP/venv/bin/pip" install easybuild pytest
+
+        - name: Run unit tests
+          run: |
+              "$RUNNER_TEMP/venv/bin/python" -m pytest -v tests/eb_hooks

.github/workflows/test-eb-hooks.yml → .github/workflows/test-eb-init-files.yml

@@ -1,5 +1,5 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
-name: Run checks on EasyBuild hooks script
+name: Run checks on EasyBuild hooks script and related init files
 on:
   push:
   pull_request:
@@ -26,30 +26,36 @@ jobs:
             eessi_stack_version: ${{matrix.EESSI_VERSION}}
             use_eessi_module: true
 
-        - name: Check whether eb_hooks.py script is up-to-date
+        - name: Check whether eb_hooks.py and related files are up-to-date
           if: ${{ github.event_name == 'pull_request' }}
           run: |
-              FILE="eb_hooks.py"
-              TEMP_FILE="$(mktemp)"
+              # Files that are installed in the same directory as the hooks, and have to be up-to-date
+              FILES=(eb_hooks.py eessi_supported_toolchains.toml)
 
               # Fetch base branch
               git fetch origin ${{ github.base_ref }}
 
-              # Check if the hooks has changed in the PR
-              if git diff --name-only origin/${{ github.base_ref }}...HEAD | grep -q "^$FILE$"; then
-                echo "Hooks changed in PR. Using PR version."
-                cp "$FILE" "$TEMP_FILE"
-              else
-                echo "File not changed in PR. Using default branch version."
-                git show origin/${{ github.base_ref }}:$FILE > "$TEMP_FILE"
-              fi
-
-              # replace <EESSI_VERSION> placeholder (as is also done in install_scripts.sh)
-              sed -i "s/<EESSI_VERSION>/${{matrix.EESSI_VERSION}}/g" "${TEMP_FILE}"
-
-              # Compare the hooks to what is shipped in the repository
               module load EESSI-extend
-              diff "$TEMP_FILE" "$EASYBUILD_HOOKS"
+
+              for FILE in "${FILES[@]}"; do
+                echo "Checking whether $FILE is up-to-date"
+                TEMP_FILE="$(mktemp)"
+
+                # Check if the file has changed in the PR
+                if git diff --name-only origin/${{ github.base_ref }}...HEAD | grep -q "^$FILE$"; then
+                  echo "$FILE changed in PR. Using PR version."
+                  cp "$FILE" "$TEMP_FILE"
+                else
+                  echo "$FILE not changed in PR. Using default branch version."
+                  git show origin/${{ github.base_ref }}:$FILE > "$TEMP_FILE"
+                fi
+
+                # replace <EESSI_VERSION> placeholder (as is also done in install_scripts.sh)
+                sed -i "s/<EESSI_VERSION>/${{matrix.EESSI_VERSION}}/g" "${TEMP_FILE}"
+
+                # Compare to what is shipped in the repository (all files are installed in the same directory as the hooks)
+                diff "$TEMP_FILE" "$(dirname "$EASYBUILD_HOOKS")/$FILE"
+              done
 
   check_eb_hooks_functionality:
     runs-on: ubuntu-24.04

.github/workflows/tests_isa_audit.yml

@@ -0,0 +1,36 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests for the tarball instruction audit
+on:
+  push:
+    paths:
+      - scripts/isa_audit/**
+      - tests/isa_audit/**
+      - .github/workflows/tests_isa_audit.yml
+  pull_request:
+    paths:
+      - scripts/isa_audit/**
+      - scripts/native_flags/references/**
+      - tests/isa_audit/**
+      - .github/workflows/tests_isa_audit.yml
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  test:
+    runs-on: ubuntu-24.04
+    steps:
+    - name: checkout
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+    - name: install compilers and decoders
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y gcc gcc-aarch64-linux-gnu llvm-20
+        # isa_audit_aarch64.py uses the LLVM 20 feature names (sve-aes and friends)
+        echo /usr/lib/llvm-20/bin >> "$GITHUB_PATH"
+        python3 -m venv "$RUNNER_TEMP/venv"
+        "$RUNNER_TEMP/venv/bin/pip" install iced-x86 pyelftools
+        echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH"
+
+    - name: test check_tarball.sh
+      run: |
+        ./tests/isa_audit/test_check_tarball.sh

EESSI-install-software.sh

@@ -234,6 +234,19 @@ else
     echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!"
 fi
 
+# Check that the compilers of all supported toolchains translate the native architecture flag into exactly the same
+# target flags on this build host as on other build hosts for the same CPU target, to ensure that binaries built on
+# different build hosts for the same CPU target are compatible
+if [[ ${GENERIC} -eq 1 ]]; then
+    echo ">> Skipping check of native compiler flags, since this is a generic build"
+elif [[ -n "${EESSI_SKIP_NATIVE_FLAGS_CHECK}" ]]; then
+    echo_yellow ">> Skipping check of native compiler flags, since \$EESSI_SKIP_NATIVE_FLAGS_CHECK is set"
+else
+    ${TOPDIR}/scripts/native_flags/check_native_flags.sh
+    check_exit_code $? ">> Native compiler flags on this build host are OK" \
+        "Native compiler flags check failed, refusing to build on this host (set \$EESSI_SKIP_NATIVE_FLAGS_CHECK to override)"
+fi
+
 # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
 export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
 # force Python's stdout and stderr streams to be unbuffered

bot/build.sh

@@ -379,4 +379,12 @@ echo "                     -- $software_layer_dir/create_tarball.sh ${TMP_IN_CON
 $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \
                      -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSIONS_SUBPATH} ${EESSI_VERSION}${EESSI_SOFTWARE_LAYER_VERSION_SUFFIX} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "$tarball_accelerators" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}
 
+# check that every instruction in the tarball is one the CPU target can run, against the same per-target
+# references as the native compiler flags check; a violation prints an 'ERROR: ' line, which check-build.sh reports
+if [[ -f ${TARBALL} ]]; then
+    $software_layer_dir/scripts/isa_audit/check_tarball.sh ${TARBALL} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \
+        $software_layer_dir/scripts/native_flags/references $software_layer_dir/scripts/isa_audit/dispatch-allow.txt \
+        2>&1 | tee -a ${tar_outerr}
+fi
+
 exit 0

eb_hooks.py

@@ -10,6 +10,14 @@
 import tarfile
 from typing import NamedTuple
 
+try:
+    import tomllib  # in the standard library since Python 3.11
+except ImportError:
+    try:
+        import tomli as tomllib  # backport for older Python versions
+    except ImportError:
+        tomllib = None
+
 import easybuild.tools.environment as env
 from easybuild.easyblocks.generic.configuremake import obtain_config_guess
 from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
@@ -67,31 +75,60 @@
 
 STACK_REPROD_SUBDIR = 'reprod'
 
-EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS = {
-    '2023.06': [
-        {'name': 'foss', 'version': '2022b'},
-        {'name': 'foss', 'version': '2023a'},
-        {'name': 'foss', 'version': '2023b'},
-    ],
-    '2025.06': [
-        {'name': 'foss', 'version': '2024a'},
-        {'name': 'foss', 'version': '2025a'},
-        {'name': 'foss', 'version': '2025b'},
-    ],
-    '2026.06': [
-        {'name': 'foss', 'version': '2026.1'},
-        {'name': 'lfoss', 'version': '2026.1'},
-    ],
-}
-if EASYBUILD_VERSION >= '5.2.0':
-    EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS['2025.06'].append(
-        {'name': 'lfoss', 'version': '2025b'}
-    )
+# Environment variable that can be used to point to a custom TOML file with the supported top-level toolchains
+SUPPORTED_TOOLCHAINS_FILE_ENVVAR = 'EESSI_SUPPORTED_TOOLCHAINS_FILE'
 
-if EASYBUILD_VERSION >= '5.3.1':
-    EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS['2025.06'].append(
-        {'name': 'rompi', 'version': '2025a'}
-    )
+
+def load_supported_top_level_toolchains():
+    """
+    Load the supported top-level toolchains per EESSI version from a TOML file.
+
+    The location of the TOML file can be set through the environment variable EESSI_SUPPORTED_TOOLCHAINS_FILE.
+    If that is not set, eessi_supported_toolchains.toml is expected next to this hooks file (both in the
+    software-layer-scripts repository, and when installed in <EESSI prefix>/init/easybuild).
+
+    Toolchains that require a more recent EasyBuild version than the one being used (as specified via
+    'min_easybuild_version') are left out.
+
+    Returns:
+        supported_toolchains (dict): maps each EESSI version to a list of dicts with the 'name' and 'version'
+            of a supported top-level toolchain
+    """
+    default_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'eessi_supported_toolchains.toml')
+    toolchains_file = os.getenv(SUPPORTED_TOOLCHAINS_FILE_ENVVAR)
+    envvar_msg = f" (set via ${SUPPORTED_TOOLCHAINS_FILE_ENVVAR})"
+    if not toolchains_file:
+        toolchains_file = default_file
+        envvar_msg = ""
+
+    if tomllib is None:
+        raise EasyBuildError("Parsing TOML files requires Python 3.11 or newer, or the 'tomli' Python package")
+
+    try:
+        with open(toolchains_file, 'rb') as fh:
+            toolchains = tomllib.load(fh)
+    except OSError as err:
+        msg = (f"Failed to read the file with supported toolchains {toolchains_file}{envvar_msg}: {err}. "
+               f"By default, it is expected next to the EasyBuild hooks file; its location can be configured "
+               f"through the environment variable {SUPPORTED_TOOLCHAINS_FILE_ENVVAR}.")
+        if envvar_msg and os.path.isfile(default_file):
+            msg += (f" Note that a file with supported toolchains does exist in the default location {default_file}. "
+                    f"If that is the file you intended to use, unset {SUPPORTED_TOOLCHAINS_FILE_ENVVAR}.")
+        raise EasyBuildError(msg)
+    except ValueError as err:
+        raise EasyBuildError(f"The file with supported toolchains {toolchains_file}{envvar_msg} "
+                             f"does not contain valid TOML: {err}")
+
+    return {
+        eessi_version: [
+            {'name': tc['name'], 'version': tc['version']} for tc in tcs
+            if EASYBUILD_VERSION >= tc.get('min_easybuild_version', '0')
+        ]
+        for eessi_version, tcs in toolchains.items()
+    }
+
+
+EESSI_SUPPORTED_TOP_LEVEL_TOOLCHAINS = load_supported_top_level_toolchains()
 
 # Supported compute capabilities by CUDA toolkit version
 # Obtained by installing all CUDAs from 12.0.0 to 13.3.0, then using:

eessi_supported_toolchains.toml

@@ -0,0 +1,28 @@
+# Top-level toolchains that are supported for installations in each EESSI version.
+# This file is read by eb_hooks.py (see load_supported_top_level_toolchains()), and has to be installed next to it.
+#
+# Each key is an EESSI version, with a list of toolchains ('name' and 'version').
+# An optional 'min_easybuild_version' can be used for toolchains that are only supported when
+# a sufficiently recent EasyBuild version is used (e.g. because older versions don't know the toolchain).
+
+# If you need to use this from a shell and don't have yq available, you can convert it to json on the fly using
+# python -c "import tomllib, json; print(json.dumps(tomllib.load(open('input.toml','rb')), indent=2))" > output.json
+
+"2023.06" = [
+    { name = "foss", version = "2022b" },
+    { name = "foss", version = "2023a" },
+    { name = "foss", version = "2023b" },
+]
+
+"2025.06" = [
+    { name = "foss", version = "2024a" },
+    { name = "foss", version = "2025a" },
+    { name = "foss", version = "2025b" },
+    { name = "lfoss", version = "2025b", min_easybuild_version = "5.2.0" },
+    { name = "rompi", version = "2025a", min_easybuild_version = "5.3.1" },
+]
+
+"2026.06" = [
+    { name = "foss", version = "2026.1" },
+    { name = "lfoss", version = "2026.1" },
+]

install_scripts.sh

@@ -222,11 +222,12 @@ if [[ -d ${host_injections_easystacks_dir} ]]; then
     copy_files_by_list ${host_injections_easystacks_dir} ${INSTALL_PREFIX}/scripts/gpu_support/nvidia/easystacks "${host_injections_easystacks[@]}"
 fi
 
-# Copy over EasyBuild hooks file used for installations
-hook_files=(
+# Copy over EasyBuild hooks file used for installations (and the files it needs, which have to be located next to it)
+easybuild_init_files=(
     eb_hooks.py
+    eessi_supported_toolchains.toml
 )
-copy_files_by_list ${TOPDIR} ${INSTALL_PREFIX}/init/easybuild "${hook_files[@]}"
+copy_files_by_list ${TOPDIR} ${INSTALL_PREFIX}/init/easybuild "${easybuild_init_files[@]}"
 
 # replace version placeholders in scripts;
 # note: the commands below are always run, regardless of whether the scripts were changed,

scripts/isa_audit/check_tarball.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Audit the instructions in an EESSI build tarball against its CPU target.
+#
+#   check_tarball.sh TARBALL EESSI_SOFTWARE_SUBDIR REFERENCE_DIR [ALLOW_FILE]
+#
+# REFERENCE_DIR is the per-target reference directory of
+# software-layer-scripts#311 (scripts/native_flags/references); the newest gcc
+# reference for the target is used. x86_64 targets are audited with
+# isa_audit.py (iced-x86), aarch64 targets with isa_audit_aarch64.py
+# (llvm-objdump). Generic targets use the baseline references in
+# references/ next to this script (x86-64 and armv8-a, written by
+# generate_generic_references.sh). A violation prints an "ERROR:" line, which
+# bot/check-build.sh already reports as a failed build.
+set -uo pipefail
+tarball=$1 subdir=$2 refdir=$3 allow=${4:-}
+here=$(dirname "$(readlink -f "$0")")
+if [ -n "${EESSI_SKIP_ISA_AUDIT:-}" ]; then
+  echo ">> instruction audit: skipped, \$EESSI_SKIP_ISA_AUDIT is set"; exit 0
+fi
+case $subdir in
+  */generic) refdir=$here/references ;;
+esac
+case ${subdir%%/*} in
+  x86_64)
+    if ! python3 -c 'import iced_x86, elftools' 2>/dev/null; then
+      echo ">> instruction audit: skipped, python3 cannot import iced_x86 and elftools (pip install iced-x86 pyelftools)"
+      exit 0
+    fi ;;
+  aarch64)
+    if ! command -v llvm-objdump >/dev/null; then
+      echo ">> instruction audit: skipped, llvm-objdump (LLVM 20 or later) is not on \$PATH"
+      exit 0
+    fi ;;
+esac
+ref=$(ls "$refdir/$subdir"/gcc-*.txt "$refdir/$subdir"/baseline.txt 2>/dev/null | sort -V | tail -1)
+if [ -z "$ref" ]; then
+  [ "${subdir%%/*}" = aarch64 ] && ref=$(ls "$refdir/$subdir"/clang-*.txt 2>/dev/null | sort -V | tail -1)
+fi
+if [ -z "$ref" ]; then
+  echo ">> instruction audit: no reference for $subdir under $refdir; skipped"
+  exit 0
+fi
+work=$(mktemp -d)
+# installations are read-only (EASYBUILD_READ_ONLY_INSTALLDIR), so make them writable before removing them
+trap 'chmod -R u+w "$work" 2>/dev/null; rm -rf "$work"' EXIT
+tar -xf "$tarball" -C "$work" || { echo "ERROR: instruction audit: cannot unpack $tarball"; exit 1; }
+case ${subdir%%/*} in
+  x86_64)  out=$(python3 "$here/isa_audit.py" ${allow:+--allow "$allow"} "$ref" "$work") ;;
+  aarch64) out=$(python3 "$here/isa_audit_aarch64.py" ${allow:+--allow "$allow"} "$ref" "$work") ;;
+  *) echo ">> instruction audit: no auditor for ${subdir%%/*}"; exit 0 ;;
+esac
+rc=$?
+echo "$out" | grep -E "^(audited|DISPATCH|VIOLATION|RESULT)" | sed 's/^/>> instruction audit: /'
+if [ $rc -ne 0 ]; then
+  echo "ERROR: instruction audit: $(echo "$out" | grep -c '^VIOLATION') violation(s) (feature, file) beyond $subdir ($(basename "$ref"))"
+  exit 1
+fi
+echo ">> instruction audit: pass against $(basename "$ref")"