devstack.yml

---
name: Devstack with rust Keystone

on:
  workflow_dispatch:
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'merge_group' && github.run_id || github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

jobs:
  changes:
    runs-on: ubuntu-latest
    # Required checks must always report a status, so the workflow triggers
    # unconditionally; this job decides whether the real work is needed.
    permissions:
      contents: read
    outputs:
      code: ${{ github.event_name != 'pull_request' || steps.filter.outputs.code == 'true' }}
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1
        with:
          egress-policy: audit

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
        id: filter
        with:
          filters: |
            code:
              - 'Cargo.toml'
              - 'Cargo.lock'
              - '.github/workflows/devstack.yml'
              - 'crates/**'
              - 'policy/**'
              - 'devstack/**'
              - 'tools/devstack-plugin-spire/**'

  build:
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: Enable cache
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: |
            ~/.cargo
          key: ${{ runner.os }}-devstack

      - name: Rust Cache
        uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

      - name: Install Rust
        uses: dtolnay/rust-toolchain@6d653acede28d24f02e3cd41383119e8b1b35921 # stable
        with:
          toolchain: stable

      - name: Install protobuf-compiler
        run: |
          sudo apt-get update
          sudo apt-get install -y protobuf-compiler libtss2-dev pkg-config

      - name: Build Keystone
        run: cargo build --release

      - name: Move artifacts to the root
        run: mv target/release/keystone target/release/keystone-manage ./

      - name: Upload built binaries
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: keystone-devstack-bin
          path: |
            keystone
            keystone-manage

  devstack:
    runs-on: ubuntu-24.04
    needs:
      - build
    permissions:
      contents: read
    env:
      OS_CLOUD: devstack-admin
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1
        with:
          egress-policy: audit

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: Materialize local branch ref for devstack's file:// plugin clone
        # actions/checkout leaves the workspace in detached HEAD, so there is
        # no refs/heads/<branch> for devstack's `git fetch origin <branch>`
        # (fetch_plugins, invoked against this checkout via `enable_plugin
        # key-rs file://...`) to find.
        run: |
          ref="$(git symbolic-ref --short -q HEAD || true)"
          if [[ -z "$ref" ]]; then
            ref="${{ github.head_ref || github.ref_name }}"
            git branch -f "$ref" HEAD
          fi
          echo "KEYSTONE_RS_GIT_REF=$ref" >> "$GITHUB_ENV"

      - name: Download built binaries
        uses: actions/download-artifact@abefc31eafcfbdf6c5336127c1346fdae79ff41c # v5.0.0
        with:
          name: keystone-devstack-bin
          path: ${{ github.workspace }}/bin

      - name: Fix binary permissions
        run: chmod u+x ${{ github.workspace }}/bin/keystone ${{ github.workspace }}/bin/keystone-manage

      - name: Remove pre-installed MySQL
        # ubuntu-24.04 runners ship with a MySQL system service already
        # running (root auth via auth_socket, no password set). devstack's
        # `enable_service mysql` then tries to configure/start its own
        # instance on top of it and fails authenticating as
        # root/DATABASE_PASSWORD against the pre-existing one. Purge it so
        # devstack's own install_database step sets it up from scratch.
        run: |
          sudo systemctl stop mysql.service || true
          sudo apt-get purge -y --auto-remove 'mysql-*' 'mariadb-*' || true
          sudo rm -rf /etc/mysql /var/lib/mysql

      - name: Clone devstack
        run: git clone https://opendev.org/openstack/devstack /opt/stack/devstack

      - name: Package SPIRE plugin as a standalone devstack plugin repo
        # devstack's enable_plugin only supports whole-repo plugins: it
        # clones the given repo to /opt/stack/<name> and looks for the
        # plugin under <clone>/devstack/ - there is no subdirectory support
        # and extra arguments are silently ignored. Pointing it at this
        # repo would clone the whole keystone tree and run the key-rs
        # plugin that lives at its devstack/ dir instead of the SPIRE one,
        # so the spire plugin is repackaged as its own tiny git repo with
        # the standard <repo>/devstack/ layout.
        run: |
          sudo mkdir -p /opt/spire-plugin/devstack
          sudo cp -rT tools/devstack-plugin-spire /opt/spire-plugin/devstack
          sudo git -C /opt/spire-plugin init -q -b main
          sudo git -C /opt/spire-plugin add -A
          sudo git -C /opt/spire-plugin \
            -c user.name=ci -c user.email=ci@localhost \
            commit -qm "SPIRE devstack plugin"

      - name: Write local.conf
        run: |
          cat <<EOF > /opt/stack/devstack/local.conf
          [[local|localrc]]
          ADMIN_PASSWORD=password
          DATABASE_PASSWORD=password
          RABBIT_PASSWORD=password
          SERVICE_PASSWORD=password
          SERVICE_TOKEN=service-token

          disable_all_services
          enable_service mysql
          enable_service rabbit
          enable_service key
          enable_service key-rs
          enable_service tempest
          enable_service spire

          # spire is listed before key-rs: no ordering dependency yet in
          # Phase 1, but Phase 3 (keystone-rs's internal SPIFFE mTLS
          # listener) will need SPIRE's trust bundle available before
          # key-rs starts, and devstack processes enable_plugin lines in
          # the order they're listed here.
          enable_plugin spire file:///opt/spire-plugin main
          enable_plugin key-rs file://${{ github.workspace }} ${{ env.KEYSTONE_RS_GIT_REF }}

          SPIRE_TRUST_DOMAIN=cloud.trust.domain

          KEYSTONE_RS_BIN_DIR=${{ github.workspace }}/bin

          LOGFILE=\$HOME/devstack.log
          LOG_COLOR=False
          EOF
          cat /opt/stack/devstack/local.conf

      - name: Run stack.sh
        working-directory: /opt/stack/devstack
        run: FORCE=yes ./stack.sh

      - name: Verify rust Keystone is serving directly
        run: curl -sf http://127.0.0.1:8080/v3

      - name: Verify Apache routes /identity to rust Keystone
        run: |
          sudo systemctl is-active devstack@key-rs
          curl -sf http://127.0.0.1/identity/v3

      - name: Issue a token via the openstack CLI
        run: |
          source /opt/stack/devstack/openrc admin admin
          openstack token issue

      - name: Verify SPIRE server and agent are healthy
        # $USER is the devstack user (STACK_USER): it owns the sockets the
        # run_process units created, so no sudo hop is needed. On the
        # GitHub runner that's "runner", not the "stack" user traditional
        # devstack docs assume.
        run: |
          /usr/local/bin/spire-server healthcheck -socketPath /opt/stack/data/spire/server.sock
          /usr/local/bin/spire-agent healthcheck -socketPath /opt/stack/data/spire/agent.sock

      - name: Verify pre-registered SPIRE entries exist
        run: |
          /usr/local/bin/spire-server entry show -socketPath /opt/stack/data/spire/server.sock | tee /tmp/spire-entries.txt
          grep -q "service/keystone" /tmp/spire-entries.txt
          grep -q "service/nova-api" /tmp/spire-entries.txt
          grep -q "service/neutron" /tmp/spire-entries.txt
          grep -q "service/nova-compute/host/" /tmp/spire-entries.txt

      - name: Verify SPIRE CA bundle was exported
        run: test -s /etc/keystone/spiffe/ca.crt

      - name: Run tempest identity tests
        # Best-effort signal only: rust Keystone doesn't yet implement the
        # full python surface tempest's identity suite exercises, so
        # failures here must not fail the job (per the request this step
        # was added for).
        working-directory: /opt/stack/tempest
        continue-on-error: true
        run: |
          source /opt/stack/devstack/openrc admin admin
          # devstack's install_tempest pip-installs tempest into its shared
          # per-stack venv (USE_VENV, default since Bobcat), not onto the
          # system PATH - that venv's bin/ is only on PATH inside stack.sh's
          # own process, so a later, separate workflow step has to source it
          # itself before the `tempest` console script is found.
          if [[ -f /opt/stack/data/venv/bin/activate ]]; then
            source /opt/stack/data/venv/bin/activate
          fi
          tempest run --regex '^tempest\.api\.identity'

      - name: Upload tempest results
        # if: always() (not failure()) since the previous step's failures
        # are intentionally swallowed by continue-on-error above.
        if: always()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: tempest-results
          path: |
            /opt/stack/tempest/tempest.log
            /opt/stack/tempest/.stestr

      - name: Dump devstack log
        if: failure()
        run: cat "$HOME/devstack.log" || true

      - name: Dump keystone (python) service log
        if: failure()
        run: sudo journalctl -u devstack@keystone --no-pager || true

      - name: Dump rust Keystone service log
        if: failure()
        run: sudo journalctl -u devstack@key-rs --no-pager || true

      - name: Dump OPA service log
        if: failure()
        run: sudo journalctl -u devstack@key-rs-opa --no-pager || true

      - name: Dump SPIRE server log
        if: failure()
        run: sudo journalctl -u devstack@spire-server --no-pager || true

      - name: Dump SPIRE agent log
        if: failure()
        run: sudo journalctl -u devstack@spire-agent --no-pager || true

      - name: Dump Apache error log
        if: failure()
        run: sudo cat /var/log/apache2/error.log || true

  # Phase 1.5 of doc/plans/spire-integration.md: a full single-node stack
  # with Nova/Cinder/Glance/Neutron, still using standard password/token
  # keystone auth (no SPIFFE-based auth cutover - that needs the patched
  # keystonemiddleware from Phase 4, not started yet). Kept as a separate,
  # initially non-blocking job rather than folded into "devstack" above:
  # this stack is materially heavier (image download, instance boot,
  # extra services) and shouldn't slow every PR's feedback loop until
  # proven stable. continue-on-error at the job level means a failure here
  # is visible but doesn't fail the overall workflow.
  devstack-full:
    runs-on: ubuntu-24.04
    needs:
      - build
    continue-on-error: true
    timeout-minutes: 60
    permissions:
      contents: read
    env:
      OS_CLOUD: devstack-admin
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1
        with:
          egress-policy: audit

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: Materialize local branch ref for devstack's file:// plugin clone
        run: |
          ref="$(git symbolic-ref --short -q HEAD || true)"
          if [[ -z "$ref" ]]; then
            ref="${{ github.head_ref || github.ref_name }}"
            git branch -f "$ref" HEAD
          fi
          echo "KEYSTONE_RS_GIT_REF=$ref" >> "$GITHUB_ENV"

      - name: Download built binaries
        uses: actions/download-artifact@abefc31eafcfbdf6c5336127c1346fdae79ff41c # v5.0.0
        with:
          name: keystone-devstack-bin
          path: ${{ github.workspace }}/bin

      - name: Fix binary permissions
        run: chmod u+x ${{ github.workspace }}/bin/keystone ${{ github.workspace }}/bin/keystone-manage

      - name: Remove pre-installed MySQL
        run: |
          sudo systemctl stop mysql.service || true
          sudo apt-get purge -y --auto-remove 'mysql-*' 'mariadb-*' || true
          sudo rm -rf /etc/mysql /var/lib/mysql

      - name: Runner package workarounds for a full devstack (nova/cinder/glance/neutron)
        # Same workarounds gophercloud/devstack-action uses to run a full
        # (non-keystone-only) devstack on GitHub-hosted Ubuntu runners:
        # - a documented runner-image /etc/hosts bug that otherwise breaks apt
        # - the pre-installed esl-erlang package conflicts with the
        #   rabbitmq-server package devstack's `enable_service rabbit` needs
        # - stray postgresql/docker.io packages that collide with devstack's
        #   own setup
        # See https://github.com/gophercloud/devstack-action's action.yaml.
        run: |
          sudo sed -i 's/^-e \+//g' /etc/hosts
          sudo apt-get update
          sudo apt-get purge -y esl-erlang || true
          sudo apt-get install -y erlang rabbitmq-server
          sudo apt-get install -y runc || true
          sudo apt-get purge -y python3-simplejson python3-pyasn1-modules postgresql* || true

      - name: Clone devstack
        run: git clone https://opendev.org/openstack/devstack /opt/stack/devstack

      - name: Package SPIRE plugin as a standalone devstack plugin repo
        run: |
          sudo mkdir -p /opt/spire-plugin/devstack
          sudo cp -rT tools/devstack-plugin-spire /opt/spire-plugin/devstack
          sudo git -C /opt/spire-plugin init -q -b main
          sudo git -C /opt/spire-plugin add -A
          sudo git -C /opt/spire-plugin \
            -c user.name=ci -c user.email=ci@localhost \
            commit -qm "SPIRE devstack plugin"

      - name: Write local.conf
        run: |
          cat <<'EOF' > /opt/stack/devstack/local.conf
          [[local|localrc]]
          ADMIN_PASSWORD=password
          DATABASE_PASSWORD=password
          RABBIT_PASSWORD=password
          SERVICE_PASSWORD=password
          SERVICE_TOKEN=service-token
          GIT_BASE=https://github.com

          disable_all_services
          enable_service mysql
          enable_service rabbit
          enable_service key
          enable_service key-rs
          enable_service tempest
          enable_service spire

          # Nova (+ placement, a hard Nova dependency in modern devstack)
          enable_service n-api n-cpu n-sch n-cond n-novnc n-crt n-api-meta
          enable_service placement-api placement-client
          # Cinder (c-vol defaults to an LVM-over-loopback backend when no
          # CINDER_DRIVER is set - no real block device needed)
          enable_service c-api c-sch c-vol
          # Glance
          enable_service g-api
          # Neutron with OVN (devstack's default ML2 mechanism driver since
          # 2023). The legacy agents (q-agt/q-dhcp/q-l3/q-meta) are mutually
          # exclusive with OVN - devstack's ovn_sanity_check aborts the run
          # if any of them is enabled. OVN's own services must be listed
          # explicitly because disable_all_services drops stackrc's defaults.
          enable_service q-svc q-ovn-agent
          enable_service ovn-controller ovn-northd ovs-vswitchd ovsdb-server

          enable_plugin spire file:///opt/spire-plugin main
          enable_plugin key-rs file://${{ github.workspace }} ${{ env.KEYSTONE_RS_GIT_REF }}

          SPIRE_TRUST_DOMAIN=cloud.trust.domain

          KEYSTONE_RS_BIN_DIR=${{ github.workspace }}/bin

          # Phase 1.6 of doc/plans/spire-integration.md: pilot
          # https://review.opendev.org/c/openstack/keystonemiddleware/+/1005601
          # (WIP as of writing) against keystone-rs's internal SPIFFE mTLS
          # listener, Nova only. LIBS_FROM_GIT pulls the exact WIP patchset
          # via its Gerrit ref (refs/changes/<last 2 digits of change
          # number>/<change number>/<patchset>) - bump KEYSTONEMIDDLEWARE_BRANCH
          # if/when a newer patchset needs picking up.
          KEYSTONE_RS_SPIFFE_INTERNAL=True
          LIBS_FROM_GIT+=keystonemiddleware
          KEYSTONEMIDDLEWARE_REPO=https://opendev.org/openstack/keystonemiddleware
          KEYSTONEMIDDLEWARE_BRANCH=refs/changes/01/1005601/3
          # The WIP patch's `spiffe` extra isn't selected by LIBS_FROM_GIT's
          # editable install, and current devstack master no longer honors
          # PYTHON_PACKAGES, so the `spiffe` package the patch imports
          # (keystonemiddleware/auth_token/_spiffe_transport.py) plus its
          # undeclared `protobuf` runtime dep are installed by the key-rs
          # plugin's install_keystone_rs (devstack/lib/keystone-rs, gated
          # on KEYSTONE_RS_SPIFFE_INTERNAL) during devstack's install
          # phase - before n-api starts during stack.sh, where a missing
          # package 500s every request and aborts stack.sh before any
          # later workflow step runs.

          # key-rs does not implement Keystone's unified limits API
          # (GET /v3/limits/model), which glance's keystone-quota path
          # (oslo.limit's Enforcer) queries on every image upload. With
          # the Apache /identity proxy pointed at key-rs that 500s and
          # aborts stack.sh in create_flavors, so until key-rs implements
          # the API keep glance on its local quota tables.
          GLANCE_ENABLE_QUOTAS=False

          [[post-config|$NOVA_CONF]]
          [keystone_authtoken]
          # Hardcoded rather than referencing $SPIRE_AGENT_SOCKET/
          # $KEYSTONE_RS_INTERNAL_PORT: local.conf's [[post-config]] values
          # aren't shell-expanded by devstack, so these must match
          # lib/spire's/lib/keystone-rs's actual defaults (also hardcoded
          # elsewhere in this job, e.g. the SPIRE healthcheck steps below).
          spiffe_agent_socket = unix:///opt/stack/data/spire/agent.sock
          auth_url = https://127.0.0.1:8081

          LOGFILE=$HOME/devstack.log
          LOG_COLOR=False
          EOF
          cat /opt/stack/devstack/local.conf

      - name: Run stack.sh
        working-directory: /opt/stack/devstack
        run: FORCE=yes ./stack.sh

      - name: Restart Nova API to pick up auth_token config
        # The key-rs plugin already pip-installed `spiffe` into the venv
        # before stack.sh started n-api, so this restart is only to pick
        # up the
        # [[post-config|$NOVA_CONF]] keystone_authtoken options applied
        # above (devstack writes local.conf's post-config sections during
        # stack.sh, but n-api may already be running by the time nova's
        # own config gets that section merged in).
        run: sudo systemctl restart devstack@n-api

      - name: Verify rust Keystone is serving directly
        run: curl -sf http://127.0.0.1:8080/v3

      - name: Issue a token via the openstack CLI
        run: |
          source /opt/stack/devstack/openrc admin admin
          openstack token issue

      - name: Verify OpenStack service health
        run: |
          source /opt/stack/devstack/openrc admin admin
          openstack compute service list
          openstack volume service list
          openstack image list
          openstack network agent list
          # devstack no longer binds per-service ports (8774/8776/9292/9696):
          # all APIs are proxied by a single Apache on port 80 via URL path
          # prefixes. -f is omitted on purpose: unauthenticated requests get
          # 3xx/401 back, which still proves the API is up and answering.
          curl -s -o /dev/null http://127.0.0.1/compute/
          curl -s -o /dev/null http://127.0.0.1/volume/v3/
          curl -s -o /dev/null http://127.0.0.1/image/
          curl -s -o /dev/null http://127.0.0.1/networking/

      - name: Smoke test - boot and delete a cirros instance
        run: |
          source /opt/stack/devstack/openrc admin admin
          image_id=$(openstack image list -f value -c ID | head -n1)
          network_id=$(openstack network list -f value -c ID --long -c "Name" | awk '$0 !~ /public/ {print $1; exit}')
          openstack server create --image "$image_id" \
            --flavor cirros256 --network "$network_id" smoke-test --wait
          openstack server delete smoke-test --wait

      - name: Verify SPIRE server and agent are healthy
        run: |
          /usr/local/bin/spire-server healthcheck -socketPath /opt/stack/data/spire/server.sock
          /usr/local/bin/spire-agent healthcheck -socketPath /opt/stack/data/spire/agent.sock

      - name: Verify pre-registered SPIRE entries exist
        run: |
          /usr/local/bin/spire-server entry show -socketPath /opt/stack/data/spire/server.sock | tee /tmp/spire-entries.txt
          grep -q "service/keystone" /tmp/spire-entries.txt
          grep -q "service/nova-api" /tmp/spire-entries.txt
          grep -q "service/neutron" /tmp/spire-entries.txt
          grep -q "service/cinder" /tmp/spire-entries.txt
          grep -q "service/glance" /tmp/spire-entries.txt
          grep -q "service/nova-compute/host/" /tmp/spire-entries.txt

      - name: Verify SPIRE CA bundle was exported
        run: test -s /etc/keystone/spiffe/ca.crt

      - name: Verify spiffe-helper processes are running
        run: |
          for svc in nova cinder glance neutron; do
            sudo systemctl is-active "devstack@spiffe-helper-$svc"
          done

      - name: Verify spiffe-helper certs were issued and are valid
        run: |
          # SPIRE does not put the SPIFFE ID in the cert's Subject (it is
          # C=US, O=SPIRE) - the identity lives in the URI SAN. So assert
          # the exact per-service identity each helper is pinned to, and
          # verify the SVID chains to the trust bundle the helper stored
          # alongside it.
          for pair in nova:nova-api cinder:cinder glance:glance neutron:neutron; do
            svc=${pair%%:*}
            dir=/opt/stack/data/spire/certs/$svc
            openssl x509 -in $dir/tls.crt -noout -checkend 0
            openssl verify -CAfile $dir/ca.crt $dir/tls.crt
            openssl x509 -in $dir/tls.crt -noout -text | grep -q "URI:spiffe://cloud.trust.domain/service/${pair#*:}"
          done

      - name: Verify keystone-rs internal SPIFFE mTLS listener started
        # Phase 1.6: confirms interface_internal actually came up in SPIFFE
        # mode (crates/keystone/src/server/listener/spiffe_tls.rs logs this
        # on startup), not just that the config option was written.
        run: sudo journalctl -u devstack@key-rs --no-pager | grep -q "SPIFFE mTLS"

      - name: Verify internal SPIFFE listener rejects unauthenticated clients
        # No client cert presented - the mTLS handshake itself must fail.
        run: |
          if curl -sf -k --max-time 5 https://127.0.0.1:8081/v3 >/dev/null 2>&1; then
            echo "internal SPIFFE listener accepted a connection with no client cert" >&2
            exit 1
          fi

      - name: Verify Nova's auth_token back-channel authenticates over SPIFFE mTLS
        # Phase 1.6 POC: nova.conf's [keystone_authtoken] points auth_url at
        # keystone-rs's internal SPIFFE listener (see local.conf above) and
        # sets spiffe_agent_socket, per
        # https://review.opendev.org/c/openstack/keystonemiddleware/+/1005601
        # (WIP). First, fetch nova-api's own SVID directly and confirm the
        # listener accepts it (positive control, independent of nova-api's
        # own process). Then confirm nova-api itself is actually serving
        # requests through that path - the "Verify OpenStack service health"/
        # smoke-test steps above already exercised nova-api's auth_token
        # validation end-to-end, so their prior success is the real
        # end-to-end signal; this step only adds the direct positive-control
        # check plus an explicit re-assertion that nova-api is still healthy
        # after the config change and restart.
        run: |
          # The SPIRE entries are pinned to the devstack user's uid
          # (unix:uid selector), and this workflow runs as that user, so the
          # fetch must NOT use sudo: the agent's "unix" workload attestor
          # reads the caller's uid, and as root it sees uid 0, matches no
          # entry and returns "No identity issued ... registered=false".
          # (Same reason the spire healthchecks above don't sudo either.)
          tmpdir=$(mktemp -d)
          /usr/local/bin/spire-agent api fetch x509 \
            -socketPath /opt/stack/data/spire/agent.sock -write "$tmpdir"
          chmod 0644 "$tmpdir"/*.pem "$tmpdir"/*.key 2>/dev/null || true
          # The agent returns the workload's entire identity set in
          # arbitrary order (svid.0.pem can be any entry, the SPIRE CLI
          # has no per-identity filter), so select by URI SAN, not index.
          want="spiffe://cloud.trust.domain/service/nova-api"
          svid=""
          for f in "$tmpdir"/svid.*.pem; do
            if openssl x509 -in "$f" -noout -ext subjectAltName \
                 | grep -qE "URI:$want(,|\$)"; then
              svid="$f"
              break
            fi
          done
          if [[ -z "$svid" ]]; then
            echo "SVID $want not among the fetched identities" >&2
            exit 1
          fi
          n=${svid##*/svid.}
          n=${n%.pem}
          key="$tmpdir/svid.$n.key"
          bundle="$tmpdir/bundle.$n.pem"
          # A SPIFFE SVID carries its identity in a URI SAN, so a
          # hostname/IP check against 127.0.0.1 can never succeed (curl
          # would reject the perfectly valid SVID). Verify the server
          # chain against the trust bundle and skip only the hostname
          # assertion, the same way the ksm SPIFFE transport does.
          python3 -c '
          import ssl, sys, urllib.request
          svid, key, bundle = sys.argv[1:4]
          ctx = ssl.create_default_context(cafile=bundle)
          ctx.check_hostname = False
          ctx.load_cert_chain(svid, key)
          opener = urllib.request.build_opener(
              urllib.request.HTTPSHandler(context=ctx))
          sys.stdout.buffer.write(opener.open("https://127.0.0.1:8081/v3", timeout=10).read())
          ' "$svid" "$key" "$bundle"

          source /opt/stack/devstack/openrc admin admin
          openstack compute service list

      - name: Configure control-plane SPIFFE authorization mapping
        # Gap identified in the Phase 1.6 review: an SVID authenticates at
        # TLS on the internal listener, but until a mapping ruleset exists
        # it carries zero authorization meaning - authenticate_by_mapping
        # fails closed (NoMatchingRule, now correctly surfaced as 401, not
        # 500 - see the KeystoneApiError::UnauthorizedNoContext fix in
        # crates/api-types/src/error_conv.rs). No code change was needed
        # for the mapping itself:
        # ValidatedSecurityContext::new_for_scope's `AuthenticationContext::
        # Mapping` arm (crates/core/src/auth.rs) already upgrades an
        # is_system rule's Unscoped scope to System("all") generically, for
        # any mapping-authenticated source. This just wires up a minimal
        # global ruleset: nova-api's own control-plane SVID (exact
        # spiffe.id match) maps to a virtual system-scoped principal
        # holding the `service` role - the same least-privilege role
        # policy/auth/token/show.rego already grants dedicated token-
        # introspection access to, mirroring a real keystonemiddleware
        # service account rather than granting full admin.
        # Prerequisite: the mapping provider is raft-only, so the key-rs
        # plugin must have brought up its single-node [distributed_storage]
        # (it does when KEYSTONE_RS_SPIFFE_INTERNAL=True, see
        # devstack/lib/keystone-rs) - without it this POST 501s.
        run: |
          source /opt/stack/devstack/openrc admin admin
          admin_token=$(openstack token issue -f value -c id)
          service_role_id=$(openstack role show service -f value -c id)

          cat > /tmp/spiffe-control-plane-ruleset.json <<EOF
          {
            "mapping": {
              "mapping_id": "ci-spiffe-control-plane",
              "domain_id": null,
              "source": {"type": "spiffe", "trust_domain": "cloud.trust.domain"},
              "domain_resolution_mode": {"type": "fixed"},
              "enabled": true,
              "rules": [
                {
                  "name": "nova-api-control-plane",
                  "match": {
                    "all_of": [
                      {
                        "type": "condition",
                        "equals": {
                          "claim": "spiffe.id",
                          "value": "spiffe://cloud.trust.domain/service/nova-api"
                        }
                      }
                    ]
                  },
                  "identity": {
                    "user_name": "svid-nova-api",
                    "is_system": true
                  },
                  "authorizations": [
                    {
                      "type": "system",
                      "system_id": "all",
                      "roles": [{"id": "$service_role_id", "name": "service"}]
                    }
                  ],
                  "groups": []
                }
              ]
            }
          }
          EOF

          curl -sf -X POST http://127.0.0.1:8080/v4/mappings/rulesets \
            -H "X-Auth-Token: $admin_token" \
            -H "Content-Type: application/json" \
            -d @/tmp/spiffe-control-plane-ruleset.json

      - name: Verify a mapped control-plane SVID performs real authz
        # Upgrades the earlier back-channel check from "transport works"
        # to "SVID identity works": a subject token is only ever inspected
        # over the internal SPIFFE listener by a caller whose SVID actually
        # matched the ruleset above and got the `service` role at system
        # scope (policy/auth/token/show.rego's dedicated `service` check).
        # nova-api's own SVID is fetched fresh, same as the earlier
        # back-channel step - it isn't reused across steps since each `run:`
        # block is a separate shell.
        run: |
          source /opt/stack/devstack/openrc admin admin
          subject_token=$(openstack token issue -f value -c id)

          tmpdir=$(mktemp -d)
          /usr/local/bin/spire-agent api fetch x509 \
            -socketPath /opt/stack/data/spire/agent.sock -write "$tmpdir"
          chmod 0644 "$tmpdir"/*.pem "$tmpdir"/*.key 2>/dev/null || true
          # The agent returns the workload's entire identity set in
          # arbitrary order (svid.0.pem can be any entry, the SPIRE CLI
          # has no per-identity filter), so select by URI SAN, not index.
          # This MUST be nova-api's SVID: it is the exact spiffe.id the
          # ruleset above matches, and any other entry fails closed with
          # 401 (the negative control below pins that down).
          want="spiffe://cloud.trust.domain/service/nova-api"
          svid=""
          for f in "$tmpdir"/svid.*.pem; do
            if openssl x509 -in "$f" -noout -ext subjectAltName \
                 | grep -qE "URI:$want(,|\$)"; then
              svid="$f"
              break
            fi
          done
          if [[ -z "$svid" ]]; then
            echo "SVID $want not among the fetched identities" >&2
            exit 1
          fi
          n=${svid##*/svid.}
          n=${n%.pem}
          key="$tmpdir/svid.$n.key"
          bundle="$tmpdir/bundle.$n.pem"

          status=$(python3 -c '
          import ssl, sys, urllib.request, urllib.error
          svid, key, bundle, token = sys.argv[1:5]
          ctx = ssl.create_default_context(cafile=bundle)
          ctx.check_hostname = False
          ctx.load_cert_chain(svid, key)
          opener = urllib.request.build_opener(
              urllib.request.HTTPSHandler(context=ctx))
          req = urllib.request.Request(
              "https://127.0.0.1:8081/v3/auth/tokens",
              headers={"X-Subject-Token": token})
          try:
              resp = opener.open(req, timeout=10)
              print(resp.status)
          except urllib.error.HTTPError as e:
              print(e.code)
          ' "$svid" "$key" "$bundle" "$subject_token")

          echo "mapped SVID token-validation status: $status"
          if [[ "$status" != "200" ]]; then
            echo "expected 200 from a ruleset-matched, service-scoped SVID" >&2
            exit 1
          fi

      - name: Verify an unmapped SVID is rejected, not silently authorized
        # Negative control for the check above: cinder's SVID matches no
        # rule in the ruleset created here (only nova-api's exact spiffe.id
        # does), so it must be denied - proving the ruleset is actually
        # gatekeeping rather than mapping every workload to system scope.
        # Reuses the cinder cert spiffe-helper already wrote to disk
        # (verified readable without sudo by the "Verify spiffe-helper
        # certs" step above) instead of fetching a second SVID by hand.
        run: |
          source /opt/stack/devstack/openrc admin admin
          subject_token=$(openstack token issue -f value -c id)

          dir=/opt/stack/data/spire/certs/cinder
          status=$(python3 -c '
          import ssl, sys, urllib.request, urllib.error
          svid, key, bundle, token = sys.argv[1:5]
          ctx = ssl.create_default_context(cafile=bundle)
          ctx.check_hostname = False
          ctx.load_cert_chain(svid, key)
          opener = urllib.request.build_opener(
              urllib.request.HTTPSHandler(context=ctx))
          req = urllib.request.Request(
              "https://127.0.0.1:8081/v3/auth/tokens",
              headers={"X-Subject-Token": token})
          try:
              resp = opener.open(req, timeout=10)
              print(resp.status)
          except urllib.error.HTTPError as e:
              print(e.code)
          ' "$dir/tls.crt" "$dir/tls.key" "$dir/ca.crt" "$subject_token")

          echo "unmapped SVID token-validation status: $status"
          if [[ "$status" != "401" ]]; then
            echo "expected 401 for an SVID absent from the ruleset, got $status" >&2
            exit 1
          fi

      - name: Dump devstack log
        if: failure()
        run: cat "$HOME/devstack.log" || true

      - name: Dump keystone (python) service log
        if: failure()
        run: sudo journalctl -u devstack@keystone --no-pager || true

      - name: Dump rust Keystone service log
        if: failure()
        run: sudo journalctl -u devstack@key-rs --no-pager || true

      - name: Dump Nova API log
        if: failure()
        run: sudo journalctl -u devstack@n-api --no-pager || true

      - name: Dump Nova Compute log
        if: failure()
        run: sudo journalctl -u devstack@n-cpu --no-pager || true

      - name: Dump Cinder API log
        if: failure()
        run: sudo journalctl -u devstack@c-api --no-pager || true

      - name: Dump Cinder Volume log
        if: failure()
        run: sudo journalctl -u devstack@c-vol --no-pager || true

      - name: Dump Glance API log
        if: failure()
        run: sudo journalctl -u devstack@g-api --no-pager || true

      - name: Dump Neutron server log
        if: failure()
        run: sudo journalctl -u devstack@q-svc --no-pager || true

      - name: Dump SPIRE server log
        if: failure()
        run: sudo journalctl -u devstack@spire-server --no-pager || true

      - name: Dump SPIRE agent log
        if: failure()
        run: sudo journalctl -u devstack@spire-agent --no-pager || true

      - name: Dump spiffe-helper logs
        if: failure()
        run: |
          for svc in nova cinder glance neutron; do
            sudo journalctl -u "devstack@spiffe-helper-$svc" --no-pager || true
          done