From 47dbc61e32a2c71d8d06caeffab6f9fef9e1014e Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 5 Oct 2026 21:20:55 +0200 Subject: [PATCH 01/62] fix(ci): reuse the existing platform App for KSail ARC authentication --- docs/operations/ksail-arc.md | 20 ++- .../external-secret.yaml | 14 +- .../ksail-analysis-runners/kustomization.yaml | 2 + .../ksail-analysis-runners/secret-store.yaml | 17 +++ .../service-account.yaml | 6 + .../infrastructure/vault-config/job.yaml | 16 ++ scripts/tests/arc-staging/auth_test.go | 139 ++++++++++++++++++ scripts/tests/arc-staging/staging_test.go | 21 ++- 8 files changed, 221 insertions(+), 14 deletions(-) create mode 100644 k8s/bases/infrastructure/ksail-analysis-runners/secret-store.yaml create mode 100644 k8s/bases/infrastructure/ksail-analysis-runners/service-account.yaml create mode 100644 scripts/tests/arc-staging/auth_test.go diff --git a/docs/operations/ksail-arc.md b/docs/operations/ksail-arc.md index acaa5e7f04..06ab66c1bc 100644 --- a/docs/operations/ksail-arc.md +++ b/docs/operations/ksail-arc.md @@ -18,11 +18,18 @@ jobs and Docker actions are deliberately unsupported. Activation requires a separate reviewed change and all of these proofs: -1. The maintainer approves an App installed only on KSail with the required - repository runner-administration permission. A maintainer provisions its three - authentication fields in the existing secret system at `github-arc-ksail`. - `arc-ksail-app` is an ExternalSecret reference, not a new or broadened shared - credential. Do not print, check in or mount the private key in a job runner. +1. Reuse the platform's existing GitHub management App from + `infrastructure/github/app`. The `arc-ksail-app` ExternalSecret maps its + `app_id`, `installation_id` and `pem` properties to ARC's three authentication + fields. Its namespaced SecretStore authenticates as the dedicated + `arc-ksail-app` credential-reader ServiceAccount. The OpenBao role grants read + on that exact credential path; the shared ESO role keeps its existing access. + Job runners use the chart's separate no-permission identity. Verify the + current installation covers KSail and grants repository + Administration read/write before activation. The pool's registration URL + remains KSail-only; the existing App installation also serves other platform + consumers. No new App, credential copy, key rotation or permission expansion + is required. Do not print, check in or mount the private key in a job runner. 2. Establish explicitly approved, isolated analysis capacity. The runner selector and toleration name `platform.devantler.tech/ksail-analysis=enabled`; no current capacity is assumed to carry that label. New billable capacity requires its @@ -76,7 +83,8 @@ before uninstalling the pool. **Suspending a HelmRelease alone does not drain an installed scale set.** Namespace/release retirement follows the platform's two-stage persistence protection; never delete a namespace to cancel a job. Verify no listener, runner or GitHub registration remains before deleting approved -temporary capacity. Retire the App credential only after its consumers are gone. +temporary capacity. Remove only the ARC ExternalSecret and its materialized +Secret; retain the shared App credential and its other platform consumers. Official references: [ARC deployment and security guidance](https://docs.github.com/en/actions/how-tos/manage-runners/use-actions-runner-controller/deploy-runner-scale-sets) and [App authentication](https://docs.github.com/en/actions/how-tos/manage-runners/use-actions-runner-controller/authenticate-to-the-api). diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/external-secret.yaml b/k8s/bases/infrastructure/ksail-analysis-runners/external-secret.yaml index f687173965..1799bc7776 100644 --- a/k8s/bases/infrastructure/ksail-analysis-runners/external-secret.yaml +++ b/k8s/bases/infrastructure/ksail-analysis-runners/external-secret.yaml @@ -7,20 +7,20 @@ spec: refreshInterval: 1h secretStoreRef: name: openbao - kind: ClusterSecretStore + kind: SecretStore target: name: arc-ksail-app creationPolicy: Owner data: - secretKey: github_app_id remoteRef: - key: github-arc-ksail - property: github_app_id + key: infrastructure/github/app + property: app_id - secretKey: github_app_installation_id remoteRef: - key: github-arc-ksail - property: github_app_installation_id + key: infrastructure/github/app + property: installation_id - secretKey: github_app_private_key remoteRef: - key: github-arc-ksail - property: github_app_private_key + key: infrastructure/github/app + property: pem diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/kustomization.yaml b/k8s/bases/infrastructure/ksail-analysis-runners/kustomization.yaml index 45d69f186f..2edc4aea69 100644 --- a/k8s/bases/infrastructure/ksail-analysis-runners/kustomization.yaml +++ b/k8s/bases/infrastructure/ksail-analysis-runners/kustomization.yaml @@ -4,4 +4,6 @@ resources: - oci-repository.yaml - helm-release.yaml - external-secret.yaml + - secret-store.yaml + - service-account.yaml - cilium-network-policy-runner.yaml diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/secret-store.yaml b/k8s/bases/infrastructure/ksail-analysis-runners/secret-store.yaml new file mode 100644 index 0000000000..4c5f127505 --- /dev/null +++ b/k8s/bases/infrastructure/ksail-analysis-runners/secret-store.yaml @@ -0,0 +1,17 @@ +apiVersion: external-secrets.io/v1 +kind: SecretStore +metadata: + name: openbao + namespace: arc-ksail-analysis +spec: + provider: + vault: + server: "http://openbao-active.openbao.svc.cluster.local:8200" + path: "secret" + version: "v2" + auth: + kubernetes: + mountPath: "kubernetes" + role: "arc-ksail-app" + serviceAccountRef: + name: "arc-ksail-app" diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/service-account.yaml b/k8s/bases/infrastructure/ksail-analysis-runners/service-account.yaml new file mode 100644 index 0000000000..1b6628a0d1 --- /dev/null +++ b/k8s/bases/infrastructure/ksail-analysis-runners/service-account.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: arc-ksail-app + namespace: arc-ksail-analysis +automountServiceAccountToken: false diff --git a/k8s/bases/infrastructure/vault-config/job.yaml b/k8s/bases/infrastructure/vault-config/job.yaml index 263a3b84ab..1731d6ff6e 100644 --- a/k8s/bases/infrastructure/vault-config/job.yaml +++ b/k8s/bases/infrastructure/vault-config/job.yaml @@ -764,6 +764,14 @@ spec: } POLICY + # ARC reuses the existing App with a separate identity. Its + # reader can reach this single credential, not the GitHub subtree. + bao policy write arc-ksail-app-readonly - <<'POLICY' + path "secret/data/infrastructure/github/app" { + capabilities = ["read"] + } + POLICY + bao policy write app-fleetdm-readonly - <<'POLICY' path "secret/data/apps/fleetdm/*" { capabilities = ["read"] @@ -1113,6 +1121,14 @@ spec: policies=infra-github-readonly \ ttl=1h + # Only ESO's TokenRequest for this credential-reader identity + # authenticates to the ARC store; job runners use another SA. + bao write auth/kubernetes/role/arc-ksail-app \ + bound_service_account_names=arc-ksail-app \ + bound_service_account_namespaces=arc-ksail-analysis \ + policies=arc-ksail-app-readonly \ + ttl=1h + # Dedicated tenant role for the unifi app. Authenticates the app's # own `unifi` ServiceAccount (used by its namespaced SecretStore, # providers/hetzner/apps/unifi/secret-store.yaml) and carries ONLY diff --git a/scripts/tests/arc-staging/auth_test.go b/scripts/tests/arc-staging/auth_test.go new file mode 100644 index 0000000000..0f775f332f --- /dev/null +++ b/scripts/tests/arc-staging/auth_test.go @@ -0,0 +1,139 @@ +package arcstaging_test + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + + "mvdan.cc/sh/v3/syntax" +) + +func TestCredentialReaderIsSeparateFromJobRunners(t *testing.T) { + component := "k8s/bases/infrastructure/ksail-analysis-runners/" + store := readYAML(t, component+"secret-store.yaml") + equal(t, field(t, store, "kind"), "SecretStore") + equal(t, field(t, store, "metadata", "name"), "openbao") + equal(t, field(t, store, "metadata", "namespace"), "arc-ksail-analysis") + vault := field(t, store, "spec", "provider", "vault") + equal(t, field(t, vault, "path"), "secret") + equal(t, field(t, vault, "version"), "v2") + auth := field(t, vault, "auth", "kubernetes") + equal(t, field(t, auth, "mountPath"), "kubernetes") + equal(t, field(t, auth, "role"), "arc-ksail-app") + equal(t, field(t, auth, "serviceAccountRef", "name"), "arc-ksail-app") + account := readYAML(t, component+"service-account.yaml") + equal(t, field(t, account, "metadata", "name"), "arc-ksail-app") + equal(t, field(t, account, "metadata", "namespace"), "arc-ksail-analysis") + equal(t, field(t, account, "automountServiceAccountToken"), false) + resources := field(t, readYAML(t, component+"kustomization.yaml"), "resources").([]any) + for _, required := range []string{"secret-store.yaml", "service-account.yaml"} { + found := false + for _, resource := range resources { + found = found || resource == required + } + if !found { + t.Fatalf("credential reader resource %s is not included in the pool", required) + } + } +} + +// Exercise only the literal authorization writes from the bootstrap script. +// The fake bao captures their actual arguments and stdin without running the +// credential seeding, initialization or any live OpenBao operation. +func captureBaoWrite(t *testing.T, prefix ...string) ([]string, string) { + t.Helper() + job := readYAML(t, "k8s/bases/infrastructure/vault-config/job.yaml") + containers := field(t, job, "spec", "template", "spec", "containers").([]any) + var source string + for _, container := range containers { + if field(t, container, "name") == "vault-config" { + command := field(t, container, "command").([]any) + source = command[len(command)-1].(string) + } + } + parsed, err := syntax.NewParser().Parse(strings.NewReader(source), "vault-config") + if err != nil { + t.Fatal(err) + } + var matches []*syntax.Stmt + syntax.Walk(parsed, func(node syntax.Node) bool { + statement, ok := node.(*syntax.Stmt) + if !ok { + return true + } + call, ok := statement.Cmd.(*syntax.CallExpr) + if !ok || len(call.Args) < len(prefix) { + return true + } + for index, expected := range prefix { + if call.Args[index].Lit() != expected { + return true + } + } + for _, argument := range call.Args { + if argument.Lit() == "" { + t.Fatal("authorization write must use literal arguments") + } + } + for _, redirect := range statement.Redirs { + if redirect.Op != syntax.Hdoc || redirect.Hdoc == nil || redirect.Hdoc.Lit() == "" { + t.Fatal("authorization payload must be a literal heredoc") + } + } + matches = append(matches, statement) + return false + }) + if len(matches) != 1 { + t.Fatalf("expected one authorization write %v, got %d", prefix, len(matches)) + } + var script bytes.Buffer + if err := syntax.NewPrinter().Print(&script, matches[0]); err != nil { + t.Fatal(err) + } + directory := t.TempDir() + fake := "#!/bin/sh\nprintf '%s\\n' \"$@\" >\"$ARC_CAPTURE_ARGS\"\n/bin/cat >\"$ARC_CAPTURE_STDIN\"\n" + if err := os.WriteFile(filepath.Join(directory, "bao"), []byte(fake), 0o700); err != nil { + t.Fatal(err) + } + argsFile := filepath.Join(directory, "args") + stdinFile := filepath.Join(directory, "stdin") + command := exec.Command("/bin/sh", "-ec", script.String()) + command.Env = []string{"PATH=" + directory, "ARC_CAPTURE_ARGS=" + argsFile, "ARC_CAPTURE_STDIN=" + stdinFile} + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("execute authorization write: %v: %s", err, output) + } + args, err := os.ReadFile(argsFile) + if err != nil { + t.Fatal(err) + } + stdin, err := os.ReadFile(stdinFile) + if err != nil { + t.Fatal(err) + } + return strings.Split(strings.TrimSuffix(string(args), "\n"), "\n"), string(stdin) +} + +func TestOpenBaoReaderCanReadOnlyTheExistingApp(t *testing.T) { + _, policy := captureBaoWrite(t, "bao", "policy", "write", "arc-ksail-app-readonly") + want := `path "secret/data/infrastructure/github/app" { capabilities = ["read"] }` + if !reflect.DeepEqual(strings.Fields(policy), strings.Fields(want)) { + t.Fatalf("ARC reader policy must grant only the existing App path, got %q", policy) + } + args, stdin := captureBaoWrite(t, "bao", "write", "auth/kubernetes/role/arc-ksail-app") + wantArgs := []string{"write", "auth/kubernetes/role/arc-ksail-app", + "bound_service_account_names=arc-ksail-app", "bound_service_account_namespaces=arc-ksail-analysis", + "policies=arc-ksail-app-readonly", "ttl=1h"} + if !reflect.DeepEqual(args, wantArgs) || stdin != "" { + t.Fatalf("ARC role must bind only its credential reader: args=%v stdin=%q", args, stdin) + } + shared, _ := captureBaoWrite(t, "bao", "write", "auth/kubernetes/role/external-secrets") + for _, argument := range shared { + if strings.HasPrefix(argument, "policies=") && (strings.Contains(argument, "github") || strings.Contains(argument, "arc-ksail")) { + t.Fatal("the shared ESO role must not gain GitHub App access") + } + } +} diff --git a/scripts/tests/arc-staging/staging_test.go b/scripts/tests/arc-staging/staging_test.go index 983cc54533..2cd51254cf 100644 --- a/scripts/tests/arc-staging/staging_test.go +++ b/scripts/tests/arc-staging/staging_test.go @@ -114,13 +114,32 @@ func TestControllerIsNamespaceScopedAndCredentialsAreExternal(t *testing.T) { secret := readYAML(t, "k8s/bases/infrastructure/ksail-analysis-runners/external-secret.yaml") equal(t, field(t, secret, "kind"), "ExternalSecret") equal(t, field(t, secret, "spec", "secretStoreRef", "name"), "openbao") + equal(t, field(t, secret, "spec", "secretStoreRef", "kind"), "SecretStore") equal(t, field(t, secret, "spec", "target", "name"), "arc-ksail-app") entries := field(t, secret, "spec", "data").([]any) if len(entries) != 3 { t.Fatal("only the three App authentication keys are expected") } + properties := map[string]string{ + "github_app_id": "app_id", + "github_app_installation_id": "installation_id", + "github_app_private_key": "pem", + } for _, entry := range entries { - equal(t, field(t, entry, "remoteRef", "key"), "github-arc-ksail") + key, ok := field(t, entry, "secretKey").(string) + if !ok { + t.Fatal("App credential target key must be a string") + } + property, ok := properties[key] + if !ok { + t.Fatalf("unexpected or duplicate App credential target key %q", key) + } + equal(t, field(t, entry, "remoteRef", "key"), "infrastructure/github/app") + equal(t, field(t, entry, "remoteRef", "property"), property) + delete(properties, key) + } + if len(properties) != 0 { + t.Fatalf("missing App credential target keys: %v", properties) } } From 50dd13b1f969a1e2c8ff489a858b42125dc24bcd Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 5 Oct 2026 22:10:26 +0200 Subject: [PATCH 02/62] feat(ci): build a restricted KSail analysis runner image --- .../publish-ksail-analysis-runner.yaml | 119 +++++++++++ docs/operations/ksail-arc.md | 14 ++ go.mod | 7 +- go.sum | 22 +- images/ksail-analysis-runner/Dockerfile | 39 ++++ images/ksail-analysis-runner/smoke.sh | 40 ++++ .../best-practices/verify-app-images.yaml | 22 +- .../tests/arc-staging/image_routing_test.go | 199 ++++++++++++++++++ scripts/tests/arc-staging/image_test.go | 81 +++++++ .../refresh-flux-ghcr-auth/contracts_test.go | 5 + .../fake_kubectl_test.go | 18 +- ...-inventory-first-party-image-signatures.sh | 26 +-- talos/cluster/verify-first-party-images.yaml | 5 + 13 files changed, 570 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/publish-ksail-analysis-runner.yaml create mode 100644 images/ksail-analysis-runner/Dockerfile create mode 100644 images/ksail-analysis-runner/smoke.sh create mode 100644 scripts/tests/arc-staging/image_routing_test.go create mode 100644 scripts/tests/arc-staging/image_test.go diff --git a/.github/workflows/publish-ksail-analysis-runner.yaml b/.github/workflows/publish-ksail-analysis-runner.yaml new file mode 100644 index 0000000000..5d4b48a173 --- /dev/null +++ b/.github/workflows/publish-ksail-analysis-runner.yaml @@ -0,0 +1,119 @@ +name: Publish KSail Analysis Runner + +on: + pull_request: + branches: [main] + paths: + - .github/workflows/publish-ksail-analysis-runner.yaml + - images/ksail-analysis-runner/** + - scripts/tests/arc-staging/** + push: + branches: [main] + paths: + - .github/workflows/publish-ksail-analysis-runner.yaml + - images/ksail-analysis-runner/** + - scripts/tests/arc-staging/** + workflow_dispatch: + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + IMAGE: ghcr.io/devantler-tech/platform-ksail-analysis-runner + IMAGE_TAG: ${{ github.sha }} + +jobs: + verify: + name: Verify restricted analysis toolchain + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - name: Harden runner + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: Check out source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Build pinned analysis image + run: | + docker build --platform linux/amd64 --tag ksail-analysis:verify images/ksail-analysis-runner + + - name: Exercise runner bootstrap and desktop compiler + run: | + docker run --rm --platform linux/amd64 --user 1001:1001 \ + --read-only --cap-drop ALL --security-opt no-new-privileges \ + --memory 4g --cpus 2 --env HOME=/runner-data \ + --tmpfs /runner-data:rw,uid=1001,gid=1001,size=2g \ + --tmpfs /tmp:rw,uid=1001,gid=1001,size=1g \ + ksail-analysis:verify /usr/local/bin/ksail-analysis-smoke + + publish: + name: Publish and sign immutable analysis image + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: verify + runs-on: ubuntu-24.04 + permissions: + contents: read + packages: write + id-token: write + steps: + - name: Harden runner + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: Check out source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Buildx + uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4.4.0 + with: + driver: docker-container + + - name: Log in to GHCR + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push immutable analysis image + id: build + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: ./images/ksail-analysis-runner + platforms: linux/amd64 + push: true + tags: ${{ env.IMAGE }}:${{ env.IMAGE_TAG }} + provenance: mode=max + sbom: true + labels: | + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.version=${{ env.IMAGE_TAG }} + + - name: Set up Cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + - name: Sign and verify published digest + env: + DIGEST: ${{ steps.build.outputs.digest }} + run: | + set -euo pipefail + test -n "${DIGEST}" + cosign sign --yes "${IMAGE}@${DIGEST}" + cosign verify \ + --certificate-identity "https://github.com/devantler-tech/platform/.github/workflows/publish-ksail-analysis-runner.yaml@refs/heads/main" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + "${IMAGE}@${DIGEST}" + printf 'Published signed image: %s@%s\n' "${IMAGE}" "${DIGEST}" diff --git a/docs/operations/ksail-arc.md b/docs/operations/ksail-arc.md index 06ab66c1bc..b6b55f0daf 100644 --- a/docs/operations/ksail-arc.md +++ b/docs/operations/ksail-arc.md @@ -14,6 +14,20 @@ zero idle runners and a maximum of one job runner. There is no container mode, Docker socket, host volume, privileged container or runner API token. Container jobs and Docker actions are deliberately unsupported. +The analysis toolchain source lives in `images/ksail-analysis-runner`. Its +digest-pinned upstream runner, frozen Ubuntu package snapshot and checksum-checked +Go and Node archives supply the desktop compiler and headers without job-time +root access. The publisher builds and exercises the image on pull requests, then +publishes, attests and signs only on a push to main. Both Kyverno and Talos accept +that workflow identity only for the exact analysis image repository. The staged +pool still requires a verified published digest before activation. + +The smoke test starts the copied runner and compiles a Go program against GTK +and WebKit as UID/GID 1001 with a read-only root filesystem and no capabilities. +The runner home and temporary files use disposable writable storage. Bootstrap +copies use `cp -R`; preserving the root directory's ownership and timestamps +would fail on a group-writable Kubernetes volume. + ## Activation gates Activation requires a separate reviewed change and all of these proofs: diff --git a/go.mod b/go.mod index 006b8e7282..1e8dae70f8 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,18 @@ module github.com/devantler-tech/platform go 1.26.6 require ( + github.com/google/cel-go v0.31.0 github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 github.com/siderolabs/talos/pkg/machinery v1.14.0 go.yaml.in/yaml/v2 v2.4.4 - go.yaml.in/yaml/v3 v3.0.5 gopkg.in/yaml.v3 v3.0.1 mvdan.cc/sh/v3 v3.14.1 sigs.k8s.io/yaml v1.6.0 ) require ( + cel.dev/expr v0.25.2 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/containerd/go-cni v1.1.13 // indirect github.com/containernetworking/cni v1.3.0 // indirect @@ -22,9 +24,12 @@ require ( github.com/sasha-s/go-deadlock v0.3.9 // indirect github.com/siderolabs/crypto v0.6.5 // indirect github.com/siderolabs/gen v0.8.7 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 // indirect golang.org/x/net v0.58.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.41.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754 // indirect google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.12 // indirect diff --git a/go.sum b/go.sum index 059c2c8d8a..91116ee7a5 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -6,18 +10,22 @@ github.com/containerd/go-cni v1.1.13 h1:eFSGOKlhoYNxpJ51KRIMHZNlg5UgocXEIEBGkY7H github.com/containerd/go-cni v1.1.13/go.mod h1:nTieub0XDRmvCZ9VI/SBG6PyqT95N4FIhxsauF1vSBI= github.com/containernetworking/cni v1.3.0 h1:v6EpN8RznAZj9765HhXQrtXgX+ECGebEYEmnuFjskwo= github.com/containernetworking/cni v1.3.0/go.mod h1:Bs8glZjjFfGPHMw6hQu82RUgEPNGEaBb9KS5KtNMnJ4= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= -github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= +github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474= +github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.31.0 h1:H0bhpFTqOvmHrBGrWKp7ZlhBm5Hh8PYUEXnwxT1LL7A= +github.com/google/cel-go v0.31.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= @@ -37,8 +45,8 @@ github.com/petermattis/goid v0.0.0-20260713124913-97594f28f5ca h1:GHSUVE4yOgX4E7 github.com/petermattis/goid v0.0.0-20260713124913-97594f28f5ca/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/planetscale/vtprotobuf v0.6.1-0.20260702190614-8ae5a48058df h1:x2ymdov8jnZLDPfI+VVcf/ZvzuZ2u36ieXTuQASMkWI= github.com/planetscale/vtprotobuf v0.6.1-0.20260702190614-8ae5a48058df/go.mod h1:araspv2uYKozbi5lrKaqpv1/Uei7eQSml8JCw2A3IRg= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= +github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/sasha-s/go-deadlock v0.3.9 h1:fiaT9rB7g5sr5ddNZvlwheclN9IP86eFW9WgqlEQV+w= @@ -67,8 +75,6 @@ go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRk go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= @@ -87,6 +93,8 @@ golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754 h1:dWeMvEJ3JhYgqSCAHUZZJgMUyfniiiCvDc72x5EqJP0= +google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754/go.mod h1:q/3oV3jAi5vwelxsVAprMBC8BcM2zmNe+IjRGd+9/ks= google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754 h1:k5CJw9e5ONCcA/u0webKt092npXuY+KeGh3Q8NAVf0g= google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= @@ -97,8 +105,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -mvdan.cc/sh/v3 v3.12.0 h1:ejKUR7ONP5bb+UGHGEG/k9V5+pRVIyD+LsZz7o8KHrI= -mvdan.cc/sh/v3 v3.12.0/go.mod h1:Se6Cj17eYSn+sNooLZiEUnNNmNxg0imoYlTu4CyaGyg= mvdan.cc/sh/v3 v3.14.1 h1:bXkhQWNHCs0KZEChF8hYS6FC+T2N9mUZLbQv9blditI= mvdan.cc/sh/v3 v3.14.1/go.mod h1:syYCoFET8w9tvevxiXUtY8/ICrU+l26jHmhJDra3Vwo= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= diff --git a/images/ksail-analysis-runner/Dockerfile b/images/ksail-analysis-runner/Dockerfile new file mode 100644 index 0000000000..8584cb70cc --- /dev/null +++ b/images/ksail-analysis-runner/Dockerfile @@ -0,0 +1,39 @@ +FROM ghcr.io/actions/actions-runner@sha256:e5496277be5d09bc968b3d64911b74e219ac4a3f2edce956a3ecf9271bea1ef4 + +ARG TARGETARCH +USER 0:0 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN printf 'APT::Snapshot "20261005T000000Z";\n' > /etc/apt/apt.conf.d/99ksail-analysis-snapshot \ + && apt-get update \ + && apt-get install --yes --no-install-recommends \ + build-essential=12.10ubuntu1 pkg-config=1.8.1-2build1 \ + libgtk-4-dev=4.14.5+ds-0ubuntu0.10 \ + libwebkitgtk-6.0-dev=2.52.6-0ubuntu0.24.04.1 \ + xz-utils=5.6.1+really5.4.5-1ubuntu0.3 ca-certificates=20260601~24.04.1 \ + && rm -rf /var/lib/apt/lists/* + +RUN set -eu; \ + case "${TARGETARCH}" in \ + amd64) go_arch=amd64; go_sha=d0f743b33e8d8945e6b1f432edd15785c70507121d6e2a723b21285eddf8b57b; \ + node_arch=x64; node_sha=df450af89261115ef9f9e3830c3eeb2cc9213b63c720b1af623cb5dcbe2e02de ;; \ + arm64) go_arch=arm64; go_sha=211ffced9dcb9633a55eac6364816ec0ddd951389a740e88fa8b3337971bdda0; \ + node_arch=arm64; node_sha=a44aeb94849a299b22df10b9e622ec2f605c2183501bc40590705131de7c740f ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \ + esac; \ + curl --fail --silent --show-error --location --max-time 120 \ + "https://go.dev/dl/go1.26.8.linux-${go_arch}.tar.gz" --output /tmp/go.tar.gz; \ + printf '%s /tmp/go.tar.gz\n' "${go_sha}" | sha256sum --check --strict; \ + tar -xzf /tmp/go.tar.gz -C /usr/local; \ + curl --fail --silent --show-error --location --max-time 120 \ + "https://nodejs.org/dist/v22.23.3/node-v22.23.3-linux-${node_arch}.tar.xz" --output /tmp/node.tar.xz; \ + printf '%s /tmp/node.tar.xz\n' "${node_sha}" | sha256sum --check --strict; \ + mkdir -p /opt/node; \ + tar -xJf /tmp/node.tar.xz -C /opt/node --strip-components=1; \ + rm -f /tmp/go.tar.gz /tmp/node.tar.xz + +COPY --chmod=0555 smoke.sh /usr/local/bin/ksail-analysis-smoke +ENV PATH="/usr/local/go/bin:/opt/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + GOFLAGS="-tags=desktop" \ + GOMEMLIMIT="8GiB" \ + CGO_ENABLED="1" +USER 1001:1001 diff --git a/images/ksail-analysis-runner/smoke.sh b/images/ksail-analysis-runner/smoke.sh new file mode 100644 index 0000000000..648ef3bbce --- /dev/null +++ b/images/ksail-analysis-runner/smoke.sh @@ -0,0 +1,40 @@ +#!/bin/sh +set -eu + +test "$(id -u)" = 1001 +test "$(id -g)" = 1001 +test "$(awk '$1 == "CapEff:" {print $2}' /proc/self/status)" = 0000000000000000 +if touch /usr/local/ksail-analysis-write-probe 2>/dev/null; then + echo 'Root filesystem unexpectedly writable' >&2 + exit 1 +fi +test "$(go version)" = "go version go1.26.8 linux/$(go env GOARCH)" +test "$(node --version)" = v22.23.3 +test "$(go env CGO_ENABLED)" = 1 +test "$(go env GOFLAGS)" = -tags=desktop +pkg-config --exists gtk4 webkitgtk-6.0 + +# Mirrors the non-root init container's copy into its writable emptyDir. +cp -R /home/runner/. "${HOME}/" +mkdir -p "${HOME}/_diag" "${HOME}/_work/_tool" "${HOME}/test" +printf '{}\n' >"${HOME}/.runner" +"${HOME}/bin/Runner.Listener" --version + +cd "${HOME}/test" +cat >main.go <<'GO' +package main + +/* +#cgo pkg-config: gtk4 webkitgtk-6.0 +#include +#include +*/ +import "C" + +import "fmt" + +func main() { fmt.Println("desktop compiler and headers work") } +GO +GOPROXY=off GOTOOLCHAIN=local go build -o desktop-compiler-check main.go +./desktop-compiler-check +printf 'PASS: non-root analysis toolchain and writable runner home\n' diff --git a/k8s/bases/infrastructure/cluster-policies/best-practices/verify-app-images.yaml b/k8s/bases/infrastructure/cluster-policies/best-practices/verify-app-images.yaml index 0b36d47407..8d36445adf 100644 --- a/k8s/bases/infrastructure/cluster-policies/best-practices/verify-app-images.yaml +++ b/k8s/bases/infrastructure/cluster-policies/best-practices/verify-app-images.yaml @@ -104,6 +104,12 @@ spec: identities: - issuer: https://token.actions.githubusercontent.com subjectRegExp: ^https://github\.com/devantler-tech/platform/\.github/workflows/publish-coroot-node-agent-hotfix\.yaml@refs/heads/main$ + - name: publishksailanalysis + cosign: + keyless: + identities: + - issuer: https://token.actions.githubusercontent.com + subjectRegExp: ^https://github\.com/devantler-tech/platform/\.github/workflows/publish-ksail-analysis-runner\.yaml@refs/heads/main$ # The private scripted zone trial has its own repository-owned release # workflow. This attestor is used only for that exact image repository. - name: publishwarzone @@ -179,7 +185,10 @@ spec: !image.startsWith('ghcr.io/devantler-tech/platform-kubescape-node-agent@') && image != 'ghcr.io/devantler-tech/platform-coroot-node-agent' && !image.startsWith('ghcr.io/devantler-tech/platform-coroot-node-agent:') && - !image.startsWith('ghcr.io/devantler-tech/platform-coroot-node-agent@')) + !image.startsWith('ghcr.io/devantler-tech/platform-coroot-node-agent@') && + image != 'ghcr.io/devantler-tech/platform-ksail-analysis-runner' && + !image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner:') && + !image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner@')) .map(image, verifyImageSignatures(image, [attestors.publishapp])) .all(e, e > 0) message: >- @@ -230,6 +239,17 @@ spec: message: >- Coroot node-agent compatibility image is not signed by the platform repository's dedicated main-branch publisher (keyless cosign) + - expression: >- + (images.containers + images.initContainers + images.ephemeralContainers) + .filter(image, + image == 'ghcr.io/devantler-tech/platform-ksail-analysis-runner' || + image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner:') || + image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner@')) + .map(image, verifyImageSignatures(image, [attestors.publishksailanalysis])) + .all(e, e > 0) + message: >- + KSail analysis runner image is not signed by the platform repository's + dedicated main-branch publisher (keyless cosign) - expression: >- (images.containers + images.initContainers + images.ephemeralContainers) .filter(image, image.startsWith('ghcr.io/devantler-tech/provider-upjet-')) diff --git a/scripts/tests/arc-staging/image_routing_test.go b/scripts/tests/arc-staging/image_routing_test.go new file mode 100644 index 0000000000..5e4a3bbaa3 --- /dev/null +++ b/scripts/tests/arc-staging/image_routing_test.go @@ -0,0 +1,199 @@ +package arcstaging_test + +import ( + "reflect" + "regexp" + "slices" + "strings" + "testing" + + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" + "github.com/google/cel-go/common/types/traits" +) + +// Evaluate the committed CEL unchanged. Only the remote cryptographic lookup +// is replaced with deterministic certificate fixtures; routing is not mocked. +func TestAnalysisImageAdmissionRoutingAndSignerIsolation(t *testing.T) { + policy := readYAML(t, "k8s/bases/infrastructure/cluster-policies/best-practices/verify-app-images.yaml") + evaluator := newImagePolicyEvaluator(t, policy) + tests := []struct { + repository, attestor, subject string + }{ + {analysisRepository, "publishksailanalysis", analysisPublisher}, + {analysisRepository + "-other", "publishapp", "https://github.com/devantler-tech/actions/.github/workflows/publish-app.yaml@" + strings.Repeat("a", 40)}, + {"ghcr.io/devantler-tech/wedding-app", "publishapp", "https://github.com/devantler-tech/actions/.github/workflows/publish-app.yaml@" + strings.Repeat("b", 40)}, + {"ghcr.io/devantler-tech/platform-kubescape-storage", "publishkubescapestorage", "https://github.com/devantler-tech/platform/.github/workflows/publish-kubescape-storage-hotfix.yaml@refs/heads/main"}, + {"ghcr.io/devantler-tech/platform-kubescape-node-agent", "publishkubescapenodeagent", "https://github.com/devantler-tech/platform/.github/workflows/publish-kubescape-node-agent-hotfix.yaml@refs/heads/main"}, + {"ghcr.io/devantler-tech/platform-coroot-node-agent", "publishcorootnodeagent", "https://github.com/devantler-tech/platform/.github/workflows/publish-coroot-node-agent-hotfix.yaml@refs/heads/main"}, + {"ghcr.io/devantler-tech/world-at-ruin/zone", "publishwarzone", "https://github.com/devantler-tech/world-at-ruin/.github/workflows/server-cd.yaml@refs/tags/v1.2.3"}, + {"ghcr.io/devantler-tech/provider-upjet-unifi", "publishprovider", "https://github.com/devantler-tech/provider-upjet-unifi/.github/workflows/publish-provider-package.yml@refs/tags/v1.2.3"}, + {"ghcr.io/devantler-tech/ksail", "ksailcd", "https://github.com/devantler-tech/ksail/.github/workflows/cd.yaml@refs/tags/v1.2.3"}, + {"ghcr.io/actions/actions-runner", "", ""}, + } + for _, fixture := range tests { + for _, suffix := range []string{"", ":revision", "@sha256:" + strings.Repeat("a", 64), ":revision@sha256:" + strings.Repeat("a", 64)} { + image := fixture.repository + suffix + for _, slot := range []string{"containers", "initContainers", "ephemeralContainers"} { + t.Run(image+"/"+slot, func(t *testing.T) { + for _, control := range []struct { + name, subject, issuer string + want bool + }{ + {"correct signer", fixture.subject, githubIssuer, true}, + {"unsigned", "", "", fixture.attestor == ""}, + {"wrong issuer", fixture.subject, "https://other.example", fixture.attestor == ""}, + {"cross workflow signer", crossSigner(fixture.subject), githubIssuer, fixture.attestor == ""}, + {"wrong ref", strings.Split(fixture.subject, "@")[0] + "@refs/heads/other", githubIssuer, fixture.attestor == ""}, + {"wrong repository", strings.Replace(fixture.subject, "devantler-tech/", "other-owner/", 1), githubIssuer, fixture.attestor == ""}, + } { + t.Run(control.name, func(t *testing.T) { + images := map[string][]string{"containers": {}, "initContainers": {}, "ephemeralContainers": {}} + images[slot] = []string{image} + admitted, calls := evaluator.evaluate(t, images, map[string]certificateFixture{ + image: {issuer: control.issuer, subject: control.subject}, + }) + wantCalls := []string{} + if fixture.attestor != "" { + wantCalls = []string{image + "/" + fixture.attestor} + } + if !reflect.DeepEqual(calls, wantCalls) { + t.Fatalf("verification calls = %v, want %v", calls, wantCalls) + } + equal(t, admitted, control.want) + }) + } + }) + } + } + } +} + +type certificateFixture struct { + issuer, subject string +} + +type imagePolicyEvaluator struct { + programs []cel.Program + attestors map[string]string + certificates map[string]certificateFixture + calls []string +} + +func newImagePolicyEvaluator(t *testing.T, policy map[string]any) *imagePolicyEvaluator { + t.Helper() + attestors := map[string]string{} + identities := map[string]map[string]any{} + for _, item := range field(t, policy, "spec", "attestors").([]any) { + name := field(t, item, "name").(string) + entries := field(t, item, "cosign", "keyless", "identities").([]any) + if len(entries) != 1 { + t.Fatalf("%s must have exactly one identity", name) + } + if _, duplicate := identities[name]; duplicate { + t.Fatalf("duplicate attestor %s", name) + } + attestors[name] = name + identities[name] = entries[0].(map[string]any) + } + if _, exists := identities["publishksailanalysis"]; !exists { + t.Fatal("analysis image lacks an independent attestor") + } + evaluator := &imagePolicyEvaluator{attestors: attestors} + env, err := cel.NewEnv( + cel.Variable("images", cel.MapType(cel.StringType, cel.ListType(cel.StringType))), + cel.Variable("attestors", cel.MapType(cel.StringType, cel.StringType)), + cel.Function("verifyImageSignatures", cel.Overload("verify_image_fixture", + []*cel.Type{cel.StringType, cel.ListType(cel.StringType)}, cel.IntType, + cel.BinaryBinding(func(imageValue, attestorValues ref.Val) ref.Val { + list := attestorValues.(traits.Lister) + if list.Size() != types.Int(1) { + return types.NewErr("exactly one attestor required") + } + name := list.Get(types.Int(0)).Value().(string) + image := imageValue.Value().(string) + evaluator.calls = append(evaluator.calls, image+"/"+name) + certificate := evaluator.certificates[image] + identity := identities[name] + if identity["issuer"] == certificate.issuer && + regexp.MustCompile(identity["subjectRegExp"].(string)).MatchString(certificate.subject) { + return types.Int(1) + } + return types.Int(0) + }))), + ) + if err != nil { + t.Fatal(err) + } + for _, validation := range field(t, policy, "spec", "validations").([]any) { + ast, issues := env.Compile(field(t, validation, "expression").(string)) + if issues.Err() != nil { + t.Fatal(issues.Err()) + } + program, err := env.Program(ast) + if err != nil { + t.Fatal(err) + } + evaluator.programs = append(evaluator.programs, program) + } + return evaluator +} + +// Each test owns an evaluator and uses its certificate fixtures sequentially. +func (e *imagePolicyEvaluator) evaluate(t *testing.T, images map[string][]string, certificates map[string]certificateFixture) (bool, []string) { + t.Helper() + e.certificates = certificates + e.calls = []string{} + admitted := true + for _, program := range e.programs { + result, _, err := program.Eval(map[string]any{"images": images, "attestors": e.attestors}) + if err != nil { + t.Fatal(err) + } + value, ok := result.Value().(bool) + if !ok { + t.Fatalf("validation returned %T", result.Value()) + } + admitted = admitted && value + } + return admitted, e.calls +} + +func TestMixedImagePodCannotHideOneUnsignedImage(t *testing.T) { + policy := readYAML(t, "k8s/bases/infrastructure/cluster-policies/best-practices/verify-app-images.yaml") + evaluator := newImagePolicyEvaluator(t, policy) + analysis := analysisRepository + ":revision" + app := "ghcr.io/devantler-tech/wedding-app:revision" + coroot := "ghcr.io/devantler-tech/platform-coroot-node-agent:revision" + images := map[string][]string{ + "containers": {analysis, app}, + "initContainers": {coroot}, + "ephemeralContainers": {"ghcr.io/actions/actions-runner:revision"}, + } + for _, unsigned := range []string{"", analysis, app, coroot} { + t.Run(unsigned, func(t *testing.T) { + certificates := map[string]certificateFixture{ + analysis: {githubIssuer, analysisPublisher}, + app: {githubIssuer, "https://github.com/devantler-tech/actions/.github/workflows/publish-app.yaml@" + strings.Repeat("a", 40)}, + coroot: {githubIssuer, "https://github.com/devantler-tech/platform/.github/workflows/publish-coroot-node-agent-hotfix.yaml@refs/heads/main"}, + } + delete(certificates, unsigned) + admitted, calls := evaluator.evaluate(t, images, certificates) + slices.Sort(calls) + wantCalls := []string{analysis + "/publishksailanalysis", app + "/publishapp", coroot + "/publishcorootnodeagent"} + slices.Sort(wantCalls) + if !reflect.DeepEqual(calls, wantCalls) { + t.Fatalf("mixed verification calls = %v, want %v", calls, wantCalls) + } + equal(t, admitted, unsigned == "") + }) + } +} + +func crossSigner(subject string) string { + if subject == analysisPublisher { + return "https://github.com/devantler-tech/actions/.github/workflows/publish-app.yaml@" + strings.Repeat("a", 40) + } + return analysisPublisher +} diff --git a/scripts/tests/arc-staging/image_test.go b/scripts/tests/arc-staging/image_test.go new file mode 100644 index 0000000000..c8ac0d1670 --- /dev/null +++ b/scripts/tests/arc-staging/image_test.go @@ -0,0 +1,81 @@ +package arcstaging_test + +import ( + "path" + "regexp" + "strings" + "testing" +) + +const analysisRepository = "ghcr.io/devantler-tech/platform-ksail-analysis-runner" +const analysisPublisher = "https://github.com/devantler-tech/platform/.github/workflows/publish-ksail-analysis-runner.yaml@refs/heads/main" +const githubIssuer = "https://token.actions.githubusercontent.com" + +func TestAnalysisImageCanBePublishedOnlyFromMain(t *testing.T) { + workflow := readYAML(t, ".github/workflows/publish-ksail-analysis-runner.yaml") + equal(t, len(field(t, workflow, "permissions").(map[string]any)), 0) + publish := field(t, workflow, "jobs", "publish") + equal(t, field(t, publish, "if"), "github.event_name == 'push' && github.ref == 'refs/heads/main'") + equal(t, field(t, publish, "needs"), "verify") + equal(t, field(t, publish, "permissions", "contents"), "read") + equal(t, field(t, publish, "permissions", "packages"), "write") + equal(t, field(t, publish, "permissions", "id-token"), "write") + equal(t, field(t, workflow, "env", "IMAGE"), analysisRepository) + equal(t, field(t, workflow, "jobs", "verify", "permissions", "contents"), "read") + equal(t, len(field(t, workflow, "jobs", "verify", "permissions").(map[string]any)), 1) + for _, job := range field(t, workflow, "jobs").(map[string]any) { + for _, step := range field(t, job, "steps").([]any) { + mapping := step.(map[string]any) + if uses, ok := mapping["uses"].(string); ok { + if !regexp.MustCompile(`@[0-9a-f]{40}$`).MatchString(uses) { + t.Fatalf("action must be immutable: %q", uses) + } + } + } + } +} + +func TestAnalysisImageUsesItsOwnExactNodeSigner(t *testing.T) { + policy := readYAML(t, "talos/cluster/verify-first-party-images.yaml") + rules := field(t, policy, "rules").([]any) + images := []string{ + analysisRepository, analysisRepository + ":revision", + analysisRepository + "@sha256:" + strings.Repeat("a", 64), + analysisRepository + "-other:revision", + "ghcr.io/devantler-tech/platform-coroot-node-agent:revision", + "ghcr.io/devantler-tech/wedding-app:revision", + } + for _, image := range images { + t.Run(image, func(t *testing.T) { + repository := strings.Split(strings.Split(image, "@")[0], ":")[0] + for _, rule := range rules { + pattern := field(t, rule, "image").(string) + match, err := path.Match(pattern, repository) + if err != nil { + t.Fatal(err) + } + if !match { + continue + } + equal(t, field(t, rule, "keyless", "issuer"), githubIssuer) + regex := regexp.MustCompile(field(t, rule, "keyless", "subjectRegex").(string)) + equal(t, regex.MatchString(analysisPublisher), repository == analysisRepository) + if repository == analysisRepository { + for _, forbidden := range []string{ + strings.Replace(analysisPublisher, "refs/heads/main", "refs/pull/1/merge", 1), + strings.Replace(analysisPublisher, "refs/heads/main", "refs/heads/other", 1), + strings.Replace(analysisPublisher, "platform/", "other/", 1), + strings.Replace(analysisPublisher, "publish-ksail-analysis-runner", "publish-coroot-node-agent-hotfix", 1), + analysisPublisher + ".other", "https://other.example/" + analysisPublisher, + } { + if regex.MatchString(forbidden) { + t.Fatalf("unexpected signer admitted: %q", forbidden) + } + } + } + return + } + t.Fatal("no image verification rule matched") + }) + } +} diff --git a/scripts/tests/refresh-flux-ghcr-auth/contracts_test.go b/scripts/tests/refresh-flux-ghcr-auth/contracts_test.go index 67eb5500f1..786e3462b3 100644 --- a/scripts/tests/refresh-flux-ghcr-auth/contracts_test.go +++ b/scripts/tests/refresh-flux-ghcr-auth/contracts_test.go @@ -639,6 +639,7 @@ func TestImageVerificationUsesOneSlowFailClosedPolicy(t *testing.T) { "name: publishkubescapestorage", "name: publishkubescapenodeagent", "name: publishcorootnodeagent", + "name: publishksailanalysis", "name: publishprovider", "name: ksailcd", "^https://github\\.com/devantler-tech/platform/\\.github/workflows/publish-kubescape-storage-hotfix\\.yaml@refs/heads/main$", @@ -655,6 +656,10 @@ func TestImageVerificationUsesOneSlowFailClosedPolicy(t *testing.T) { "image != 'ghcr.io/devantler-tech/platform-coroot-node-agent'", "image == 'ghcr.io/devantler-tech/platform-coroot-node-agent'", "verifyImageSignatures(image, [attestors.publishcorootnodeagent])", + "^https://github\\.com/devantler-tech/platform/\\.github/workflows/publish-ksail-analysis-runner\\.yaml@refs/heads/main$", + "image != 'ghcr.io/devantler-tech/platform-ksail-analysis-runner'", + "image == 'ghcr.io/devantler-tech/platform-ksail-analysis-runner'", + "verifyImageSignatures(image, [attestors.publishksailanalysis])", "image.startsWith('ghcr.io/devantler-tech/ksail')", } { requireContains(t, policy, expected) diff --git a/scripts/tests/refresh-flux-ghcr-auth/fake_kubectl_test.go b/scripts/tests/refresh-flux-ghcr-auth/fake_kubectl_test.go index d44aa229ad..dd86e36df9 100644 --- a/scripts/tests/refresh-flux-ghcr-auth/fake_kubectl_test.go +++ b/scripts/tests/refresh-flux-ghcr-auth/fake_kubectl_test.go @@ -1108,17 +1108,18 @@ func fakeKubectlPatchConsolidatedImageValidatingPolicy(args []string, patchFile spec, _ := patch["spec"].(map[string]any) webhookConfiguration, _ := spec["webhookConfiguration"].(map[string]any) attestors, _ := spec["attestors"].([]any) - if webhookConfiguration["timeoutSeconds"] != float64(30) || len(attestors) != 7 { + if webhookConfiguration["timeoutSeconds"] != float64(30) || len(attestors) != 8 { return commandFailure(91, "consolidated image-validating policy patch omitted its timeout or attestors") } storageAttestorValid := false kubescapeNodeAgentAttestorValid := false corootNodeAgentAttestorValid := false + analysisAttestorValid := false warZoneAttestorValid := false for _, rawAttestor := range attestors { attestor, _ := rawAttestor.(map[string]any) name, _ := attestor["name"].(string) - if name != "publishkubescapestorage" && name != "publishkubescapenodeagent" && name != "publishcorootnodeagent" && name != "publishwarzone" { + if name != "publishkubescapestorage" && name != "publishkubescapenodeagent" && name != "publishcorootnodeagent" && name != "publishwarzone" && name != "publishksailanalysis" { continue } cosign, _ := attestor["cosign"].(map[string]any) @@ -1137,6 +1138,8 @@ func fakeKubectlPatchConsolidatedImageValidatingPolicy(args []string, patchFile kubescapeNodeAgentAttestorValid = issuerValid && subject == "^https://github\\.com/devantler-tech/platform/\\.github/workflows/publish-kubescape-node-agent-hotfix\\.yaml@refs/heads/main$" case "publishcorootnodeagent": corootNodeAgentAttestorValid = issuerValid && subject == "^https://github\\.com/devantler-tech/platform/\\.github/workflows/publish-coroot-node-agent-hotfix\\.yaml@refs/heads/main$" + case "publishksailanalysis": + analysisAttestorValid = issuerValid && subject == "^https://github\\.com/devantler-tech/platform/\\.github/workflows/publish-ksail-analysis-runner\\.yaml@refs/heads/main$" case "publishwarzone": warZoneAttestorValid = issuerValid && subject == "^https://github\\.com/devantler-tech/world-at-ruin/\\.github/workflows/server-cd\\.yaml@refs/tags/v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$" } @@ -1148,6 +1151,8 @@ func fakeKubectlPatchConsolidatedImageValidatingPolicy(args []string, patchFile dedicatedRoutesKubescapeNodeAgent := false genericExcludesCorootNodeAgent := false dedicatedRoutesCorootNodeAgent := false + genericExcludesAnalysis := false + dedicatedRoutesAnalysis := false genericExcludesWarZone := false dedicatedRoutesWarZone := false for _, rawValidation := range validations { @@ -1166,6 +1171,9 @@ func fakeKubectlPatchConsolidatedImageValidatingPolicy(args []string, patchFile genericExcludesCorootNodeAgent = strings.Contains(expression, "image != 'ghcr.io/devantler-tech/platform-coroot-node-agent'") && strings.Contains(expression, "!image.startsWith('ghcr.io/devantler-tech/platform-coroot-node-agent:')") && strings.Contains(expression, "!image.startsWith('ghcr.io/devantler-tech/platform-coroot-node-agent@')") + genericExcludesAnalysis = strings.Contains(expression, "image != 'ghcr.io/devantler-tech/platform-ksail-analysis-runner'") && + strings.Contains(expression, "!image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner:')") && + strings.Contains(expression, "!image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner@')") } if strings.Contains(expression, "attestors.publishkubescapestorage") { dedicatedRoutesStorage = strings.Contains(expression, "image == 'ghcr.io/devantler-tech/platform-kubescape-storage'") && @@ -1187,8 +1195,14 @@ func fakeKubectlPatchConsolidatedImageValidatingPolicy(args []string, patchFile strings.Contains(expression, "image.startsWith('ghcr.io/devantler-tech/world-at-ruin/zone:')") && strings.Contains(expression, "image.startsWith('ghcr.io/devantler-tech/world-at-ruin/zone@')") } + if strings.Contains(expression, "attestors.publishksailanalysis") { + dedicatedRoutesAnalysis = strings.Contains(expression, "image == 'ghcr.io/devantler-tech/platform-ksail-analysis-runner'") && + strings.Contains(expression, "image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner:')") && + strings.Contains(expression, "image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner@')") + } } if !storageAttestorValid || !kubescapeNodeAgentAttestorValid || !corootNodeAgentAttestorValid || + !analysisAttestorValid || !genericExcludesAnalysis || !dedicatedRoutesAnalysis || !warZoneAttestorValid || !genericExcludesWarZone || !dedicatedRoutesWarZone || !genericExcludesStorage || !genericExcludesKubescapeNodeAgent || !genericExcludesCorootNodeAgent || !dedicatedRoutesStorage || !dedicatedRoutesKubescapeNodeAgent || !dedicatedRoutesCorootNodeAgent { diff --git a/scripts/tests/test-inventory-first-party-image-signatures.sh b/scripts/tests/test-inventory-first-party-image-signatures.sh index a8b78253f0..d4e8d42741 100755 --- a/scripts/tests/test-inventory-first-party-image-signatures.sh +++ b/scripts/tests/test-inventory-first-party-image-signatures.sh @@ -252,31 +252,27 @@ RC=$? set -e check "an unreadable rules file -> exit 2" 2 'not readable' "$RC" "$OUT" -# --- 11. the REAL rules file parses, and keeps its seven ordered rules ------ +# --- 11. the REAL rules file parses, and keeps its ordered rules ------ # Guards the shape this script reads from drifting without anyone noticing: the catch-all must stay # LAST, or ksail, provider packages, and the compatibility images get held to the app # identity and go ImagePullBackOff. real="${repo_root}/talos/cluster/verify-first-party-images.yaml" # macOS ships bash 3.2, which has no `mapfile` — read the globs without it. real_globs="$(yq -r '.rules[].image' "$real")" -real_count="$(printf '%s\n' "$real_globs" | grep -c .)" -real_last="$(printf '%s\n' "$real_globs" | grep . | tail -1)" -real_compatibility_rules="$(printf '%s\n' "$real_globs" | grep . | tail -5 | head -3)" -expected_compatibility_rules="$(printf '%s\n' \ +expected_globs="$(printf '%s\n' \ + 'ghcr.io/devantler-tech/ksail*' \ + 'ghcr.io/devantler-tech/provider-upjet-*' \ 'ghcr.io/devantler-tech/platform-kubescape-storage' \ 'ghcr.io/devantler-tech/platform-kubescape-node-agent' \ - 'ghcr.io/devantler-tech/platform-coroot-node-agent')" -if [ "$real_count" -ne 7 ]; then - echo "FAIL the real rules file has ${real_count} rules, expected 7" - failures=$((failures + 1)) -elif [ "$real_compatibility_rules" != "$expected_compatibility_rules" ]; then - echo "FAIL the three dedicated compatibility rules must precede the exact zone rule and catch-all" - failures=$((failures + 1)) -elif [ "$real_last" != 'ghcr.io/devantler-tech/*' ]; then - echo "FAIL the catch-all is not the last rule (${real_last})" + 'ghcr.io/devantler-tech/platform-coroot-node-agent' \ + 'ghcr.io/devantler-tech/platform-ksail-analysis-runner' \ + 'ghcr.io/devantler-tech/world-at-ruin/zone' \ + 'ghcr.io/devantler-tech/*')" +if [ "$real_globs" != "$expected_globs" ]; then + echo "FAIL the real rules must retain all eight exact ordered image routes" failures=$((failures + 1)) else - echo "ok real rules: seven rules, dedicated compatibility and zone rules before catch-all" + echo "ok real rules: eight ordered rules, dedicated publishers before catch-all" fi # The real file must also satisfy the completeness gate the fixtures exercise. diff --git a/talos/cluster/verify-first-party-images.yaml b/talos/cluster/verify-first-party-images.yaml index 2bbb451203..d0fe713f62 100644 --- a/talos/cluster/verify-first-party-images.yaml +++ b/talos/cluster/verify-first-party-images.yaml @@ -168,6 +168,11 @@ rules: keyless: issuer: https://token.actions.githubusercontent.com subjectRegex: ^https://github\.com/devantler-tech/platform/\.github/workflows/publish-coroot-node-agent-hotfix\.yaml@refs/heads/main$ + # The repository-scoped analysis runner has its own main-only publisher. + - image: ghcr.io/devantler-tech/platform-ksail-analysis-runner + keyless: + issuer: https://token.actions.githubusercontent.com + subjectRegex: ^https://github\.com/devantler-tech/platform/\.github/workflows/publish-ksail-analysis-runner\.yaml@refs/heads/main$ # Exact zone repository, before the generic first-party app rule. The private # trial uses the game's own stable release workflow, not the shared publisher. - image: ghcr.io/devantler-tech/world-at-ruin/zone From 0e02a362ab5555cbac58c21a4f1a7fede9d238c6 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 5 Oct 2026 22:30:54 +0200 Subject: [PATCH 03/62] fix(ci): allow smoke binaries on disposable test mounts --- .github/workflows/publish-ksail-analysis-runner.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-ksail-analysis-runner.yaml b/.github/workflows/publish-ksail-analysis-runner.yaml index 5d4b48a173..4ca931ba20 100644 --- a/.github/workflows/publish-ksail-analysis-runner.yaml +++ b/.github/workflows/publish-ksail-analysis-runner.yaml @@ -51,8 +51,8 @@ jobs: docker run --rm --platform linux/amd64 --user 1001:1001 \ --read-only --cap-drop ALL --security-opt no-new-privileges \ --memory 4g --cpus 2 --env HOME=/runner-data \ - --tmpfs /runner-data:rw,uid=1001,gid=1001,size=2g \ - --tmpfs /tmp:rw,uid=1001,gid=1001,size=1g \ + --tmpfs /runner-data:rw,exec,nosuid,nodev,uid=1001,gid=1001,size=2g \ + --tmpfs /tmp:rw,exec,nosuid,nodev,uid=1001,gid=1001,size=1g \ ksail-analysis:verify /usr/local/bin/ksail-analysis-smoke publish: From b6bb8599a37e1863dd7656d7293da0c8ff625d6e Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 5 Oct 2026 23:00:28 +0200 Subject: [PATCH 04/62] fix(ci): consume the KSail-owned analysis image publisher --- .../publish-ksail-analysis-runner.yaml | 119 ------------------ docs/operations/ksail-arc.md | 5 +- images/ksail-analysis-runner/Dockerfile | 39 ------ images/ksail-analysis-runner/smoke.sh | 40 ------ .../best-practices/verify-app-images.yaml | 20 +-- .../tests/arc-staging/image_routing_test.go | 3 +- scripts/tests/arc-staging/image_test.go | 30 +---- .../refresh-flux-ghcr-auth/contracts_test.go | 6 +- .../fake_kubectl_test.go | 14 +-- ...-inventory-first-party-image-signatures.sh | 6 +- talos/cluster/verify-first-party-images.yaml | 20 +-- 11 files changed, 44 insertions(+), 258 deletions(-) delete mode 100644 .github/workflows/publish-ksail-analysis-runner.yaml delete mode 100644 images/ksail-analysis-runner/Dockerfile delete mode 100644 images/ksail-analysis-runner/smoke.sh diff --git a/.github/workflows/publish-ksail-analysis-runner.yaml b/.github/workflows/publish-ksail-analysis-runner.yaml deleted file mode 100644 index 4ca931ba20..0000000000 --- a/.github/workflows/publish-ksail-analysis-runner.yaml +++ /dev/null @@ -1,119 +0,0 @@ -name: Publish KSail Analysis Runner - -on: - pull_request: - branches: [main] - paths: - - .github/workflows/publish-ksail-analysis-runner.yaml - - images/ksail-analysis-runner/** - - scripts/tests/arc-staging/** - push: - branches: [main] - paths: - - .github/workflows/publish-ksail-analysis-runner.yaml - - images/ksail-analysis-runner/** - - scripts/tests/arc-staging/** - workflow_dispatch: - -permissions: {} - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false - -env: - IMAGE: ghcr.io/devantler-tech/platform-ksail-analysis-runner - IMAGE_TAG: ${{ github.sha }} - -jobs: - verify: - name: Verify restricted analysis toolchain - runs-on: ubuntu-24.04 - permissions: - contents: read - steps: - - name: Harden runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 - with: - egress-policy: audit - - - name: Check out source - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Build pinned analysis image - run: | - docker build --platform linux/amd64 --tag ksail-analysis:verify images/ksail-analysis-runner - - - name: Exercise runner bootstrap and desktop compiler - run: | - docker run --rm --platform linux/amd64 --user 1001:1001 \ - --read-only --cap-drop ALL --security-opt no-new-privileges \ - --memory 4g --cpus 2 --env HOME=/runner-data \ - --tmpfs /runner-data:rw,exec,nosuid,nodev,uid=1001,gid=1001,size=2g \ - --tmpfs /tmp:rw,exec,nosuid,nodev,uid=1001,gid=1001,size=1g \ - ksail-analysis:verify /usr/local/bin/ksail-analysis-smoke - - publish: - name: Publish and sign immutable analysis image - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - needs: verify - runs-on: ubuntu-24.04 - permissions: - contents: read - packages: write - id-token: write - steps: - - name: Harden runner - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 - with: - egress-policy: audit - - - name: Check out source - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Set up Buildx - uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4.4.0 - with: - driver: docker-container - - - name: Log in to GHCR - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push immutable analysis image - id: build - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 - with: - context: ./images/ksail-analysis-runner - platforms: linux/amd64 - push: true - tags: ${{ env.IMAGE }}:${{ env.IMAGE_TAG }} - provenance: mode=max - sbom: true - labels: | - org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} - org.opencontainers.image.revision=${{ github.sha }} - org.opencontainers.image.version=${{ env.IMAGE_TAG }} - - - name: Set up Cosign - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - - - name: Sign and verify published digest - env: - DIGEST: ${{ steps.build.outputs.digest }} - run: | - set -euo pipefail - test -n "${DIGEST}" - cosign sign --yes "${IMAGE}@${DIGEST}" - cosign verify \ - --certificate-identity "https://github.com/devantler-tech/platform/.github/workflows/publish-ksail-analysis-runner.yaml@refs/heads/main" \ - --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ - "${IMAGE}@${DIGEST}" - printf 'Published signed image: %s@%s\n' "${IMAGE}" "${DIGEST}" diff --git a/docs/operations/ksail-arc.md b/docs/operations/ksail-arc.md index b6b55f0daf..0a7b66ab1c 100644 --- a/docs/operations/ksail-arc.md +++ b/docs/operations/ksail-arc.md @@ -14,11 +14,12 @@ zero idle runners and a maximum of one job runner. There is no container mode, Docker socket, host volume, privileged container or runner API token. Container jobs and Docker actions are deliberately unsupported. -The analysis toolchain source lives in `images/ksail-analysis-runner`. Its +The analysis toolchain source lives in the KSail repository's +`images/ksail-analysis-runner`. Its digest-pinned upstream runner, frozen Ubuntu package snapshot and checksum-checked Go and Node archives supply the desktop compiler and headers without job-time root access. The publisher builds and exercises the image on pull requests, then -publishes, attests and signs only on a push to main. Both Kyverno and Talos accept +publishes, attests and signs only on a push to KSail main. Both Kyverno and Talos accept that workflow identity only for the exact analysis image repository. The staged pool still requires a verified published digest before activation. diff --git a/images/ksail-analysis-runner/Dockerfile b/images/ksail-analysis-runner/Dockerfile deleted file mode 100644 index 8584cb70cc..0000000000 --- a/images/ksail-analysis-runner/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -FROM ghcr.io/actions/actions-runner@sha256:e5496277be5d09bc968b3d64911b74e219ac4a3f2edce956a3ecf9271bea1ef4 - -ARG TARGETARCH -USER 0:0 -SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN printf 'APT::Snapshot "20261005T000000Z";\n' > /etc/apt/apt.conf.d/99ksail-analysis-snapshot \ - && apt-get update \ - && apt-get install --yes --no-install-recommends \ - build-essential=12.10ubuntu1 pkg-config=1.8.1-2build1 \ - libgtk-4-dev=4.14.5+ds-0ubuntu0.10 \ - libwebkitgtk-6.0-dev=2.52.6-0ubuntu0.24.04.1 \ - xz-utils=5.6.1+really5.4.5-1ubuntu0.3 ca-certificates=20260601~24.04.1 \ - && rm -rf /var/lib/apt/lists/* - -RUN set -eu; \ - case "${TARGETARCH}" in \ - amd64) go_arch=amd64; go_sha=d0f743b33e8d8945e6b1f432edd15785c70507121d6e2a723b21285eddf8b57b; \ - node_arch=x64; node_sha=df450af89261115ef9f9e3830c3eeb2cc9213b63c720b1af623cb5dcbe2e02de ;; \ - arm64) go_arch=arm64; go_sha=211ffced9dcb9633a55eac6364816ec0ddd951389a740e88fa8b3337971bdda0; \ - node_arch=arm64; node_sha=a44aeb94849a299b22df10b9e622ec2f605c2183501bc40590705131de7c740f ;; \ - *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \ - esac; \ - curl --fail --silent --show-error --location --max-time 120 \ - "https://go.dev/dl/go1.26.8.linux-${go_arch}.tar.gz" --output /tmp/go.tar.gz; \ - printf '%s /tmp/go.tar.gz\n' "${go_sha}" | sha256sum --check --strict; \ - tar -xzf /tmp/go.tar.gz -C /usr/local; \ - curl --fail --silent --show-error --location --max-time 120 \ - "https://nodejs.org/dist/v22.23.3/node-v22.23.3-linux-${node_arch}.tar.xz" --output /tmp/node.tar.xz; \ - printf '%s /tmp/node.tar.xz\n' "${node_sha}" | sha256sum --check --strict; \ - mkdir -p /opt/node; \ - tar -xJf /tmp/node.tar.xz -C /opt/node --strip-components=1; \ - rm -f /tmp/go.tar.gz /tmp/node.tar.xz - -COPY --chmod=0555 smoke.sh /usr/local/bin/ksail-analysis-smoke -ENV PATH="/usr/local/go/bin:/opt/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ - GOFLAGS="-tags=desktop" \ - GOMEMLIMIT="8GiB" \ - CGO_ENABLED="1" -USER 1001:1001 diff --git a/images/ksail-analysis-runner/smoke.sh b/images/ksail-analysis-runner/smoke.sh deleted file mode 100644 index 648ef3bbce..0000000000 --- a/images/ksail-analysis-runner/smoke.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh -set -eu - -test "$(id -u)" = 1001 -test "$(id -g)" = 1001 -test "$(awk '$1 == "CapEff:" {print $2}' /proc/self/status)" = 0000000000000000 -if touch /usr/local/ksail-analysis-write-probe 2>/dev/null; then - echo 'Root filesystem unexpectedly writable' >&2 - exit 1 -fi -test "$(go version)" = "go version go1.26.8 linux/$(go env GOARCH)" -test "$(node --version)" = v22.23.3 -test "$(go env CGO_ENABLED)" = 1 -test "$(go env GOFLAGS)" = -tags=desktop -pkg-config --exists gtk4 webkitgtk-6.0 - -# Mirrors the non-root init container's copy into its writable emptyDir. -cp -R /home/runner/. "${HOME}/" -mkdir -p "${HOME}/_diag" "${HOME}/_work/_tool" "${HOME}/test" -printf '{}\n' >"${HOME}/.runner" -"${HOME}/bin/Runner.Listener" --version - -cd "${HOME}/test" -cat >main.go <<'GO' -package main - -/* -#cgo pkg-config: gtk4 webkitgtk-6.0 -#include -#include -*/ -import "C" - -import "fmt" - -func main() { fmt.Println("desktop compiler and headers work") } -GO -GOPROXY=off GOTOOLCHAIN=local go build -o desktop-compiler-check main.go -./desktop-compiler-check -printf 'PASS: non-root analysis toolchain and writable runner home\n' diff --git a/k8s/bases/infrastructure/cluster-policies/best-practices/verify-app-images.yaml b/k8s/bases/infrastructure/cluster-policies/best-practices/verify-app-images.yaml index 8d36445adf..1602d9660d 100644 --- a/k8s/bases/infrastructure/cluster-policies/best-practices/verify-app-images.yaml +++ b/k8s/bases/infrastructure/cluster-policies/best-practices/verify-app-images.yaml @@ -109,7 +109,7 @@ spec: keyless: identities: - issuer: https://token.actions.githubusercontent.com - subjectRegExp: ^https://github\.com/devantler-tech/platform/\.github/workflows/publish-ksail-analysis-runner\.yaml@refs/heads/main$ + subjectRegExp: ^https://github\.com/devantler-tech/ksail/\.github/workflows/publish-ksail-analysis-runner\.yaml@refs/heads/main$ # The private scripted zone trial has its own repository-owned release # workflow. This attestor is used only for that exact image repository. - name: publishwarzone @@ -186,9 +186,9 @@ spec: image != 'ghcr.io/devantler-tech/platform-coroot-node-agent' && !image.startsWith('ghcr.io/devantler-tech/platform-coroot-node-agent:') && !image.startsWith('ghcr.io/devantler-tech/platform-coroot-node-agent@') && - image != 'ghcr.io/devantler-tech/platform-ksail-analysis-runner' && - !image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner:') && - !image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner@')) + image != 'ghcr.io/devantler-tech/ksail-analysis-runner' && + !image.startsWith('ghcr.io/devantler-tech/ksail-analysis-runner:') && + !image.startsWith('ghcr.io/devantler-tech/ksail-analysis-runner@')) .map(image, verifyImageSignatures(image, [attestors.publishapp])) .all(e, e > 0) message: >- @@ -242,9 +242,9 @@ spec: - expression: >- (images.containers + images.initContainers + images.ephemeralContainers) .filter(image, - image == 'ghcr.io/devantler-tech/platform-ksail-analysis-runner' || - image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner:') || - image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner@')) + image == 'ghcr.io/devantler-tech/ksail-analysis-runner' || + image.startsWith('ghcr.io/devantler-tech/ksail-analysis-runner:') || + image.startsWith('ghcr.io/devantler-tech/ksail-analysis-runner@')) .map(image, verifyImageSignatures(image, [attestors.publishksailanalysis])) .all(e, e > 0) message: >- @@ -260,7 +260,11 @@ spec: publish-provider-package workflow from a release tag (keyless cosign) - expression: >- (images.containers + images.initContainers + images.ephemeralContainers) - .filter(image, image.startsWith('ghcr.io/devantler-tech/ksail')) + .filter(image, + image.startsWith('ghcr.io/devantler-tech/ksail') && + image != 'ghcr.io/devantler-tech/ksail-analysis-runner' && + !image.startsWith('ghcr.io/devantler-tech/ksail-analysis-runner:') && + !image.startsWith('ghcr.io/devantler-tech/ksail-analysis-runner@')) .map(image, verifyImageSignatures(image, [attestors.ksailcd])) .all(e, e > 0) message: >- diff --git a/scripts/tests/arc-staging/image_routing_test.go b/scripts/tests/arc-staging/image_routing_test.go index 5e4a3bbaa3..0f3a43940f 100644 --- a/scripts/tests/arc-staging/image_routing_test.go +++ b/scripts/tests/arc-staging/image_routing_test.go @@ -22,7 +22,7 @@ func TestAnalysisImageAdmissionRoutingAndSignerIsolation(t *testing.T) { repository, attestor, subject string }{ {analysisRepository, "publishksailanalysis", analysisPublisher}, - {analysisRepository + "-other", "publishapp", "https://github.com/devantler-tech/actions/.github/workflows/publish-app.yaml@" + strings.Repeat("a", 40)}, + {analysisRepository + "-other", "ksailcd", "https://github.com/devantler-tech/ksail/.github/workflows/cd.yaml@refs/tags/v1.2.3"}, {"ghcr.io/devantler-tech/wedding-app", "publishapp", "https://github.com/devantler-tech/actions/.github/workflows/publish-app.yaml@" + strings.Repeat("b", 40)}, {"ghcr.io/devantler-tech/platform-kubescape-storage", "publishkubescapestorage", "https://github.com/devantler-tech/platform/.github/workflows/publish-kubescape-storage-hotfix.yaml@refs/heads/main"}, {"ghcr.io/devantler-tech/platform-kubescape-node-agent", "publishkubescapenodeagent", "https://github.com/devantler-tech/platform/.github/workflows/publish-kubescape-node-agent-hotfix.yaml@refs/heads/main"}, @@ -30,6 +30,7 @@ func TestAnalysisImageAdmissionRoutingAndSignerIsolation(t *testing.T) { {"ghcr.io/devantler-tech/world-at-ruin/zone", "publishwarzone", "https://github.com/devantler-tech/world-at-ruin/.github/workflows/server-cd.yaml@refs/tags/v1.2.3"}, {"ghcr.io/devantler-tech/provider-upjet-unifi", "publishprovider", "https://github.com/devantler-tech/provider-upjet-unifi/.github/workflows/publish-provider-package.yml@refs/tags/v1.2.3"}, {"ghcr.io/devantler-tech/ksail", "ksailcd", "https://github.com/devantler-tech/ksail/.github/workflows/cd.yaml@refs/tags/v1.2.3"}, + {"ghcr.io/devantler-tech/ksail-steer", "ksailcd", "https://github.com/devantler-tech/ksail/.github/workflows/cd.yaml@refs/tags/v1.2.3"}, {"ghcr.io/actions/actions-runner", "", ""}, } for _, fixture := range tests { diff --git a/scripts/tests/arc-staging/image_test.go b/scripts/tests/arc-staging/image_test.go index c8ac0d1670..975258e022 100644 --- a/scripts/tests/arc-staging/image_test.go +++ b/scripts/tests/arc-staging/image_test.go @@ -7,34 +7,10 @@ import ( "testing" ) -const analysisRepository = "ghcr.io/devantler-tech/platform-ksail-analysis-runner" -const analysisPublisher = "https://github.com/devantler-tech/platform/.github/workflows/publish-ksail-analysis-runner.yaml@refs/heads/main" +const analysisRepository = "ghcr.io/devantler-tech/ksail-analysis-runner" +const analysisPublisher = "https://github.com/devantler-tech/ksail/.github/workflows/publish-ksail-analysis-runner.yaml@refs/heads/main" const githubIssuer = "https://token.actions.githubusercontent.com" -func TestAnalysisImageCanBePublishedOnlyFromMain(t *testing.T) { - workflow := readYAML(t, ".github/workflows/publish-ksail-analysis-runner.yaml") - equal(t, len(field(t, workflow, "permissions").(map[string]any)), 0) - publish := field(t, workflow, "jobs", "publish") - equal(t, field(t, publish, "if"), "github.event_name == 'push' && github.ref == 'refs/heads/main'") - equal(t, field(t, publish, "needs"), "verify") - equal(t, field(t, publish, "permissions", "contents"), "read") - equal(t, field(t, publish, "permissions", "packages"), "write") - equal(t, field(t, publish, "permissions", "id-token"), "write") - equal(t, field(t, workflow, "env", "IMAGE"), analysisRepository) - equal(t, field(t, workflow, "jobs", "verify", "permissions", "contents"), "read") - equal(t, len(field(t, workflow, "jobs", "verify", "permissions").(map[string]any)), 1) - for _, job := range field(t, workflow, "jobs").(map[string]any) { - for _, step := range field(t, job, "steps").([]any) { - mapping := step.(map[string]any) - if uses, ok := mapping["uses"].(string); ok { - if !regexp.MustCompile(`@[0-9a-f]{40}$`).MatchString(uses) { - t.Fatalf("action must be immutable: %q", uses) - } - } - } - } -} - func TestAnalysisImageUsesItsOwnExactNodeSigner(t *testing.T) { policy := readYAML(t, "talos/cluster/verify-first-party-images.yaml") rules := field(t, policy, "rules").([]any) @@ -64,7 +40,7 @@ func TestAnalysisImageUsesItsOwnExactNodeSigner(t *testing.T) { for _, forbidden := range []string{ strings.Replace(analysisPublisher, "refs/heads/main", "refs/pull/1/merge", 1), strings.Replace(analysisPublisher, "refs/heads/main", "refs/heads/other", 1), - strings.Replace(analysisPublisher, "platform/", "other/", 1), + strings.Replace(analysisPublisher, "ksail/", "other/", 1), strings.Replace(analysisPublisher, "publish-ksail-analysis-runner", "publish-coroot-node-agent-hotfix", 1), analysisPublisher + ".other", "https://other.example/" + analysisPublisher, } { diff --git a/scripts/tests/refresh-flux-ghcr-auth/contracts_test.go b/scripts/tests/refresh-flux-ghcr-auth/contracts_test.go index 786e3462b3..885d380a27 100644 --- a/scripts/tests/refresh-flux-ghcr-auth/contracts_test.go +++ b/scripts/tests/refresh-flux-ghcr-auth/contracts_test.go @@ -656,9 +656,9 @@ func TestImageVerificationUsesOneSlowFailClosedPolicy(t *testing.T) { "image != 'ghcr.io/devantler-tech/platform-coroot-node-agent'", "image == 'ghcr.io/devantler-tech/platform-coroot-node-agent'", "verifyImageSignatures(image, [attestors.publishcorootnodeagent])", - "^https://github\\.com/devantler-tech/platform/\\.github/workflows/publish-ksail-analysis-runner\\.yaml@refs/heads/main$", - "image != 'ghcr.io/devantler-tech/platform-ksail-analysis-runner'", - "image == 'ghcr.io/devantler-tech/platform-ksail-analysis-runner'", + "^https://github\\.com/devantler-tech/ksail/\\.github/workflows/publish-ksail-analysis-runner\\.yaml@refs/heads/main$", + "image != 'ghcr.io/devantler-tech/ksail-analysis-runner'", + "image == 'ghcr.io/devantler-tech/ksail-analysis-runner'", "verifyImageSignatures(image, [attestors.publishksailanalysis])", "image.startsWith('ghcr.io/devantler-tech/ksail')", } { diff --git a/scripts/tests/refresh-flux-ghcr-auth/fake_kubectl_test.go b/scripts/tests/refresh-flux-ghcr-auth/fake_kubectl_test.go index dd86e36df9..f804de330a 100644 --- a/scripts/tests/refresh-flux-ghcr-auth/fake_kubectl_test.go +++ b/scripts/tests/refresh-flux-ghcr-auth/fake_kubectl_test.go @@ -1139,7 +1139,7 @@ func fakeKubectlPatchConsolidatedImageValidatingPolicy(args []string, patchFile case "publishcorootnodeagent": corootNodeAgentAttestorValid = issuerValid && subject == "^https://github\\.com/devantler-tech/platform/\\.github/workflows/publish-coroot-node-agent-hotfix\\.yaml@refs/heads/main$" case "publishksailanalysis": - analysisAttestorValid = issuerValid && subject == "^https://github\\.com/devantler-tech/platform/\\.github/workflows/publish-ksail-analysis-runner\\.yaml@refs/heads/main$" + analysisAttestorValid = issuerValid && subject == "^https://github\\.com/devantler-tech/ksail/\\.github/workflows/publish-ksail-analysis-runner\\.yaml@refs/heads/main$" case "publishwarzone": warZoneAttestorValid = issuerValid && subject == "^https://github\\.com/devantler-tech/world-at-ruin/\\.github/workflows/server-cd\\.yaml@refs/tags/v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$" } @@ -1171,9 +1171,9 @@ func fakeKubectlPatchConsolidatedImageValidatingPolicy(args []string, patchFile genericExcludesCorootNodeAgent = strings.Contains(expression, "image != 'ghcr.io/devantler-tech/platform-coroot-node-agent'") && strings.Contains(expression, "!image.startsWith('ghcr.io/devantler-tech/platform-coroot-node-agent:')") && strings.Contains(expression, "!image.startsWith('ghcr.io/devantler-tech/platform-coroot-node-agent@')") - genericExcludesAnalysis = strings.Contains(expression, "image != 'ghcr.io/devantler-tech/platform-ksail-analysis-runner'") && - strings.Contains(expression, "!image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner:')") && - strings.Contains(expression, "!image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner@')") + genericExcludesAnalysis = strings.Contains(expression, "image != 'ghcr.io/devantler-tech/ksail-analysis-runner'") && + strings.Contains(expression, "!image.startsWith('ghcr.io/devantler-tech/ksail-analysis-runner:')") && + strings.Contains(expression, "!image.startsWith('ghcr.io/devantler-tech/ksail-analysis-runner@')") } if strings.Contains(expression, "attestors.publishkubescapestorage") { dedicatedRoutesStorage = strings.Contains(expression, "image == 'ghcr.io/devantler-tech/platform-kubescape-storage'") && @@ -1196,9 +1196,9 @@ func fakeKubectlPatchConsolidatedImageValidatingPolicy(args []string, patchFile strings.Contains(expression, "image.startsWith('ghcr.io/devantler-tech/world-at-ruin/zone@')") } if strings.Contains(expression, "attestors.publishksailanalysis") { - dedicatedRoutesAnalysis = strings.Contains(expression, "image == 'ghcr.io/devantler-tech/platform-ksail-analysis-runner'") && - strings.Contains(expression, "image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner:')") && - strings.Contains(expression, "image.startsWith('ghcr.io/devantler-tech/platform-ksail-analysis-runner@')") + dedicatedRoutesAnalysis = strings.Contains(expression, "image == 'ghcr.io/devantler-tech/ksail-analysis-runner'") && + strings.Contains(expression, "image.startsWith('ghcr.io/devantler-tech/ksail-analysis-runner:')") && + strings.Contains(expression, "image.startsWith('ghcr.io/devantler-tech/ksail-analysis-runner@')") } } if !storageAttestorValid || !kubescapeNodeAgentAttestorValid || !corootNodeAgentAttestorValid || diff --git a/scripts/tests/test-inventory-first-party-image-signatures.sh b/scripts/tests/test-inventory-first-party-image-signatures.sh index d4e8d42741..ba9b8a856d 100755 --- a/scripts/tests/test-inventory-first-party-image-signatures.sh +++ b/scripts/tests/test-inventory-first-party-image-signatures.sh @@ -260,12 +260,12 @@ real="${repo_root}/talos/cluster/verify-first-party-images.yaml" # macOS ships bash 3.2, which has no `mapfile` — read the globs without it. real_globs="$(yq -r '.rules[].image' "$real")" expected_globs="$(printf '%s\n' \ + 'ghcr.io/devantler-tech/ksail-analysis-runner' \ 'ghcr.io/devantler-tech/ksail*' \ 'ghcr.io/devantler-tech/provider-upjet-*' \ 'ghcr.io/devantler-tech/platform-kubescape-storage' \ 'ghcr.io/devantler-tech/platform-kubescape-node-agent' \ 'ghcr.io/devantler-tech/platform-coroot-node-agent' \ - 'ghcr.io/devantler-tech/platform-ksail-analysis-runner' \ 'ghcr.io/devantler-tech/world-at-ruin/zone' \ 'ghcr.io/devantler-tech/*')" if [ "$real_globs" != "$expected_globs" ]; then @@ -412,8 +412,8 @@ fi # clause of the identity. Same parity rule as section 12: admission (Kyverno) and the kubelet # pull (Talos) must carry the same identity AND issuer, or a ksail image accepted by one is # refused by the other as an ImagePullBackOff neither file explains alone. -talos_ksail_identity="$(yq -r '.rules[] | select(.image == "ghcr.io/devantler-tech/ksail*") | .keyless.subjectRegex' "$real")" -talos_ksail_issuer="$(yq -r '.rules[] | select(.image == "ghcr.io/devantler-tech/ksail*") | .keyless.issuer' "$real")" +talos_ksail_identity="$(yq -r '.rules[] | select(.image | test("^ghcr\\.io/devantler-tech/ksail\\*$")) | .keyless.subjectRegex' "$real")" +talos_ksail_issuer="$(yq -r '.rules[] | select(.image | test("^ghcr\\.io/devantler-tech/ksail\\*$")) | .keyless.issuer' "$real")" kyverno_ksail_identity="$(yq -r '.spec.attestors[] | select(.name == "ksailcd") | .cosign.keyless.identities[].subjectRegExp' "$kyverno_policy")" kyverno_ksail_issuer="$(yq -r '.spec.attestors[] | select(.name == "ksailcd") | .cosign.keyless.identities[].issuer' "$kyverno_policy")" diff --git a/talos/cluster/verify-first-party-images.yaml b/talos/cluster/verify-first-party-images.yaml index d0fe713f62..82789abf99 100644 --- a/talos/cluster/verify-first-party-images.yaml +++ b/talos/cluster/verify-first-party-images.yaml @@ -57,7 +57,7 @@ # pods, a compromised registry serving a tampered image). # # Scope: FIRST-PARTY images (ghcr.io/devantler-tech/*). All are keyless-cosign -# signed, but by seven different signing identities, so they need seven rules: +# signed through eight signing routes: # 1. ghcr.io/devantler-tech/ksail — the ksail-operator image — is signed by # ksail's OWN release pipeline (devantler-tech/ksail .github/workflows/ # cd.yaml; see that repo's GoReleaser `docker_signs` step), at a release tag. @@ -71,13 +71,15 @@ # dedicated publish-kubescape-node-agent-hotfix.yaml workflow on main. # 5. ghcr.io/devantler-tech/platform-coroot-node-agent — signed by the # dedicated publish-coroot-node-agent-hotfix.yaml workflow on main. -# 6. the private World at Ruin zone is signed by that repository's server-cd +# 6. ghcr.io/devantler-tech/ksail-analysis-runner is signed by KSail's +# dedicated main-only publish-ksail-analysis-runner.yaml workflow. +# 7. the private World at Ruin zone is signed by that repository's server-cd # workflow on stable release tags only. -# 7. the app images (wedding-app, ascoachingogvaner, …) are signed by the +# 8. the app images (wedding-app, ascoachingogvaner, …) are signed by the # shared publish-app.yaml workflow in devantler-tech/actions, pinned by # commit SHA — the SAME form their Flux OCIRepository verify blocks and the # verify-app-images ImageValidatingPolicy pin (#3022). -# Talos evaluates rules in order, first match wins, so the six specific rules +# Talos evaluates rules in order, first match wins, so the seven specific rules # MUST precede the ghcr.io/devantler-tech/* catch-all. # # FAIL-OPEN by design for everything else: an image matching NO rule is pulled @@ -103,6 +105,11 @@ apiVersion: v1alpha1 kind: ImageVerificationConfig rules: + # The analysis image must precede the broader KSail release-image route. + - image: ghcr.io/devantler-tech/ksail-analysis-runner + keyless: + issuer: https://token.actions.githubusercontent.com + subjectRegex: ^https://github\.com/devantler-tech/ksail/\.github/workflows/publish-ksail-analysis-runner\.yaml@refs/heads/main$ # ksail-operator image — signed by ksail's OWN release pipeline (cd.yaml), # a different keyless identity than publish-app.yaml. MUST precede the # catch-all below (first-match-wins). Requires ksail >= the first release @@ -168,11 +175,6 @@ rules: keyless: issuer: https://token.actions.githubusercontent.com subjectRegex: ^https://github\.com/devantler-tech/platform/\.github/workflows/publish-coroot-node-agent-hotfix\.yaml@refs/heads/main$ - # The repository-scoped analysis runner has its own main-only publisher. - - image: ghcr.io/devantler-tech/platform-ksail-analysis-runner - keyless: - issuer: https://token.actions.githubusercontent.com - subjectRegex: ^https://github\.com/devantler-tech/platform/\.github/workflows/publish-ksail-analysis-runner\.yaml@refs/heads/main$ # Exact zone repository, before the generic first-party app rule. The private # trial uses the game's own stable release workflow, not the shared publisher. - image: ghcr.io/devantler-tech/world-at-ruin/zone From 3b618f9ebe3ad21288b9e450836171379fda51a6 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 5 Oct 2026 23:01:29 +0200 Subject: [PATCH 05/62] fix(ci): prepare isolated analysis capacity and writable runner storage --- docs/operations/ksail-arc.md | 19 +++-- .../ksail-analysis-runners/helm-release.yaml | 36 ++++++++ ...-job-stranded-autoscaler-node-alerter.yaml | 2 +- ksail.prod.yaml | 19 ++++- scripts/tests/arc-staging/runtime_test.go | 85 +++++++++++++++++++ scripts/tests/arc-staging/staging_test.go | 7 +- 6 files changed, 153 insertions(+), 15 deletions(-) create mode 100644 scripts/tests/arc-staging/runtime_test.go diff --git a/docs/operations/ksail-arc.md b/docs/operations/ksail-arc.md index b6b55f0daf..e458c54a15 100644 --- a/docs/operations/ksail-arc.md +++ b/docs/operations/ksail-arc.md @@ -44,11 +44,15 @@ Activation requires a separate reviewed change and all of these proofs: remains KSail-only; the existing App installation also serves other platform consumers. No new App, credential copy, key rotation or permission expansion is required. Do not print, check in or mount the private key in a job runner. -2. Establish explicitly approved, isolated analysis capacity. The runner selector - and toleration name `platform.devantler.tech/ksail-analysis=enabled`; no current - capacity is assumed to carry that label. New billable capacity requires its - own approval and an exact deletion/readback plan. Do not label a busy production - worker to make a pending analysis fit. +2. Use the declared `autoscale-ksail-analysis` CX53 pool for isolated analysis + capacity. It has a minimum of zero and maximum of one node, sharing the + unchanged cluster ceiling of nine nodes and account ceiling of ten. The + selector and NoSchedule taint both name + `platform.devantler.tech/ksail-analysis=enabled`; ordinary workloads and the + warm-capacity buffer do not tolerate that taint. Verify the label and taint + on the actual autoscaled node. Do not label a busy production worker to make + a pending analysis fit. After calibration, verify the pool scales back to + zero; capacity outside these bounds requires its own approval. 3. Verify scheduler reservations, node allocatable capacity, ephemeral storage, ResourceQuota and LimitRange. Provisional requests are 12Gi/3 CPU for one runner, with limits of 14Gi/3.5 CPU in the analysis namespace. The controller and @@ -61,6 +65,11 @@ Activation requires a separate reviewed change and all of these proofs: 4. Render both pinned charts, including their CRDs and namespace-scoped RBAC. Confirm the App Secret is absent from runner volumes, the no-permission runner service account is used, and the chart preserves the listener name and labels. + Runner storage consists only of a 40Gi disk emptyDir for its home and a 2Gi + disk emptyDir for temporary files. A restricted init container copies the + baked runner into the group-writable home volume as UID/GID 1001. Both the + bootstrap and job container keep their root filesystems read-only and drop + every capability. Verify those settings on the generated runner pod. Test admission, API isolation and denied internal egress, not only YAML validity. 5. Prove the actual managed Go and JavaScript Code Quality jobs on an ephemeral runner. The official runner image is not a hosted-runner software clone. Check diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml b/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml index 7d744de6d4..710ec99d71 100644 --- a/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml +++ b/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml @@ -44,8 +44,38 @@ spec: runAsNonRoot: true runAsUser: 1001 runAsGroup: 1001 + fsGroup: 1001 seccompProfile: type: RuntimeDefault + volumes: + - name: runner-home + emptyDir: + sizeLimit: 40Gi + - name: runner-tmp + emptyDir: + sizeLimit: 2Gi + initContainers: + - name: init-runner-home + image: ghcr.io/actions/actions-runner@sha256:e5496277be5d09bc968b3d64911b74e219ac4a3f2edce956a3ecf9271bea1ef4 + command: ["/bin/sh", "-ec", "cp -R /home/runner/. /runner-data/"] + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + resources: + requests: + cpu: 100m + memory: 256Mi + ephemeral-storage: 1Gi + limits: + cpu: "1" + memory: 512Mi + ephemeral-storage: 4Gi + volumeMounts: + - name: runner-home + mountPath: /runner-data containers: - name: runner image: ghcr.io/actions/actions-runner@sha256:e5496277be5d09bc968b3d64911b74e219ac4a3f2edce956a3ecf9271bea1ef4 @@ -53,8 +83,14 @@ spec: securityContext: allowPrivilegeEscalation: false privileged: false + readOnlyRootFilesystem: true capabilities: drop: [ALL] + volumeMounts: + - name: runner-home + mountPath: /home/runner + - name: runner-tmp + mountPath: /tmp resources: requests: cpu: "3" diff --git a/k8s/providers/hetzner/infrastructure/coroot/cron-job-stranded-autoscaler-node-alerter.yaml b/k8s/providers/hetzner/infrastructure/coroot/cron-job-stranded-autoscaler-node-alerter.yaml index 94f71a6e01..5a65e08246 100644 --- a/k8s/providers/hetzner/infrastructure/coroot/cron-job-stranded-autoscaler-node-alerter.yaml +++ b/k8s/providers/hetzner/infrastructure/coroot/cron-job-stranded-autoscaler-node-alerter.yaml @@ -98,7 +98,7 @@ spec: # Space-separated, and kept equal to the `pools[].name` list in # ksail.prod.yaml by scripts/tests/test-stranded-autoscaler-node-alerter.sh. - name: DECLARED_POOLS - value: "autoscale-cx43 autoscale-cx53" + value: "autoscale-cx43 autoscale-cx53 autoscale-ksail-analysis" securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/ksail.prod.yaml b/ksail.prod.yaml index 8b7f1070f1..058c9f8f80 100644 --- a/ksail.prod.yaml +++ b/ksail.prod.yaml @@ -62,9 +62,9 @@ spec: # static baseline. 9 = the Hetzner account cap (10) MINUS ONE RESERVED SLOT: # 6 static (3 control-planes + 3 workers) + up to 3 autoscaler nodes. KSail # passes it to cluster-autoscaler --max-nodes-total, which counts every node, - # so this total framing already matches the runtime. Each pool max is 4, so - # any single CX type can still fill the autoscaler headroom (or any mix); the - # cluster total stays <= 9. + # so this total framing already matches the runtime. The general pools each + # allow 4 nodes; the isolated analysis pool allows 1. Any mix shares the + # same cluster ceiling of 9. # # THE RESERVED SLOT IS LOAD-BEARING — DO NOT RAISE THIS TO THE ACCOUNT CAP. # `deploy-prod` ensures the autoscaler config secret on EVERY deploy, and that @@ -125,6 +125,19 @@ spec: location: fsn1 min: 0 max: 4 + # Zero idle analysis capacity. The taint prevents ordinary workloads + # and warm-capacity buffers from keeping this larger worker alive. + - name: autoscale-ksail-analysis + serverType: cx53 + location: fsn1 + min: 0 + max: 1 + labels: + platform.devantler.tech/ksail-analysis: enabled + taints: + - key: platform.devantler.tech/ksail-analysis + value: enabled + effect: NoSchedule # These components are installed by ksail. There is a known race on # Hetzner clusters between ksail's metrics-server install and Flux's # kubelet-serving-cert-approver install (required because Talos diff --git a/scripts/tests/arc-staging/runtime_test.go b/scripts/tests/arc-staging/runtime_test.go new file mode 100644 index 0000000000..0a3ab597d2 --- /dev/null +++ b/scripts/tests/arc-staging/runtime_test.go @@ -0,0 +1,85 @@ +package arcstaging_test + +import ( + "reflect" + "testing" +) + +func TestAnalysisCapacityIsIsolatedAndKeepsTheClusterCeiling(t *testing.T) { + config := readYAML(t, "ksail.prod.yaml") + node := field(t, config, "spec", "cluster", "autoscaler", "node") + equal(t, field(t, node, "maxNodesTotal"), 9) + equal(t, field(t, config, "spec", "provider", "hetzner", "serverLimit"), 10) + pools := field(t, node, "pools").([]any) + count := 0 + for _, pool := range pools { + if field(t, pool, "name") != "autoscale-ksail-analysis" { + continue + } + count++ + equal(t, field(t, pool, "serverType"), "cx53") + equal(t, field(t, pool, "min"), 0) + equal(t, field(t, pool, "max"), 1) + equal(t, field(t, pool, "labels", "platform.devantler.tech/ksail-analysis"), "enabled") + wantTaints := []any{map[string]any{ + "key": "platform.devantler.tech/ksail-analysis", "value": "enabled", "effect": "NoSchedule", + }} + if !reflect.DeepEqual(field(t, pool, "taints"), wantTaints) { + t.Fatal("analysis nodes must reject ordinary workload scheduling") + } + } + equal(t, count, 1) + buffer := readYAML(t, "k8s/providers/hetzner/infrastructure/overprovisioning/pod-template.yaml") + if _, exists := field(t, buffer, "template", "spec").(map[string]any)["tolerations"]; exists { + t.Fatal("ordinary warm-capacity buffers must not tolerate the analysis taint") + } +} + +func TestRunnerHasOnlyBoundedDisposableStorageAndRestrictedBootstrap(t *testing.T) { + release := readYAML(t, "k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml") + spec := field(t, release, "spec", "values", "template", "spec") + assertRunnerStorage(t, spec) +} + +func assertRunnerStorage(t *testing.T, spec any) { + t.Helper() + equal(t, field(t, spec, "securityContext", "fsGroup"), 1001) + wantVolumes := []any{ + map[string]any{"name": "runner-home", "emptyDir": map[string]any{"sizeLimit": "40Gi"}}, + map[string]any{"name": "runner-tmp", "emptyDir": map[string]any{"sizeLimit": "2Gi"}}, + } + if !reflect.DeepEqual(field(t, spec, "volumes"), wantVolumes) { + t.Fatal("only two explicitly bounded disk emptyDirs are permitted") + } + runner := field(t, spec, "containers").([]any)[0] + equal(t, field(t, runner, "securityContext", "readOnlyRootFilesystem"), true) + wantMounts := []any{ + map[string]any{"name": "runner-home", "mountPath": "/home/runner"}, + map[string]any{"name": "runner-tmp", "mountPath": "/tmp"}, + } + if !reflect.DeepEqual(field(t, runner, "volumeMounts"), wantMounts) { + t.Fatal("runner may mount only its disposable home and temporary storage") + } + initializers := field(t, spec, "initContainers").([]any) + equal(t, len(initializers), 1) + initial := initializers[0] + equal(t, field(t, initial, "name"), "init-runner-home") + equal(t, field(t, initial, "image"), field(t, runner, "image")) + wantCommand := []any{"/bin/sh", "-ec", "cp -R /home/runner/. /runner-data/"} + if !reflect.DeepEqual(field(t, initial, "command"), wantCommand) { + t.Fatal("bootstrap must copy the baked runner without root-only preservation") + } + for _, name := range []string{"privileged", "allowPrivilegeEscalation"} { + equal(t, field(t, initial, "securityContext", name), false) + } + equal(t, field(t, initial, "securityContext", "readOnlyRootFilesystem"), true) + if !reflect.DeepEqual(field(t, initial, "securityContext", "capabilities", "drop"), []any{"ALL"}) { + t.Fatal("bootstrap must drop every capability") + } + wantInitMounts := []any{map[string]any{"name": "runner-home", "mountPath": "/runner-data"}} + if !reflect.DeepEqual(field(t, initial, "volumeMounts"), wantInitMounts) { + t.Fatal("bootstrap may mount only the empty destination volume") + } + equal(t, field(t, initial, "resources", "requests", "memory"), "256Mi") + equal(t, field(t, initial, "resources", "limits", "memory"), "512Mi") +} diff --git a/scripts/tests/arc-staging/staging_test.go b/scripts/tests/arc-staging/staging_test.go index 2cd51254cf..19e7161b80 100644 --- a/scripts/tests/arc-staging/staging_test.go +++ b/scripts/tests/arc-staging/staging_test.go @@ -94,12 +94,7 @@ func TestRepositoryScopedPoolCannotCreateUnboundedOrPrivilegedRunners(t *testing } equal(t, field(t, runner, "resources", "requests", "memory"), "12Gi") equal(t, field(t, runner, "resources", "limits", "memory"), "14Gi") - if _, exists := spec.(map[string]any)["volumes"]; exists { - t.Fatal("no host, credential or persistent volumes may reach a job runner") - } - if _, exists := spec.(map[string]any)["initContainers"]; exists { - t.Fatal("no privileged bootstrap container is permitted") - } + assertRunnerStorage(t, spec) listener := field(t, values, "listenerTemplate", "spec", "containers").([]any) if len(listener) != 1 { t.Fatal("listener must be separately bounded") From 8f33913a56e0389cbfaa6b5d3c85363b89d38ee1 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 5 Oct 2026 23:20:28 +0200 Subject: [PATCH 06/62] fix(ci): name the KSail analysis signer in admission errors --- .../cluster-policies/best-practices/verify-app-images.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/bases/infrastructure/cluster-policies/best-practices/verify-app-images.yaml b/k8s/bases/infrastructure/cluster-policies/best-practices/verify-app-images.yaml index 1602d9660d..948dfe7eb9 100644 --- a/k8s/bases/infrastructure/cluster-policies/best-practices/verify-app-images.yaml +++ b/k8s/bases/infrastructure/cluster-policies/best-practices/verify-app-images.yaml @@ -248,7 +248,7 @@ spec: .map(image, verifyImageSignatures(image, [attestors.publishksailanalysis])) .all(e, e > 0) message: >- - KSail analysis runner image is not signed by the platform repository's + KSail analysis runner image is not signed by the KSail repository's dedicated main-branch publisher (keyless cosign) - expression: >- (images.containers + images.initContainers + images.ephemeralContainers) From 69efb4f913684738592b6426208ff80d213bfffb Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 00:16:37 +0200 Subject: [PATCH 07/62] test(security): include the analysis signer in live verifier fixtures --- .../test-validate-image-verifier-liveness.sh | 58 ++++++++++++++----- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/scripts/tests/test-validate-image-verifier-liveness.sh b/scripts/tests/test-validate-image-verifier-liveness.sh index e3620c7b35..d3ae261e69 100755 --- a/scripts/tests/test-validate-image-verifier-liveness.sh +++ b/scripts/tests/test-validate-image-verifier-liveness.sh @@ -157,6 +157,7 @@ readonly roots_type='tuftrustedroots.security.talos.dev' readonly rules_owner='security.ImageVerificationConfigController' readonly roots_owner='security.TUFTrustedRootController' readonly ksail_pattern='ghcr.io/devantler-tech/ksail*' +readonly analysis_pattern='ghcr.io/devantler-tech/ksail-analysis-runner' readonly provider_pattern='ghcr.io/devantler-tech/provider-upjet-*' readonly storage_pattern='ghcr.io/devantler-tech/platform-kubescape-storage' readonly kubescape_node_agent_pattern='ghcr.io/devantler-tech/platform-kubescape-node-agent' @@ -212,13 +213,14 @@ write_roots() { write_healthy_rules() { local node="$1" { - resource_obj 0000 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${ksail_pattern}" - resource_obj 0001 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${provider_pattern}" - resource_obj 0002 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${storage_pattern}" - resource_obj 0003 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${kubescape_node_agent_pattern}" - resource_obj 0004 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${coroot_node_agent_pattern}" - resource_obj 0005 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${zone_pattern}" - resource_obj 0006 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${app_pattern}" + resource_obj 0000 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${analysis_pattern}" + resource_obj 0001 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${ksail_pattern}" + resource_obj 0002 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${provider_pattern}" + resource_obj 0003 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${storage_pattern}" + resource_obj 0004 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${kubescape_node_agent_pattern}" + resource_obj 0005 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${coroot_node_agent_pattern}" + resource_obj 0006 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${zone_pattern}" + resource_obj 0007 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${app_pattern}" } | write_rules "${node}" } @@ -244,7 +246,7 @@ run_script() { healthy_node good output="$(run_script TALOS_NODES=good 2>&1)" || fail "case 1: expected exit 0 for a node that can enforce" require_text "${output}" 'OK good' 'case 1: reports the healthy node' -require_text "${output}" '7 rule(s) in phase running' 'case 1: counts every declared running rule' +require_text "${output}" '8 rule(s) in phase running' 'case 1: counts every declared running rule' require_text "${output}" 'with exact declared decisions' 'case 1: confirms complete rule decisions were compared' require_text "${output}" 'All 1 node(s) can enforce image verification.' 'case 1: reports the summary' @@ -281,13 +283,14 @@ require_text "${output}" 'declared rule set' 'case 1a: names the incomplete poli # =========================================================================== write_node driftrules { - resource_obj 0000 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${ksail_pattern}" - resource_obj 0001 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${provider_pattern}" - resource_obj 0002 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${storage_pattern}" - resource_obj 0003 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${kubescape_node_agent_pattern}" - resource_obj 0004 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${coroot_node_agent_pattern}" - resource_obj 0005 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${zone_pattern}" - resource_obj 0006 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' 'ghcr.io/devantler-tech/stale-*' + resource_obj 0000 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${analysis_pattern}" + resource_obj 0001 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${ksail_pattern}" + resource_obj 0002 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${provider_pattern}" + resource_obj 0003 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${storage_pattern}" + resource_obj 0004 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${kubescape_node_agent_pattern}" + resource_obj 0005 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${coroot_node_agent_pattern}" + resource_obj 0006 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' "${zone_pattern}" + resource_obj 0007 running "${rules_owner}" 'ImageVerificationRules.security.talos.dev' 'ghcr.io/devantler-tech/stale-*' } | write_rules driftrules resource_obj trusted_root.json running "${roots_owner}" 'TUFTrustedRoots.security.talos.dev' | write_roots driftrules @@ -297,6 +300,31 @@ output="$(run_script TALOS_NODES=driftrules 2>&1)" || status=$? require_text "${output}" 'FAIL driftrules' 'case 1b: names the node with policy drift' require_text "${output}" 'declared rule set' 'case 1b: names the drifted policy' +# The prepared analysis route is part of every node's policy, even before ARC +# is activated. Missing it or substituting the legacy KSail release signer +# must remain red while all other routes and trust material stay healthy. +healthy_node oldanalysispolicy +analysis_rules="${fixtures}/oldanalysispolicy/resources/${rules_type}" +jq -s --arg pattern "${analysis_pattern}" 'map(select(.spec.imagePattern != $pattern))[]' \ + "${analysis_rules}" >"${work_dir}/changed-rules.json" +mv "${work_dir}/changed-rules.json" "${analysis_rules}" +status=0 +output="$(run_script TALOS_NODES=oldanalysispolicy 2>&1)" || status=$? +[[ "${status}" -eq 1 ]] || fail 'missing exact analysis route must fail' +require_text "${output}" 'declared rule set' 'missing analysis route must name policy drift' + +healthy_node analysissignerdrift +release_verifier=$(jq -c --arg pattern "${ksail_pattern}" \ + '.[] | select(.imagePattern == $pattern) | .keylessVerifier' <<<"${fixture_rules_json}") +analysis_rules="${fixtures}/analysissignerdrift/resources/${rules_type}" +jq -s --arg pattern "${analysis_pattern}" --argjson verifier "${release_verifier}" \ + 'map(if .spec.imagePattern == $pattern then .spec.keylessVerifier = $verifier else . end)[]' \ + "${analysis_rules}" >"${work_dir}/changed-rules.json" +mv "${work_dir}/changed-rules.json" "${analysis_rules}" +status=0 +output="$(run_script TALOS_NODES=analysissignerdrift 2>&1)" || status=$? +[[ "${status}" -eq 1 ]] || fail 'legacy KSail release signer must not satisfy the analysis route' + # A previously healthy six-rule fleet cannot clear activation of the new zone. # The existing app catch-all is present, but uses the wrong signer for this image. healthy_node oldzonepolicy From fe867842d96c3685a01582c26d8a90b8a4b7058a Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 00:54:05 +0200 Subject: [PATCH 08/62] feat(ci): enforce bounded KSail ARC runtime acceptance --- .github/actions/deploy-prod/action.yml | 15 + .github/workflows/ci.yaml | 9 + docs/operations/ksail-arc.md | 58 +++- .../tests/test-verify-ksail-arc-runtime.sh | 221 +++++++++++++ scripts/verify-ksail-arc-runtime.sh | 306 ++++++++++++++++++ scripts/verify-ksail-arc-runtime/budget.go | 157 +++++++++ .../verify-ksail-arc-runtime/budget_test.go | 51 +++ scripts/verify-ksail-arc-runtime/denial.go | 94 ++++++ .../verify-ksail-arc-runtime/denial_test.go | 49 +++ scripts/verify-ksail-arc-runtime/image.go | 63 ++++ .../verify-ksail-arc-runtime/image_test.go | 43 +++ scripts/verify-ksail-arc-runtime/main.go | 120 +++++++ scripts/verify-ksail-arc-runtime/pod.go | 151 +++++++++ scripts/verify-ksail-arc-runtime/pod_test.go | 136 ++++++++ 14 files changed, 1456 insertions(+), 17 deletions(-) create mode 100644 scripts/tests/test-verify-ksail-arc-runtime.sh create mode 100644 scripts/verify-ksail-arc-runtime.sh create mode 100644 scripts/verify-ksail-arc-runtime/budget.go create mode 100644 scripts/verify-ksail-arc-runtime/budget_test.go create mode 100644 scripts/verify-ksail-arc-runtime/denial.go create mode 100644 scripts/verify-ksail-arc-runtime/denial_test.go create mode 100644 scripts/verify-ksail-arc-runtime/image.go create mode 100644 scripts/verify-ksail-arc-runtime/image_test.go create mode 100644 scripts/verify-ksail-arc-runtime/main.go create mode 100644 scripts/verify-ksail-arc-runtime/pod.go create mode 100644 scripts/verify-ksail-arc-runtime/pod_test.go diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index 8a36a36142..7481903293 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -554,6 +554,21 @@ runs: shell: bash run: ./scripts/verify-gateway-routes-applied.sh + - name: Verify the deployed KSail ARC canary + if: >- + !cancelled() && + steps.wait_flux_revision.outcome == 'success' && + ( + steps.cluster_update.outcome == 'skipped' || + steps.wait_prod_api_stability.outcome == 'success' + ) + shell: bash + env: + PLATFORM_MANIFEST_DIGEST: ${{ steps.publish_platform_manifest.outputs.digest }} + run: | + export KUBECONFIG="${HOME}/.kube/config" + bash scripts/verify-ksail-arc-runtime.sh --if-active + - name: Verify the deployed data product UI release # The protected deployment must prove current Apps/chart/Helm identities, # all current Pods and public endpoints before its candidate can merge. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0bf5fd3bfc..a2510b5a08 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -149,6 +149,12 @@ jobs: - name: 🛑 Validate inactive ARC staging boundaries run: go test ./scripts/tests/arc-staging + - name: 🔬 Validate bounded ARC runtime acceptance + run: | + go test -race ./scripts/verify-ksail-arc-runtime + shellcheck scripts/verify-ksail-arc-runtime.sh scripts/tests/test-verify-ksail-arc-runtime.sh + bash scripts/tests/test-verify-ksail-arc-runtime.sh + # Shared Wedding backup access is retired on this proof's word, so it must # pass only on an AccessDenied for every access, after a positive control # and against an object the shared credential showed exists. These cases @@ -482,6 +488,9 @@ jobs: - '.github/scripts/setup-supply-chain-tools.sh' - 'scripts/tests/test-setup-supply-chain-tools.sh' - '.github/actions/deploy-prod/action.yml' + - 'scripts/verify-ksail-arc-runtime.sh' + - 'scripts/verify-ksail-arc-runtime/**' + - 'scripts/tests/test-verify-ksail-arc-runtime.sh' - '.github/workflows/dr-rebuild.yaml' - '.github/workflows/validate-image-verifier-liveness.yaml' - 'scripts/tests/test-openbao-oidc-role.sh' diff --git a/docs/operations/ksail-arc.md b/docs/operations/ksail-arc.md index 55ab9e78f5..ccb2ff4dad 100644 --- a/docs/operations/ksail-arc.md +++ b/docs/operations/ksail-arc.md @@ -23,7 +23,7 @@ publishes, attests and signs only on a push to KSail main. Both Kyverno and Talo that workflow identity only for the exact analysis image repository. The staged pool still requires a verified published digest before activation. -The smoke test starts the copied runner and compiles a Go program against GTK +The image smoke test executes the copied runner's version command and compiles a Go program against GTK and WebKit as UID/GID 1001 with a read-only root filesystem and no capabilities. The runner home and temporary files use disposable writable storage. Bootstrap copies use `cp -R`; preserving the root directory's ownership and timestamps @@ -31,7 +31,9 @@ would fail on a group-writable Kubernetes volume. ## Activation gates -Activation requires a separate reviewed change and all of these proofs: +Activation uses a separate reviewed change. Complete the preparation checks +first, then prove bounded runtime acceptance and actual job execution before +routing managed analysis: 1. Reuse the platform's existing GitHub management App from `infrastructure/github/app`. The `arc-ksail-app` ExternalSecret maps its @@ -71,14 +73,34 @@ Activation requires a separate reviewed change and all of these proofs: baked runner into the group-writable home volume as UID/GID 1001. Both the bootstrap and job container keep their root filesystems read-only and drop every capability. Verify those settings on the generated runner pod. - Test admission, API isolation and denied internal egress, not only YAML validity. -5. Prove the actual managed Go and JavaScript Code Quality jobs on an ephemeral - runner. The official runner image is not a hosted-runner software clone. Check - the required tools, GitHub proxy, network allowlist and non-root installation - path; do not enable sudo/privilege escalation to make setup pass. If a dedicated - reproducible tool image is required, deliver and verify it separately. -6. Read the repository-assigned runner registration, then change managed Code - Quality to the verified `ksail-code-quality` label and read configuration back. + Publish and verify the KSail-owned analysis image on main before replacing + the staging image pin. +5. Reference and unsuspend the controller in production's controller layer and + the pool in production's infrastructure layer. The existing layer dependency + creates both namespaces and reconciles controller readiness before the pool. + The protected deployment runs `scripts/verify-ksail-arc-runtime.sh --if-active` + against its exact published revision. It requires the registered scale-set + identity and the unchanged node/runner bounds, verifies the image's main + signing identity and selected amd64 manifest, and starts one bounded probe + from the actual runner template. The proof checks the complete admitted Pod + spec and both runtime image identities, quota and allocatable reservations, + toolchain compilation, absent API token and allowed external HTTPS. Restricted + admission must reject privileged and host-volume controls. Healthy API and + internal endpoints must remain unreachable from the probe, with each attempt + correlated to an actual Cilium policy denial at its source, target, port, + node and time. A timeout, observer loss or transport failure cannot pass. + Cleanup atomically binds deletion to the created Pod UID and verifies the + dedicated pool returns to zero. An unknown create outcome or replaced Pod + fails cleanup; it never authorizes deletion of an unbound object. +6. Dispatch KSail's main-only `verify-ksail-arc-delivery.yaml` preflight with + `enable_arc=true`. Join that actual registered job to its restricted runner + Pod, signed image, allowed network path and cleanup before changing managed + Code Quality. The image's version command and the platform probe do not prove + an authenticated runner job. Read the repository-assigned registration, + then change managed Code Quality to the verified `ksail-code-quality` label + and read configuration back. Preserve the other setup fields. Check the + required tools, GitHub proxy, network allowlist and non-root installation + path; do not enable sudo or privilege escalation to make setup pass. Preserve all root, nested-module and desktop extraction/source guards. Meet KSail #7131's five consecutive successful managed runs and verify completed runner pods and their registration are removed before declaring that bug fixed. @@ -92,13 +114,15 @@ not activation proof. ## Rollout and recovery -After the gates above are approved, first reference and unsuspend the controller -in the controller layer and verify it is healthy. That component creates both -namespaces before the chart installs its namespace-scoped RBAC. The controller's -network policy also covers the listener, which ARC creates in that namespace. -Only then reference the pool in -the infrastructure layer, where its external secret store already exists. Keep -the unique analysis label out of ordinary build/test/provider workflows. +Production activation follows steps 1–5; actual job proof and managed calibration +follow in step 6. Managed analysis stays on its previous runner configuration +until the registered-job proof completes. The controller's network policy covers +the listener, which ARC creates in that namespace. Keep the unique analysis label +out of ordinary build/test/provider workflows. The runtime verifier has no live +access while both production references are absent, refuses partial activation, +and runs only inside the protected Platform deployment. Its offline regression +suite exercises the complete lifecycle and intercepted negative controls with +fixtures; those fixtures never count as live acceptance. To stop admitting jobs, restore managed Code Quality's prior runner configuration and verify the readback, then use a reviewed values change to set both runner diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh new file mode 100644 index 0000000000..f2ed78db1d --- /dev/null +++ b/scripts/tests/test-verify-ksail-arc-runtime.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/bin" "$scratch/scripts" "$scratch/k8s/providers/hetzner/infrastructure/controllers" \ + "$scratch/k8s/bases/infrastructure/ksail-analysis-runners" +cp "$root/scripts/verify-ksail-arc-runtime.sh" "$scratch/scripts/" +cp "$root/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml" \ + "$scratch/k8s/bases/infrastructure/ksail-analysis-runners/" +go build -o "$scratch/verifier" "$root/scripts/verify-ksail-arc-runtime" +export ARC_TEST_ROOT="$scratch" +printf '{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{},"layers":[]}' >"$scratch/runtime-image" +runtime_digest="sha256:$(shasum -a 256 "$scratch/runtime-image" | cut -d' ' -f1)" +export runtime_digest +jq -cn --arg child "$runtime_digest" '{schemaVersion:2,mediaType:"application/vnd.oci.image.index.v1+json", + manifests:[{digest:$child,platform:{os:"linux",architecture:"amd64"}}]}' >"$scratch/image" +digest="sha256:$(shasum -a 256 "$scratch/image" | cut -d' ' -f1)" +export digest +image="ghcr.io/devantler-tech/ksail-analysis-runner@$digest" +image="$image" yq -i '.spec.values.template.spec.containers[0].image=strenv(image) | + .spec.values.template.spec.initContainers[0].image=strenv(image)' \ + "$scratch/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml" +yq -o=json '.spec.values' "$scratch/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml" \ + | jq '{metadata:{generation:1,annotations:{"runner-scale-set-id":"1"}}, + status:{phase:"Running",observedGeneration:1},spec:.} | + .spec.template.spec.serviceAccountName="ksail-code-quality-gha-rs-no-permission"' >"$scratch/ars" +cat >"$scratch/scripts/wait-for-platform-flux-revision.sh" <<'SH' +#!/usr/bin/env bash +[[ "$1" == "$digest" ]] +SH +cat >"$scratch/bin/timeout" <<'SH' +#!/usr/bin/env bash +export ARC_TEST_TIMEOUT=$1 +shift +exec "$@" +SH +cat >"$scratch/bin/date" <<'SH' +#!/usr/bin/env bash +printf '2026-10-05T22:00:00.000000000Z\n' +SH +cat >"$scratch/bin/go" <<'SH' +#!/usr/bin/env bash +[[ "$1" == run && "$2" == ./scripts/verify-ksail-arc-runtime ]] || exit 91 +shift 2 +exec "$ARC_TEST_ROOT/verifier" "$@" +SH +cat >"$scratch/bin/cosign" <<'SH' +#!/usr/bin/env bash +[[ "$*" == *"--certificate-identity https://github.com/devantler-tech/ksail/.github/workflows/publish-ksail-analysis-runner.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com ghcr.io/devantler-tech/ksail-analysis-runner@$digest" ]] || exit 92 +[[ "$ARC_TEST_CASE" != bad-signature ]] || exit 1 +SH +cat >"$scratch/bin/docker" <<'SH' +#!/usr/bin/env bash +if [[ "$*" == "buildx imagetools inspect ghcr.io/devantler-tech/ksail-analysis-runner@$digest --raw" ]]; then + cat "$ARC_TEST_ROOT/image" +elif [[ "$*" == "buildx imagetools inspect ghcr.io/devantler-tech/ksail-analysis-runner@$runtime_digest --raw" ]]; then + cat "$ARC_TEST_ROOT/runtime-image" +else exit 93; fi +[[ "$ARC_TEST_CASE" != tampered-image ]] || printf ' ' +SH +cat >"$scratch/bin/kubectl" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +[[ "$1" == --context && "$2" == admin@prod ]] || exit 94 +shift 3 # context pair and request-timeout +args="$*" +case "$args" in + *'get deployment cluster-autoscaler-hetzner-cluster-autoscaler '*) + printf '%s\n' --cloud-provider=hetzner --max-nodes-total=9 --nodes=0:1:cx53:fsn1:autoscale-ksail-analysis + [[ "$ARC_TEST_CASE" != wider-ceiling ]] || printf '%s\n' --max-nodes-total=10 ;; + *'get kustomization '*) + jq -n --arg revision "latest@$digest" '{metadata:{generation:1},status:{observedGeneration:1, + lastAppliedRevision:$revision,conditions:[{type:"Ready",status:"True"}]}}' ;; + *'get autoscalingrunnerset.actions.github.com '*) + if [[ "$ARC_TEST_CASE" == no-registration ]]; then jq '.metadata.annotations["runner-scale-set-id"]="0"' "$ARC_TEST_ROOT/ars" + else cat "$ARC_TEST_ROOT/ars"; fi ;; + *'get resourcequotas '*) + if [[ "$ARC_TEST_CASE" == full-quota ]]; then + printf '{"items":[{"spec":{"hard":{"pods":"1"}},"status":{"hard":{"pods":"1"},"used":{"pods":"1"}}}]}' + elif [[ "$ARC_TEST_CASE" == stale-quota ]]; then + printf '{"items":[{"spec":{"hard":{"pods":"1"}},"status":{"hard":{"pods":"3"},"used":{"pods":"0"}}}]}' + else printf '{"items":[]}'; fi ;; + *'get pods -l platform.devantler.tech/arc-role=runner '*|*'get pods -l platform.devantler.tech/arc-runtime-probe'*) + printf '{"items":[]}' ;; + 'get nodes -o json') + if [[ "$ARC_TEST_CASE" == failed-node-cleanup && -e "$ARC_TEST_ROOT/deleted" ]]; then exit 1; fi + if [[ "$ARC_TEST_CASE" == unknown-create && -e "$ARC_TEST_ROOT/live-pod" ]]; then exit 1; fi + if [[ "$ARC_TEST_CASE" == replacement-race && -e "$ARC_TEST_ROOT/replacement-preserved" ]]; then exit 1; fi + if [[ -e "$ARC_TEST_ROOT/live-pod" ]]; then + printf '{"items":[{"metadata":{"name":"autoscale-ksail-analysis-0123456789abcdef","uid":"autoscale-ksail-analysis-0123456789abcdef-uid","labels":{"platform.devantler.tech/ksail-analysis":"enabled"}}}]}' + else + printf '{"items":[{"metadata":{"name":"baseline-node","uid":"baseline-uid","labels":{}}}]}' + fi ;; + 'create --dry-run=server -f '*) + file=$4 + if jq -e '.spec.containers[0].securityContext.privileged == true or any(.spec.volumes[]; .hostPath != null)' "$file" >/dev/null; then + if [[ "$ARC_TEST_CASE" == admission-transport ]]; then printf 'connection refused\n' >&2 + else printf 'violates PodSecurity restricted: privileged hostPath\n' >&2; fi + exit 1 + fi ;; + 'create -f '*) + cp "$3" "$ARC_TEST_ROOT/desired-pod" + jq --arg image "ghcr.io/devantler-tech/ksail-analysis-runner@$digest" ' + .metadata.uid="owned-uid" | .spec.nodeName="autoscale-ksail-analysis-0123456789abcdef" | + .status={podIP:"192.0.2.10", + containerStatuses:[{name:"runner",ready:true,restartCount:0,containerID:"runtime-id",imageID:$image}], + initContainerStatuses:[{name:"init-runner-home",restartCount:0,imageID:$image,state:{terminated:{exitCode:0}}}]}' \ + "$3" >"$ARC_TEST_ROOT/live-pod" + if [[ "$ARC_TEST_CASE" == unknown-create ]]; then exit 1; fi + cat "$ARC_TEST_ROOT/live-pod" ;; + *'wait pod/'*'--for=condition=Ready --timeout=600s') + [[ "$ARC_TEST_TIMEOUT" == 630s ]] || exit 95 ;; + *'wait pod/'*'--for=delete --timeout=25s') ;; + *'get pod arc-proof-'*) + [[ -e "$ARC_TEST_ROOT/live-pod" ]] || exit 0 + if [[ "$ARC_TEST_CASE" == mutated-init ]]; then + jq '.spec.initContainers[0].securityContext.privileged=true' "$ARC_TEST_ROOT/live-pod" + else cat "$ARC_TEST_ROOT/live-pod"; fi ;; + 'delete --raw='*) + [[ "$ARC_TEST_CASE" != replacement-race ]] || { touch "$ARC_TEST_ROOT/replacement-preserved"; exit 1; } + jq -e '.kind=="DeleteOptions" and .preconditions.uid=="owned-uid" and .gracePeriodSeconds==5' "$4" >/dev/null || exit 96 + rm "$ARC_TEST_ROOT/live-pod" + touch "$ARC_TEST_ROOT/deleted" ;; + 'get node autoscale-ksail-analysis-0123456789abcdef '*) + jq -n '{metadata:{name:"autoscale-ksail-analysis-0123456789abcdef",uid:"autoscale-ksail-analysis-0123456789abcdef-uid",labels:{"platform.devantler.tech/ksail-analysis":"enabled","node.kubernetes.io/instance-type":"cx53"}}, + spec:{taints:[{key:"platform.devantler.tech/ksail-analysis",value:"enabled",effect:"NoSchedule"}]}, + status:{nodeInfo:{operatingSystem:"linux",architecture:"amd64"},allocatable:{cpu:"15",memory:"30Gi","ephemeral-storage":"100Gi"}, + conditions:[{type:"Ready",status:"True"}]}}' ;; + 'get pods -A --field-selector spec.nodeName=autoscale-ksail-analysis-0123456789abcdef '*) + printf 'P\towned-uid\tRunning\nR\t3\t12Gi\t32Gi\nP\tsystem-uid\tRunning\nR\t100m\t1Gi\t1Gi\n' + if [[ "$ARC_TEST_CASE" == insufficient-memory ]]; then printf 'R\t100m\t25Gi\t1Gi\n'; fi ;; + *'get pods -l k8s-app=cilium '*) + printf '{"items":[{"metadata":{"name":"observer","uid":"observer-uid"},"spec":{"nodeName":"autoscale-ksail-analysis-0123456789abcdef"}, + "status":{"podIP":"192.0.2.11","containerStatuses":[{"name":"cilium-agent","ready":true,"restartCount":0,"containerID":"observer-id"}],"conditions":[{"type":"Ready","status":"True"}]}}]}' ;; + *'get configmap cilium-config '*) + printf '{"data":{"cluster-name":"fixture"}}' ;; + *'exec observer -c cilium-agent -- hubble status '*) ;; + *'get pods -l k8s-app=kube-dns '*) + printf '{"items":[{"metadata":{"name":"dns","uid":"dns-uid"},"spec":{"nodeName":"baseline-node"}, + "status":{"podIP":"192.0.2.12","containerStatuses":[{"name":"dns","ready":true,"restartCount":0,"containerID":"dns-id"}],"conditions":[{"type":"Ready","status":"True"}]}}]}' ;; + *'get endpoints kubernetes '*) + printf '{"subsets":[{"addresses":[{"ip":"192.0.2.13"}],"ports":[{"name":"https","port":6443}]}]}' ;; + 'get --raw /api/v1/namespaces/kube-system/pods/dns:8181/proxy/ready') + [[ "$ARC_TEST_CASE" != unhealthy-target ]] || { printf 'not ready'; exit 0; } + printf OK ;; + '--server=https://192.0.2.13:6443 get --raw /readyz') printf ok ;; + *'exec arc-proof-'*'-- curl '*) + if [[ "$ARC_TEST_CASE" == curl-transport ]]; then exit 1; fi + if [[ "$ARC_TEST_CASE" == allowed-egress ]]; then exit 0; fi + exit 28 ;; + *'exec arc-proof-'*'-- test -f /tmp/arc-proof-ready') ;; + *'exec observer -c cilium-agent -- hubble observe '*) + destination=192.0.2.12 port=8181 target_ns=kube-system target_pod=dns + [[ "$args" != *'--to-ip 192.0.2.13'* ]] || { destination=192.0.2.13; port=6443; target_ns=''; target_pod=''; } + verdict=DROPPED source=192.0.2.10 + [[ "$ARC_TEST_CASE" != forwarded-flow ]] || verdict=FORWARDED + [[ "$ARC_TEST_CASE" != wrong-source ]] || source=192.0.2.99 + jq -n --arg verdict "$verdict" --arg source "$source" --arg destination "$destination" --argjson port "$port" \ + --arg target_ns "$target_ns" --arg target_pod "$target_pod" '{flow:{verdict:$verdict,drop_reason_desc:"POLICY_DENIED", + traffic_direction:"EGRESS",source:{namespace:"arc-ksail-analysis",pod_name:"arc-proof-123-1"}, + destination:{namespace:$target_ns,pod_name:$target_pod},IP:{source:$source,destination:$destination}, + l4:{TCP:{destination_port:$port,flags:{SYN:true}}},node_name:"fixture/autoscale-ksail-analysis-0123456789abcdef",time:"2026-10-05T22:00:00Z"}}' + [[ "$ARC_TEST_CASE" != observer-loss ]] || printf '{"lost_events":{"num_events_lost":"1"}}' + [[ "$ARC_TEST_CASE" != observer-diagnostics ]] || printf 'observer warning\n' >&2 ;; + *'get pod observer '*) + printf '{"metadata":{"name":"observer","uid":"observer-uid"},"spec":{"nodeName":"autoscale-ksail-analysis-0123456789abcdef"}, + "status":{"podIP":"192.0.2.11","containerStatuses":[{"name":"cilium-agent","ready":true,"restartCount":0,"containerID":"observer-id"}]}}' ;; + *'get pod dns '*) + printf '{"metadata":{"name":"dns","uid":"dns-uid"},"spec":{"nodeName":"baseline-node"}, + "status":{"podIP":"192.0.2.12","containerStatuses":[{"name":"dns","ready":true,"restartCount":0,"containerID":"dns-id"}]}}' ;; + *) printf 'unrecognized fixture command\n' >&2; exit 97 ;; +esac +SH +chmod +x "$scratch/bin/"* + +run_case() { + local name=$1 expected=$2 code=0 + rm -f "$scratch/live-pod" "$scratch/deleted" "$scratch/replacement-preserved" + ( + cd "$scratch" + PATH="$scratch/bin:$PATH" GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform \ + GITHUB_RUN_ID=123 GITHUB_RUN_ATTEMPT=1 PLATFORM_MANIFEST_DIGEST="$digest" ARC_TEST_CASE="$name" \ + bash scripts/verify-ksail-arc-runtime.sh --if-active + ) >"$scratch/stdout" 2>"$scratch/stderr" || code=$? + if [[ "$expected" == pass ]]; then + [[ "$code" == 0 && -e "$scratch/deleted" ]] || { printf 'FAIL: %s (exit %s)\n' "$name" "$code" >&2; cat "$scratch/stderr" >&2; exit 1; } + rg -q '^PASS: ARC registration' "$scratch/stdout" || exit 1 + else + [[ "$code" != 0 ]] || { printf 'FAIL: accepted %s\n' "$name" >&2; exit 1; } + case "$name" in + replacement-race) [[ -e "$scratch/replacement-preserved" && ! -e "$scratch/deleted" && "$code" == 4 ]] || exit 1 ;; + unknown-create) [[ -e "$scratch/live-pod" && ! -e "$scratch/deleted" && "$code" == 4 ]] || exit 1 ;; + failed-node-cleanup) [[ -e "$scratch/deleted" && "$code" == 4 ]] || exit 1 ;; + esac + fi + printf 'PASS: %s\n' "$name" +} + +printf 'resources: []\n' >"$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" +printf 'resources: []\n' >"$scratch/k8s/providers/hetzner/infrastructure/kustomization.yaml" +( + cd "$scratch" + PATH="$scratch/bin:$PATH" bash scripts/verify-ksail-arc-runtime.sh --if-active +) >"$scratch/inactive" +rg -q 'inactive source; no runtime access' "$scratch/inactive" +printf 'resources: ["../../../../bases/infrastructure/controllers/actions-runner-controller/"]\n' \ + >"$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" +printf 'resources: ["../../../bases/infrastructure/ksail-analysis-runners/"]\n' \ + >"$scratch/k8s/providers/hetzner/infrastructure/kustomization.yaml" +if (cd "$scratch"; PATH="$scratch/bin:$PATH" GITHUB_ACTIONS=false bash scripts/verify-ksail-arc-runtime.sh --if-active) \ + >"$scratch/unauthorized-out" 2>"$scratch/unauthorized-error"; then exit 1; fi +rg -q 'FAIL at deployment-identity' "$scratch/unauthorized-error" +run_case complete-proof pass +for name in no-registration bad-signature tampered-image wider-ceiling full-quota stale-quota admission-transport mutated-init \ + unknown-create replacement-race insufficient-memory unhealthy-target curl-transport allowed-egress \ + forwarded-flow wrong-source observer-loss observer-diagnostics failed-node-cleanup; do + run_case "$name" fail +done +printf 'PASS: inactive and unauthorized sources never reach runtime access\n' diff --git a/scripts/verify-ksail-arc-runtime.sh b/scripts/verify-ksail-arc-runtime.sh new file mode 100644 index 0000000000..52456953f6 --- /dev/null +++ b/scripts/verify-ksail-arc-runtime.sh @@ -0,0 +1,306 @@ +#!/usr/bin/env bash +# CI-owned, bounded runtime acceptance. No Secret or generated JIT configuration is read. +set -euo pipefail +umask 077 + +fail() { printf 'ARC acceptance: FAIL at %s\n' "$1" >&2; exit 1; } +[[ "$#" -eq 1 && "$1" == --if-active ]] || fail arguments +readonly controller_overlay=k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml +readonly pool_overlay=k8s/providers/hetzner/infrastructure/kustomization.yaml +controller_active=$(yq '[.resources[] | select(. == "../../../../bases/infrastructure/controllers/actions-runner-controller/")] | length' "$controller_overlay") +pool_active=$(yq '[.resources[] | select(. == "../../../bases/infrastructure/ksail-analysis-runners/")] | length' "$pool_overlay") +if [[ "$controller_active" == 0 && "$pool_active" == 0 ]]; then + printf 'ARC acceptance: inactive source; no runtime access\n' + exit 0 +fi +[[ "$controller_active" == 1 && "$pool_active" == 1 ]] || fail partial-activation +[[ "${GITHUB_ACTIONS:-}" == true && "${GITHUB_REPOSITORY:-}" == devantler-tech/platform ]] || fail deployment-identity +[[ "${GITHUB_RUN_ID:-}" =~ ^[1-9][0-9]{0,19}$ && "${GITHUB_RUN_ATTEMPT:-}" =~ ^[1-9][0-9]{0,2}$ ]] || fail run-identity +[[ "${PLATFORM_MANIFEST_DIGEST:-}" =~ ^sha256:[0-9a-f]{64}$ ]] || fail revision +readonly context=admin@prod namespace=arc-ksail-analysis +readonly ownership_label=platform.devantler.tech/arc-runtime-probe +readonly probe="arc-proof-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" +scratch=$(mktemp -d) +readonly scratch +created=false +probe_uid='' +stage=initialization +trap 'printf "ARC acceptance: FAIL at %s\n" "$stage" >&2' ERR + +kc() { timeout 35s kubectl --context "$context" --request-timeout=30s "$@" 2>/dev/null; } +wait_ready() { + timeout 630s kubectl --context "$context" --request-timeout=610s -n "$namespace" \ + wait "pod/$probe" --for=condition=Ready --timeout=600s 2>/dev/null +} +quiet() { "$@" >/dev/null 2>"$scratch/command-error"; } + +cleanup() { + local result=$? failed=0 remaining='' + trap - EXIT ERR INT TERM + if "$created"; then + # A timed-out create may have succeeded; without its response UID, fail closed. + # A DELETE precondition binds the mutation atomically, including replacement races. + local current + if [[ -z "$probe_uid" ]]; then + failed=1 + else + current=$(kc -n "$namespace" get pod "$probe" --ignore-not-found -o json) || failed=1 + if [[ -n "$current" ]]; then + if ! jq -e --arg name "$probe" --arg key "$ownership_label" --arg owner "$probe" --arg uid "$probe_uid" \ + '.metadata.name == $name and .metadata.labels[$key] == $owner and .metadata.uid == $uid' \ + <<<"$current" >/dev/null; then + failed=1 + else + jq -n --arg uid "$probe_uid" '{apiVersion:"v1",kind:"DeleteOptions", + preconditions:{uid:$uid},gracePeriodSeconds:5,propagationPolicy:"Background"}' >"$scratch/delete.json" + quiet kc delete --raw="/api/v1/namespaces/$namespace/pods/$probe" -f "$scratch/delete.json" || failed=1 + quiet kc -n "$namespace" wait "pod/$probe" --for=delete --timeout=25s || failed=1 + fi + fi + fi + current=$(kc -n "$namespace" get pod "$probe" --ignore-not-found -o json) || failed=1 + [[ -z "$current" ]] || failed=1 + remaining=$(kc -n "$namespace" get pods -l "$ownership_label=$probe" -o json) || failed=1 + jq -e '.items | length == 0' <<<"$remaining" >/dev/null || failed=1 + # The dedicated min-zero pool may take the configured unneeded interval to drain. + # Never delete nodes or resize a shared pool to manufacture cleanup. + local deadline=$((SECONDS + 1200)) + while ((SECONDS < deadline)); do + remaining=$(kc get nodes -o json) || { failed=1; break; } + if jq -e '.items | all(.[]; .metadata.labels["platform.devantler.tech/ksail-analysis"] != "enabled" and + (.metadata.name | startswith("autoscale-ksail-analysis-") | not))' <<<"$remaining" >/dev/null; then break; fi + sleep 10 + done + jq -e '.items | all(.[]; .metadata.labels["platform.devantler.tech/ksail-analysis"] != "enabled" and + (.metadata.name | startswith("autoscale-ksail-analysis-") | not))' <<<"$remaining" >/dev/null || failed=1 + fi + rm -rf "$scratch" + if [[ "$failed" != 0 ]]; then printf 'ARC acceptance: FAIL cleanup\n' >&2; exit 4; fi + printf 'ARC acceptance: cleanup verified\n' + exit "$result" +} +trap cleanup EXIT +trap 'exit 143' INT TERM + +stage=deployed-revision +quiet bash scripts/wait-for-platform-flux-revision.sh "$PLATFORM_MANIFEST_DIGEST" +for layer in infrastructure apps; do + kc -n flux-system get kustomization "$layer" -o json >"$scratch/layer.json" + jq -e --arg revision "latest@$PLATFORM_MANIFEST_DIGEST" \ + '.status.lastAppliedRevision == $revision and .status.observedGeneration == .metadata.generation and + any(.status.conditions[]; .type == "Ready" and .status == "True")' "$scratch/layer.json" >/dev/null +done + +stage=registration-and-bounds +kc -n "$namespace" get autoscalingrunnerset.actions.github.com ksail-code-quality -o json >"$scratch/ars.json" +jq -e ' + .status.phase == "Running" and .status.observedGeneration == .metadata.generation and + ((.metadata.annotations["runner-scale-set-id"] | tonumber) > 0) and + .spec.githubConfigUrl == "https://github.com/devantler-tech/ksail" and + .spec.githubConfigSecret == "arc-ksail-app" and .spec.runnerScaleSetName == "ksail-code-quality" and + .spec.minRunners == 0 and .spec.maxRunners == 1 and + .spec.template.spec.serviceAccountName == "ksail-code-quality-gha-rs-no-permission" and + .spec.template.spec.automountServiceAccountToken == false and + .spec.template.spec.securityContext.runAsUser == 1001 and + .spec.template.spec.securityContext.runAsGroup == 1001 and + .spec.template.spec.securityContext.fsGroup == 1001 and + .spec.template.spec.securityContext.seccompProfile.type == "RuntimeDefault" and + (.spec.template.spec.containers | length) == 1 and + (.spec.template.spec.initContainers | length) == 1 and + .spec.template.spec.containers[0].resources.requests.memory == "12Gi" and + .spec.template.spec.containers[0].resources.limits.memory == "14Gi" and + .spec.template.spec.containers[0].resources.requests.cpu == "3" and + .spec.template.spec.containers[0].resources.limits.cpu == "3500m" and + .spec.template.spec.containers[0].resources.requests["ephemeral-storage"] == "32Gi" and + .spec.template.spec.containers[0].resources.limits["ephemeral-storage"] == "48Gi" and + .spec.template.spec.nodeSelector["platform.devantler.tech/ksail-analysis"] == "enabled" and + .spec.template.spec.tolerations == [{"key":"platform.devantler.tech/ksail-analysis","operator":"Equal","value":"enabled","effect":"NoSchedule"}] and + .spec.template.spec.volumes == [{"name":"runner-home","emptyDir":{"sizeLimit":"40Gi"}},{"name":"runner-tmp","emptyDir":{"sizeLimit":"2Gi"}}] and + all((.spec.template.spec.containers + .spec.template.spec.initContainers)[]; + .securityContext.readOnlyRootFilesystem == true and .securityContext.privileged == false and + .securityContext.allowPrivilegeEscalation == false and .securityContext.capabilities.drop == ["ALL"] and + (.env // [] | length) == 0 and .envFrom == null and + (.image | test("^ghcr.io/devantler-tech/ksail-analysis-runner@sha256:[0-9a-f]{64}$"))) and + .spec.template.spec.initContainers[0].image == .spec.template.spec.containers[0].image +' "$scratch/ars.json" >/dev/null +image=$(jq -r '.spec.template.spec.containers[0].image' "$scratch/ars.json") +expected_image=$(yq '.spec.values.template.spec.containers[0].image' k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml) +[[ "$image" == "$expected_image" ]] || fail image-drift +stage=immutable-image +pinned_digest=${image##*@} +quiet timeout 90s cosign verify \ + --certificate-identity 'https://github.com/devantler-tech/ksail/.github/workflows/publish-ksail-analysis-runner.yaml@refs/heads/main' \ + --certificate-oidc-issuer 'https://token.actions.githubusercontent.com' "$image" +timeout 90s docker buildx imagetools inspect "$image" --raw >"$scratch/image.json" 2>"$scratch/registry-error" +[[ ! -s "$scratch/registry-error" ]] || fail registry-diagnostics +go run ./scripts/verify-ksail-arc-runtime verify-image --digest "$pinned_digest" \ + <"$scratch/image.json" >"$scratch/runtime-digest" +runtime_digest=$(cat "$scratch/runtime-digest") +readonly runtime_image="ghcr.io/devantler-tech/ksail-analysis-runner@$runtime_digest" +if [[ "$runtime_digest" != "$pinned_digest" ]]; then + timeout 90s docker buildx imagetools inspect "$runtime_image" --raw >"$scratch/runtime-image.json" 2>"$scratch/registry-error" + [[ ! -s "$scratch/registry-error" ]] || fail registry-diagnostics + go run ./scripts/verify-ksail-arc-runtime verify-image --digest "$runtime_digest" \ + <"$scratch/runtime-image.json" >"$scratch/runtime-readback" + cmp -s "$scratch/runtime-digest" "$scratch/runtime-readback" || fail runtime-descriptor +fi +stage=idle-pool-baseline +kc -n kube-system get deployment cluster-autoscaler-hetzner-cluster-autoscaler \ + -o jsonpath='{range .spec.template.spec.containers[*].args[*]}{.}{"\n"}{end}' >"$scratch/autoscaler-args" +declared_ceiling=$(rg '^--max-nodes-total=' "$scratch/autoscaler-args") +declared_pool=$(rg '^--nodes=.*:autoscale-ksail-analysis$' "$scratch/autoscaler-args") +declared_provider=$(rg '^--cloud-provider=' "$scratch/autoscaler-args") +[[ "$declared_ceiling" == --max-nodes-total=9 && \ + "$declared_pool" == --nodes=0:1:cx53:fsn1:autoscale-ksail-analysis && \ + "$declared_provider" == --cloud-provider=hetzner ]] || fail autoscaler-boundary +kc -n "$namespace" get pods -l platform.devantler.tech/arc-role=runner -o json \ + | jq -e '.items | length == 0' >/dev/null +kc -n "$namespace" get pods -l "$ownership_label" -o json \ + | jq -e '.items | length == 0' >/dev/null +kc get nodes -o json >"$scratch/nodes.json" +jq -e --arg key platform.devantler.tech/ksail-analysis \ + '(.items | length) < 9 and all(.items[]; .metadata.labels[$key] != "enabled" and + (.metadata.name | startswith("autoscale-ksail-analysis-") | not) and + (.metadata.uid | type == "string" and length > 0))' "$scratch/nodes.json" >/dev/null + +stage=restricted-admission +kc -n "$namespace" get resourcequotas -o json >"$scratch/quotas.json" +quiet go run ./scripts/verify-ksail-arc-runtime verify-quota <"$scratch/quotas.json" +# Retain the actual chart-generated template; replace only process/lifecycle and run identity. +jq --arg name "$probe" --arg ns "$namespace" --arg key "$ownership_label" ' + {apiVersion:"v1",kind:"Pod",metadata:{name:$name,namespace:$ns, + labels:(.spec.template.metadata.labels + {($key):$name})},spec:.spec.template.spec} | + .spec.restartPolicy="Never" | .spec.activeDeadlineSeconds=1200 | + .spec.containers[0].command=["/bin/sh","-ec", + "/usr/local/bin/ksail-analysis-smoke --live-runner; test ! -e /var/run/secrets/kubernetes.io/serviceaccount/token; curl --proto =https --tlsv1.2 --fail --silent --show-error --max-time 20 --output /dev/null https://api.github.com/zen; printf ready > /tmp/arc-proof-ready; sleep 900"] | + .spec.containers[0].readinessProbe={exec:{command:["/bin/sh","-ec","test -f /tmp/arc-proof-ready"]},periodSeconds:2} +' "$scratch/ars.json" >"$scratch/pod.json" +quiet kc create --dry-run=server -f "$scratch/pod.json" +for negative in privileged hostpath; do + if [[ "$negative" == privileged ]]; then + jq '.spec.containers[0].securityContext.privileged=true' "$scratch/pod.json" >"$scratch/negative.json" + else + jq '.spec.volumes += [{"name":"forbidden-host","hostPath":{"path":"/","type":"Directory"}}]' "$scratch/pod.json" >"$scratch/negative.json" + fi + if timeout 35s kubectl --context "$context" --request-timeout=30s create --dry-run=server \ + -f "$scratch/negative.json" >"$scratch/denied-out" 2>"$scratch/denied-error"; then fail negative-admission; fi + # An API/transport failure does not prove admission interception. + rg -q 'violates PodSecurity.*restricted|admission webhook.*denied the request' "$scratch/denied-error" || fail unproven-admission-denial + if [[ "$negative" == privileged ]]; then + rg -qi 'privileged' "$scratch/denied-error" || fail unrelated-admission-denial + else + rg -qi 'hostPath|host.path' "$scratch/denied-error" || fail unrelated-admission-denial + fi +done +created=true +kc create -f "$scratch/pod.json" -o json >"$scratch/created-pod.json" +probe_uid=$(jq -er --arg name "$probe" --arg ns "$namespace" --arg key "$ownership_label" \ + 'select(.metadata.name == $name and .metadata.namespace == $ns and .metadata.labels[$key] == $name) | + .metadata.uid | select(type == "string" and length > 0)' "$scratch/created-pod.json") +quiet wait_ready +kc -n "$namespace" get pod "$probe" -o json >"$scratch/live-pod.json" +jq -e --arg uid "$probe_uid" '.metadata.uid == $uid' "$scratch/live-pod.json" >/dev/null +probe_ip=$(jq -r '.status.podIP' "$scratch/live-pod.json") +probe_node=$(jq -r '.spec.nodeName' "$scratch/live-pod.json") +quiet go run ./scripts/verify-ksail-arc-runtime verify-pod --desired "$scratch/pod.json" \ + --actual "$scratch/live-pod.json" --runtime-image "$runtime_image" +kc get node "$probe_node" -o json >"$scratch/node.json" +jq -e --slurpfile before "$scratch/nodes.json" ' + .metadata.uid as $uid | ($uid | type == "string" and length > 0) and + all($before[0].items[]; .metadata.uid != $uid) and + (.metadata.name | test("^autoscale-ksail-analysis-[0-9a-f]{16}$")) and + .metadata.labels["platform.devantler.tech/ksail-analysis"] == "enabled" and + .metadata.labels["node.kubernetes.io/instance-type"] == "cx53" and + .status.nodeInfo.operatingSystem == "linux" and .status.nodeInfo.architecture == "amd64" and + any(.spec.taints[]; .key == "platform.devantler.tech/ksail-analysis" and .value == "enabled" and .effect == "NoSchedule") and + any(.status.conditions[]; .type == "Ready" and .status == "True")' "$scratch/node.json" >/dev/null +node_uid=$(jq -er '.metadata.uid' "$scratch/node.json") +kc get nodes -o json | jq -e --arg uid "$node_uid" --arg name "$probe_node" \ + '(.items | length) <= 9 and any(.items[]; .metadata.uid == $uid and .metadata.name == $name) and + ([.items[] | select(.metadata.labels["platform.devantler.tech/ksail-analysis"] == "enabled" or + (.metadata.name | startswith("autoscale-ksail-analysis-")))] | length) == 1' >/dev/null +stage=allocatable-headroom +# Print only UIDs/phases and resource quantities, never runner environment/JIT data. +kc get pods -A --field-selector "spec.nodeName=$probe_node" -o jsonpath='{range .items[*]}P{"\t"}{.metadata.uid}{"\t"}{.status.phase}{"\n"}{range .spec.containers[*]}R{"\t"}{.resources.requests.cpu}{"\t"}{.resources.requests.memory}{"\t"}{.resources.requests.ephemeral-storage}{"\n"}{end}{range .spec.initContainers[*]}R{"\t"}{.resources.requests.cpu}{"\t"}{.resources.requests.memory}{"\t"}{.resources.requests.ephemeral-storage}{"\n"}{end}R{"\t"}{.spec.overhead.cpu}{"\t"}{.spec.overhead.memory}{"\t"}{.spec.overhead.ephemeral-storage}{"\n"}{end}' >"$scratch/reservations" +quiet go run ./scripts/verify-ksail-arc-runtime verify-budget --node "$scratch/node.json" \ + --reservations "$scratch/reservations" --probe-uid "$probe_uid" + +stage='observer-and-healthy-targets' +kc -n kube-system get pods -l k8s-app=cilium --field-selector "spec.nodeName=$probe_node" -o json \ + | jq -e '[.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))] | + if length == 1 then .[0] else error("observer count") end' >"$scratch/agent.json" +agent=$(jq -r '.metadata.name' "$scratch/agent.json") +kc -n kube-system get configmap cilium-config -o json >"$scratch/cilium-config.json" +cluster=$(jq -er '.data["cluster-name"] | select(length > 0)' "$scratch/cilium-config.json") +observer="$cluster/$probe_node" +quiet kc -n kube-system exec "$agent" -c cilium-agent -- hubble status \ + --server unix:///var/run/cilium/hubble.sock --timeout 5s --request-timeout 5s --output json +kc -n kube-system get pods -l k8s-app=kube-dns -o json \ + | jq -e '[.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))][0] | + select(.metadata.uid != null and .status.podIP != null)' >"$scratch/dns.json" +dns_pod=$(jq -r '.metadata.name' "$scratch/dns.json") +dns_ip=$(jq -r '.status.podIP' "$scratch/dns.json") +kc -n default get endpoints kubernetes -o json >"$scratch/api.json" +api_ip=$(jq -er '.subsets[0].addresses[0].ip' "$scratch/api.json") +api_port=$(jq -er '.subsets[0].ports[] | select(.name == "https") | .port' "$scratch/api.json") + +healthy_dns() { + local reply + reply=$(kc get --raw "/api/v1/namespaces/kube-system/pods/$dns_pod:8181/proxy/ready") + [[ "$reply" == OK ]] || fail unhealthy-internal-control +} +healthy_api() { + local reply + reply=$(kc --server="https://$api_ip:$api_port" get --raw /readyz) + [[ "$reply" == ok ]] || fail unhealthy-api-control +} +identity() { + jq -c '{uid:.metadata.uid,ip:.status.podIP,node:.spec.nodeName, + containers:[.status.containerStatuses[]|{name,containerID,restartCount,ready}]}' +} +negative_egress() { + local destination=$1 port=$2 url=$3 target_ns=$4 target_pod=$5 start end result + start=$(date -u +%Y-%m-%dT%H:%M:%S.%NZ) + # Only curl's connect/total timeout counts as a denied attempt; exec/transport failure does not. + if kc -n "$namespace" exec "$probe" -c runner -- curl --noproxy '*' --connect-timeout 3 \ + --max-time 5 --silent --show-error --output /dev/null "$url" >"$scratch/attempt-out" 2>"$scratch/attempt-error"; then + fail network-isolation + else + result=$? + fi + [[ "$result" == 28 ]] || fail unproven-curl-timeout + # Re-prove that exec still reaches this process after the bounded attempt. + quiet kc -n "$namespace" exec "$probe" -c runner -- test -f /tmp/arc-proof-ready + end=$(date -u +%Y-%m-%dT%H:%M:%S.%NZ) + timeout 20s kubectl --context "$context" --request-timeout=15s -n kube-system exec "$agent" -c cilium-agent -- \ + hubble observe --server unix:///var/run/cilium/hubble.sock --timeout 5s --since "$start" --until "$end" \ + --from-ip "$probe_ip" --from-pod "$namespace/$probe" --to-ip "$destination" --to-port "$port" \ + --protocol tcp --traffic-direction egress --verdict DROPPED --drop-reason-desc POLICY_DENIED --output jsonpb \ + >"$scratch/flows" 2>"$scratch/observer-error" + [[ ! -s "$scratch/observer-error" ]] || fail observer-diagnostics + quiet go run ./scripts/verify-ksail-arc-runtime --pod "$probe" --source "$probe_ip" \ + --destination "$destination" --port "$port" --node "$observer" --since "$start" --until "$end" \ + --destination-namespace "$target_ns" --destination-pod "$target_pod" <"$scratch/flows" +} +stage=intercepted-egress-denials +healthy_dns +healthy_api +negative_egress "$dns_ip" 8181 "http://$dns_ip:8181/ready" kube-system "$dns_pod" +negative_egress "$api_ip" "$api_port" "https://$api_ip:$api_port/readyz" '' '' +healthy_dns +healthy_api +kc -n "$namespace" get pod "$probe" -o json | identity >"$scratch/probe-after" +identity <"$scratch/live-pod.json" >"$scratch/probe-before" +cmp -s "$scratch/probe-before" "$scratch/probe-after" || fail probe-churn +kc -n kube-system get pod "$agent" -o json | identity >"$scratch/agent-after" +identity <"$scratch/agent.json" >"$scratch/agent-before" +cmp -s "$scratch/agent-before" "$scratch/agent-after" || fail observer-churn +kc -n kube-system get pod "$dns_pod" -o json | identity >"$scratch/dns-after" +identity <"$scratch/dns.json" >"$scratch/dns-before" +cmp -s "$scratch/dns-before" "$scratch/dns-after" || fail target-churn +kc -n default get endpoints kubernetes -o json | jq -c '.subsets' >"$scratch/api-after" +jq -c '.subsets' "$scratch/api.json" >"$scratch/api-before" +cmp -s "$scratch/api-before" "$scratch/api-after" || fail api-target-churn + +stage=complete +printf 'PASS: ARC registration, restricted image, admission and two intercepted egress denials; manifest=%s\n' "$PLATFORM_MANIFEST_DIGEST" diff --git a/scripts/verify-ksail-arc-runtime/budget.go b/scripts/verify-ksail-arc-runtime/budget.go new file mode 100644 index 0000000000..24918628f4 --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/budget.go @@ -0,0 +1,157 @@ +package main + +import ( + "bufio" + "encoding/json" + "errors" + "math/big" + "regexp" + "strings" +) + +var quantityPattern = regexp.MustCompile(`^([0-9]+(?:\.[0-9]+)?)([a-zA-Z]*)$`) + +func quantity(value string) (*big.Rat, error) { + if value == "" { + return new(big.Rat), nil + } + parts := quantityPattern.FindStringSubmatch(value) + if parts == nil { + return nil, errors.New("unknown resource quantity") + } + number, ok := new(big.Rat).SetString(parts[1]) + if !ok { + return nil, errors.New("invalid resource quantity") + } + scale := map[string]string{"": "1", "n": "0.000000001", "u": "0.000001", "m": "0.001", + "k": "1000", "M": "1000000", "G": "1000000000", "T": "1000000000000", + "Ki": "1024", "Mi": "1048576", "Gi": "1073741824", "Ti": "1099511627776"} + factor, ok := scale[parts[2]] + if !ok { + return nil, errors.New("unsupported resource unit") + } + ratio, _ := new(big.Rat).SetString(factor) + return number.Mul(number, ratio), nil +} + +// Sum main, init and overhead reservations conservatively. Serial init requests +// are deliberately added rather than subtracted, preserving node headroom. +func verifyBudget(nodeJSON, reservations []byte, probeUID string) error { + var node struct { + Status struct{ Allocatable map[string]string } + } + if json.Unmarshal(nodeJSON, &node) != nil { + return errors.New("invalid node capacity") + } + total := map[string]*big.Rat{} + for resource, budget := range map[string]string{"cpu": "3500m", "memory": "14Gi", "ephemeral-storage": "48Gi"} { + total[resource], _ = quantity(budget) + if node.Status.Allocatable[resource] == "" { + return errors.New("missing allocatable capacity") + } + } + scanner := bufio.NewScanner(strings.NewReader(string(reservations))) + seenHeader, skip := false, false + for scanner.Scan() { + fields := strings.Split(scanner.Text(), "\t") + switch fields[0] { + case "P": + if len(fields) != 3 || fields[1] == "" { + return errors.New("invalid reservation identity") + } + seenHeader = true + skip = fields[1] == probeUID || fields[2] == "Succeeded" || fields[2] == "Failed" + case "R": + if !seenHeader || len(fields) != 4 { + return errors.New("incomplete reservations") + } + if skip { + continue + } + for i, resource := range []string{"cpu", "memory", "ephemeral-storage"} { + value, err := quantity(fields[i+1]) + if err != nil { + return err + } + total[resource].Add(total[resource], value) + } + default: + return errors.New("unknown reservation record") + } + } + if scanner.Err() != nil || !seenHeader { + return errors.New("missing complete reservation read") + } + for resource, reserved := range total { + allocatable, err := quantity(node.Status.Allocatable[resource]) + if err != nil { + return err + } + if reserved.Cmp(allocatable) > 0 { + return errors.New("insufficient allocatable headroom") + } + } + return nil +} + +func verifyQuota(data []byte) error { + var quotas struct { + Items []struct { + Spec struct { + Scopes []string + ScopeSelector json.RawMessage + Hard map[string]string + } + Status struct{ Hard, Used map[string]string } + } + } + if json.Unmarshal(data, "as) != nil || quotas.Items == nil { + return errors.New("invalid quota read") + } + for _, q := range quotas.Items { + if len(q.Spec.Scopes) != 0 || len(q.Spec.ScopeSelector) != 0 { + return errors.New("unresolved scoped quota") + } + if len(q.Spec.Hard) == 0 { + return errors.New("missing declared quota limits") + } + for resource, needed := range map[string]string{ + "pods": "1", "count/pods": "1", "requests.cpu": "3", "requests.memory": "12Gi", "requests.ephemeral-storage": "32Gi", + "limits.cpu": "3500m", "limits.memory": "14Gi", "limits.ephemeral-storage": "48Gi", + "cpu": "3", "memory": "12Gi", "ephemeral-storage": "32Gi", + } { + hard, exists := q.Spec.Hard[resource] + if !exists { + if _, stale := q.Status.Hard[resource]; stale { + return errors.New("stale quota limit") + } + continue + } + accounted, exists := q.Status.Hard[resource] + if !exists { + return errors.New("missing quota limit accounting") + } + used, exists := q.Status.Used[resource] + if !exists { + return errors.New("incomplete quota accounting") + } + limit, err := quantity(hard) + if err != nil { + return err + } + observed, err := quantity(accounted) + if err != nil || observed.Cmp(limit) != 0 { + return errors.New("stale quota limit accounting") + } + current, err := quantity(used) + if err != nil { + return err + } + additional, _ := quantity(needed) + if new(big.Rat).Add(current, additional).Cmp(limit) > 0 { + return errors.New("insufficient namespace quota") + } + } + } + return nil +} diff --git a/scripts/verify-ksail-arc-runtime/budget_test.go b/scripts/verify-ksail-arc-runtime/budget_test.go new file mode 100644 index 0000000000..67e7d6a994 --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/budget_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "strings" + "testing" +) + +func TestCapacityAccountsForNeighborsAndInitReservations(t *testing.T) { + node := `{"status":{"allocatable":{"cpu":"7.8","memory":"30Gi","ephemeral-storage":"100Gi"}}}` + reservations := "P\towned\tRunning\nR\t3\t12Gi\t32Gi\nP\tsystem\tRunning\nR\t200m\t1Gi\t1Gi\nR\t100m\t256Mi\t1Gi\n" + if err := verifyBudget([]byte(node), []byte(reservations), "owned"); err != nil { + t.Fatal(err) + } + for _, item := range []struct{ name, node, records string }{ + {"memory", strings.ReplaceAll(node, "30Gi", "14Gi"), reservations}, + {"cpu", strings.ReplaceAll(node, "7.8", "3.5"), reservations}, + {"storage", strings.ReplaceAll(node, "100Gi", "48Gi"), reservations}, + {"unknown-quantity", node, strings.ReplaceAll(reservations, "200m", "not-known")}, + {"missing-read", node, ""}, + {"truncated-record", node, "P\tsystem\tRunning\nR\t200m\t1Gi"}, + } { + t.Run(item.name, func(t *testing.T) { + if verifyBudget([]byte(item.node), []byte(item.records), "owned") == nil { + t.Fatal("accepted insufficient or unknown headroom") + } + }) + } +} + +func TestQuotaRequiresCompleteHeadroom(t *testing.T) { + good := `{"items":[{"spec":{"hard":{"pods":"3","requests.memory":"16Gi","limits.memory":"16Gi"}},"status":{"hard":{"pods":"3","requests.memory":"16Gi","limits.memory":"16Gi"},"used":{"pods":"1","requests.memory":"1Gi","limits.memory":"1Gi"}}}]}` + if err := verifyQuota([]byte(good)); err != nil { + t.Fatal(err) + } + if err := verifyQuota([]byte(`{"items":[]}`)); err != nil { + t.Fatal(err) + } + for _, bad := range []string{ + strings.ReplaceAll(good, `"pods":"3"`, `"pods":"1"`), + strings.ReplaceAll(good, `"limits.memory":"16Gi"`, `"limits.memory":"14Gi"`), + strings.ReplaceAll(good, `"requests.memory":"1Gi",`, ``), + `{"items":[{"spec":{"hard":{"pods":"1"}},"status":{}}]}`, + `{"items":[{"spec":{"hard":{"pods":"1"}},"status":{"hard":{"pods":"5"},"used":{"pods":"1"}}}]}`, + `{"items":[{"spec":{"hard":{"pods":"1"},"scopes":["Terminating"]},"status":{"hard":{"pods":"1"},"used":{"pods":"0"}}}]}`, + `{}`, `{"items":[{"status":{"hard":{"pods":"unknown"},"used":{"pods":"0"}}}]}`, + } { + if verifyQuota([]byte(bad)) == nil { + t.Fatal("accepted incomplete or insufficient quota") + } + } +} diff --git a/scripts/verify-ksail-arc-runtime/denial.go b/scripts/verify-ksail-arc-runtime/denial.go new file mode 100644 index 0000000000..db26e7fa08 --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/denial.go @@ -0,0 +1,94 @@ +package main + +import ( + "encoding/json" + "errors" + "io" + "time" +) + +type denialTarget struct { + namespace, pod, source, destination, node string + destinationNamespace, destinationPod string + port uint16 + since, until time.Time +} + +type endpointIdentity struct { + Namespace string `json:"namespace"` + Pod string `json:"pod_name"` +} +type deniedFlow struct { + Verdict string `json:"verdict"` + Reason string `json:"drop_reason_desc"` + Direction string `json:"traffic_direction"` + Source endpointIdentity `json:"source"` + Destination endpointIdentity `json:"destination"` + IP struct { + Source string `json:"source"` + Destination string `json:"destination"` + } `json:"IP"` + L4 struct { + TCP struct { + DestinationPort uint16 `json:"destination_port"` + Flags struct { + SYN bool `json:"SYN"` + } `json:"flags"` + } `json:"TCP"` + } `json:"l4"` + Node string `json:"node_name"` + Time string `json:"time"` +} + +func verifyDenial(input io.Reader, expected denialTarget) error { + limited := &io.LimitedReader{R: input, N: 4 << 20} + decoder := json.NewDecoder(limited) + matched := false + for { + var response map[string]json.RawMessage + if err := decoder.Decode(&response); err != nil { + if errors.Is(err, io.EOF) { + break + } + return errors.New("malformed observer response") + } + for _, field := range []string{"lost_events", "lostEvents", "node_status", "nodeStatus"} { + if _, exists := response[field]; exists { + return errors.New("incomplete observer evidence") + } + } + raw, exists := response["flow"] + if !exists { + return errors.New("unknown observer response") + } + var flow deniedFlow + if err := json.Unmarshal(raw, &flow); err != nil { + return errors.New("malformed flow") + } + if matchesDenial(flow, expected) { + matched = true + } + } + if limited.N <= 0 { + return errors.New("observer evidence exceeds the bound") + } + if !matched { + return errors.New("no correlated policy denial") + } + return nil +} + +func matchesDenial(flow deniedFlow, expected denialTarget) bool { + observed, err := time.Parse(time.RFC3339Nano, flow.Time) + if err != nil || observed.Before(expected.since) || observed.After(expected.until) { + return false + } + if flow.Verdict != "DROPPED" || flow.Reason != "POLICY_DENIED" || flow.Direction != "EGRESS" || + flow.Source.Namespace != expected.namespace || flow.Source.Pod != expected.pod || + flow.IP.Source != expected.source || flow.IP.Destination != expected.destination || + flow.L4.TCP.DestinationPort != expected.port || !flow.L4.TCP.Flags.SYN || flow.Node != expected.node { + return false + } + return (expected.destinationNamespace == "" || flow.Destination.Namespace == expected.destinationNamespace) && + (expected.destinationPod == "" || flow.Destination.Pod == expected.destinationPod) +} diff --git a/scripts/verify-ksail-arc-runtime/denial_test.go b/scripts/verify-ksail-arc-runtime/denial_test.go new file mode 100644 index 0000000000..b5d87ecb02 --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/denial_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +func TestPolicyDenialMustBindEveryIdentityAndTheAttemptWindow(t *testing.T) { + t.Parallel() + expected := denialTarget{ + namespace: "arc-ksail-analysis", pod: "probe-123", source: "192.0.2.2", + destination: "192.0.2.3", port: 8181, node: "default/worker", + since: time.Date(2026, 10, 5, 22, 0, 0, 0, time.UTC), + until: time.Date(2026, 10, 5, 22, 0, 10, 0, time.UTC), + } + valid := `{"flow":{"verdict":"DROPPED","drop_reason_desc":"POLICY_DENIED","traffic_direction":"EGRESS","source":{"namespace":"arc-ksail-analysis","pod_name":"probe-123"},"IP":{"source":"192.0.2.2","destination":"192.0.2.3"},"l4":{"TCP":{"destination_port":8181,"flags":{"SYN":true}}},"node_name":"default/worker","time":"2026-10-05T22:00:05Z"}}` + cases := []struct { + name, input string + want bool + }{ + {"correlated policy drop", valid, true}, + {"timeout only", "", false}, + {"malformed observer", "{", false}, + {"other source", strings.Replace(valid, "192.0.2.2", "192.0.2.4", 1), false}, + {"same pod prefix", strings.Replace(valid, "probe-123", "probe-123-other", 1), false}, + {"other namespace", strings.Replace(valid, "arc-ksail-analysis", "other", 1), false}, + {"other destination", strings.Replace(valid, "192.0.2.3", "192.0.2.4", 1), false}, + {"other port", strings.Replace(valid, "8181", "443", 1), false}, + {"other node", strings.Replace(valid, "default/worker", "default/other", 1), false}, + {"outside attempt", strings.Replace(valid, "22:00:05", "21:59:59", 1), false}, + {"unrecognized timestamp", strings.Replace(valid, "2026-10-05T22:00:05Z", "unknown", 1), false}, + {"other direction", strings.Replace(valid, "EGRESS", "INGRESS", 1), false}, + {"other reason", strings.Replace(valid, "POLICY_DENIED", "POLICY_DENY", 1), false}, + {"forwarded event", strings.Replace(valid, "DROPPED", "FORWARDED", 1), false}, + {"not a connection attempt", strings.Replace(valid, `"SYN":true`, `"SYN":false`, 1), false}, + {"lost events", valid + "\n" + `{"lost_events":{"num_events_lost":1}}`, false}, + {"observer node failure", valid + "\n" + `{"node_status":{"state_change":"NODE_ERROR"}}`, false}, + {"trailing malformed response", valid + "\n{", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := verifyDenial(strings.NewReader(tc.input), expected) + if (err == nil) != tc.want { + t.Fatalf("verdict success = %v, want %v", err == nil, tc.want) + } + }) + } +} diff --git a/scripts/verify-ksail-arc-runtime/image.go b/scripts/verify-ksail-arc-runtime/image.go new file mode 100644 index 0000000000..3d24905dce --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/image.go @@ -0,0 +1,63 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" +) + +func runtimeDigest(data []byte, digest string) (string, error) { + if !validDigest(digest) || len(data) > 4<<20 { + return "", errors.New("invalid image evidence") + } + // Buildx --raw may append one display newline. Accept only bytes whose hash + // equals the immutable signed descriptor, never reserialized JSON or a tag. + if fmt.Sprintf("sha256:%x", sha256.Sum256(data)) != digest { + if len(data) == 0 || data[len(data)-1] != '\n' || fmt.Sprintf("sha256:%x", sha256.Sum256(data[:len(data)-1])) != digest { + return "", errors.New("image descriptor mismatch") + } + data = data[:len(data)-1] + } + var manifest struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Manifests []struct { + Digest string `json:"digest"` + Platform struct{ OS, Architecture, Variant string } `json:"platform"` + } `json:"manifests"` + } + if json.Unmarshal(data, &manifest) != nil || manifest.SchemaVersion != 2 { + return "", errors.New("invalid image manifest") + } + switch manifest.MediaType { + case "application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json": + return digest, nil + case "application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json": + runtime := "" + for _, item := range manifest.Manifests { + if item.Platform.OS != "linux" || item.Platform.Architecture != "amd64" { + continue + } + if runtime != "" || item.Platform.Variant != "" || !validDigest(item.Digest) { + return "", errors.New("ambiguous amd64 runtime") + } + runtime = item.Digest + } + if runtime == "" { + return "", errors.New("missing amd64 runtime") + } + return runtime, nil + default: + return "", errors.New("unsupported image manifest") + } +} + +func validDigest(digest string) bool { + if len(digest) != 71 || digest[:7] != "sha256:" { + return false + } + data, err := hex.DecodeString(digest[7:]) + return err == nil && len(data) == 32 +} diff --git a/scripts/verify-ksail-arc-runtime/image_test.go b/scripts/verify-ksail-arc-runtime/image_test.go new file mode 100644 index 0000000000..a4a069f78a --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/image_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "crypto/sha256" + "fmt" + "strings" + "testing" +) + +func TestImmutableRuntimeDescriptorJoin(t *testing.T) { + child := "sha256:" + strings.Repeat("a", 64) + index := fmt.Sprintf(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"digest":%q,"platform":{"os":"linux","architecture":"amd64"}},{"digest":"attestation","platform":{"os":"unknown","architecture":"unknown"}}]}`, child) + digest := func(data string) string { return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(data))) } + for _, suffix := range []string{"", "\n"} { + got, err := runtimeDigest([]byte(index+suffix), digest(index)) + if err != nil || got != child { + t.Fatalf("signed index join: %s %v", got, err) + } + } + for name, data := range map[string]string{ + "tampered": index + " ", + "no-runtime": strings.ReplaceAll(index, "amd64", "arm64"), + "malformed-digest": strings.ReplaceAll(index, child, "latest"), + "unsupported-schema": strings.ReplaceAll(index, `"schemaVersion":2`, `"schemaVersion":1`), + "unknown-media": strings.ReplaceAll(index, "application/vnd.oci.image.index.v1+json", "unknown"), + "trailing-json": index + "{}", + "ambiguous": fmt.Sprintf(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"digest":%q,"platform":{"os":"linux","architecture":"amd64"}},{"digest":%q,"platform":{"os":"linux","architecture":"amd64"}}]}`, child, child), + } { + t.Run(name, func(t *testing.T) { + expected := digest(data) + if name == "tampered" { + expected = digest(index) + } + if _, err := runtimeDigest([]byte(data), expected); err == nil { + t.Fatal("accepted unproven runtime descriptor") + } + }) + } + manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{},"layers":[]}` + if got, err := runtimeDigest([]byte(manifest), digest(manifest)); err != nil || got != digest(manifest) { + t.Fatal(got, err) + } +} diff --git a/scripts/verify-ksail-arc-runtime/main.go b/scripts/verify-ksail-arc-runtime/main.go new file mode 100644 index 0000000000..3bdc08a5dc --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/main.go @@ -0,0 +1,120 @@ +package main + +import ( + "flag" + "fmt" + "io" + "net" + "os" + "time" +) + +func main() { + if len(os.Args) > 1 && os.Args[1] == "verify-image" { + flags := flag.NewFlagSet("verify-image", flag.ContinueOnError) + digest := flags.String("digest", "", "signed image descriptor") + err := flags.Parse(os.Args[2:]) + var selected string + if err == nil && flags.NArg() == 0 { + var data []byte + data, err = io.ReadAll(io.LimitReader(os.Stdin, (4<<20)+1)) + if err == nil { + selected, err = runtimeDigest(data, *digest) + } + } + if err != nil || selected == "" { + fmt.Fprintln(os.Stderr, "ARC acceptance: immutable image proof failed") + os.Exit(1) + } + fmt.Println(selected) + return + } + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "ARC acceptance: correlated denial proof failed") + os.Exit(1) + } + fmt.Println("PASS: ARC runtime evidence verified") +} + +func run() error { + if len(os.Args) > 1 && os.Args[1] == "verify-budget" { + flags := flag.NewFlagSet("verify-budget", flag.ContinueOnError) + node := flags.String("node", "", "private node readback") + reservations := flags.String("reservations", "", "filtered resource requests") + uid := flags.String("probe-uid", "", "owned probe UID") + if err := flags.Parse(os.Args[2:]); err != nil { + return err + } + if flags.NArg() != 0 || *node == "" || *reservations == "" || *uid == "" { + return fmt.Errorf("missing capacity readback") + } + data, err := os.ReadFile(*node) + if err != nil { + return err + } + requests, err := os.ReadFile(*reservations) + if err != nil { + return err + } + return verifyBudget(data, requests, *uid) + } + if len(os.Args) > 1 && os.Args[1] == "verify-quota" { + if len(os.Args) != 2 { + return fmt.Errorf("invalid quota arguments") + } + data, err := io.ReadAll(io.LimitReader(os.Stdin, (4<<20)+1)) + if err != nil || len(data) > 4<<20 { + return fmt.Errorf("invalid quota evidence") + } + return verifyQuota(data) + } + if len(os.Args) > 1 && os.Args[1] == "verify-pod" { + flags := flag.NewFlagSet("verify-pod", flag.ContinueOnError) + desired := flags.String("desired", "", "private desired Pod file") + actual := flags.String("actual", "", "private admitted Pod file") + image := flags.String("runtime-image", "", "verified platform image") + if err := flags.Parse(os.Args[2:]); err != nil { + return err + } + if flags.NArg() != 0 || *desired == "" || *actual == "" { + return fmt.Errorf("missing pod readback") + } + want, err := os.ReadFile(*desired) + if err != nil { + return err + } + got, err := os.ReadFile(*actual) + if err != nil { + return err + } + return verifyPod(want, got, *image) + } + pod := flag.String("pod", "", "exact owned probe name") + source := flag.String("source", "", "pinned probe address") + destination := flag.String("destination", "", "healthy target address") + destinationPod := flag.String("destination-pod", "", "optional pinned target pod") + destinationNamespace := flag.String("destination-namespace", "", "optional pinned target namespace") + node := flag.String("node", "", "exact Hubble observer identity") + port := flag.Uint("port", 0, "target TCP port") + since := flag.String("since", "", "attempt start") + until := flag.String("until", "", "attempt end") + flag.Parse() + start, err := time.Parse(time.RFC3339Nano, *since) + if err != nil { + return err + } + end, err := time.Parse(time.RFC3339Nano, *until) + if err != nil { + return err + } + if flag.NArg() != 0 || *pod == "" || *node == "" || + net.ParseIP(*source) == nil || net.ParseIP(*destination) == nil || + *port == 0 || *port > 65535 || end.Before(start) || end.Sub(start) > 30*time.Second { + return fmt.Errorf("invalid bounded target") + } + return verifyDenial(os.Stdin, denialTarget{ + namespace: "arc-ksail-analysis", pod: *pod, source: *source, destination: *destination, + destinationNamespace: *destinationNamespace, destinationPod: *destinationPod, + node: *node, port: uint16(*port), since: start, until: end, + }) +} diff --git a/scripts/verify-ksail-arc-runtime/pod.go b/scripts/verify-ksail-arc-runtime/pod.go new file mode 100644 index 0000000000..49d3d2811e --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/pod.go @@ -0,0 +1,151 @@ +package main + +import ( + "encoding/hex" + "encoding/json" + "errors" + "reflect" + "strings" +) + +func verifyPod(desired, actual []byte, runtimeImage string) error { + var expected, observed map[string]any + if json.Unmarshal(desired, &expected) != nil || json.Unmarshal(actual, &observed) != nil { + return errors.New("invalid pod readback") + } + want, ok := expected["spec"].(map[string]any) + if !ok { + return errors.New("missing desired spec") + } + got, ok := observed["spec"].(map[string]any) + if !ok { + return errors.New("missing actual spec") + } + if err := normalizePodSpec(want); err != nil { + return err + } + if err := normalizePodSpec(got); err != nil { + return err + } + if !reflect.DeepEqual(want, got) { + return errors.New("admitted pod changed runtime fields") + } + digest := strings.TrimPrefix(runtimeImage, "ghcr.io/devantler-tech/ksail-analysis-runner@sha256:") + decoded, decodeErr := hex.DecodeString(digest) + if digest == runtimeImage || decodeErr != nil || len(decoded) != 32 { + return errors.New("unproven runtime image") + } + status, ok := observed["status"].(map[string]any) + if !ok { + return errors.New("missing runtime status") + } + for _, pair := range [][2]string{{"containers", "containerStatuses"}, {"initContainers", "initContainerStatuses"}} { + containers, ok := got[pair[0]].([]any) + if !ok || len(containers) != 1 { + return errors.New("unexpected container count") + } + statuses, ok := status[pair[1]].([]any) + if !ok || len(statuses) != 1 { + return errors.New("incomplete runtime status") + } + c := containers[0].(map[string]any) + s, ok := statuses[0].(map[string]any) + imageID := strings.TrimPrefix(stringField(s, "imageID"), "docker-pullable://") + // Containerd may report the requested index identity. Both representations + // are bound to the verified signed index and its unique linux/amd64 child; + // the caller separately verifies the actual node architecture. + requested := stringField(c, "image") + if !strings.HasPrefix(requested, "ghcr.io/devantler-tech/ksail-analysis-runner@") || + !validDigest(strings.TrimPrefix(requested, "ghcr.io/devantler-tech/ksail-analysis-runner@")) { + return errors.New("unproven requested image") + } + if !ok || s["name"] != c["name"] || s["restartCount"] != float64(0) || + (imageID != runtimeImage && imageID != requested) { + return errors.New("runtime identity mismatch") + } + if pair[0] == "containers" { + if s["ready"] != true { + return errors.New("runtime not ready") + } + } else { + state, ok := s["state"].(map[string]any) + if !ok { + return errors.New("missing init result") + } + terminated, ok := state["terminated"].(map[string]any) + if !ok || terminated["exitCode"] != float64(0) { + return errors.New("init failed") + } + } + } + return nil +} + +func stringField(m map[string]any, key string) string { value, _ := m[key].(string); return value } + +// Kubernetes core/v1 defaults and the defaulttolerationseconds admission plugin +// add these exact values. Unknown fields and every nondefault value remain in the +// comparison, including added containers, credentials, mounts and permissions. +func normalizePodSpec(spec map[string]any) error { + delete(spec, "nodeName") // Scheduler assignment is verified separately against the dedicated node. + for key, value := range map[string]any{ + "dnsPolicy": "ClusterFirst", "schedulerName": "default-scheduler", + "terminationGracePeriodSeconds": float64(30), "enableServiceLinks": true, + "priority": float64(0), "preemptionPolicy": "PreemptLowerPriority", + } { + removeDefault(spec, key, value) + } + if alias, exists := spec["serviceAccount"]; exists { + if alias != spec["serviceAccountName"] { + return errors.New("service account alias mismatch") + } + delete(spec, "serviceAccount") + } + if tolerations, ok := spec["tolerations"].([]any); ok { + filtered := make([]any, 0, len(tolerations)) + for _, item := range tolerations { + t, ok := item.(map[string]any) + if !ok { + return errors.New("invalid toleration") + } + if len(t) == 4 && (t["key"] == "node.kubernetes.io/not-ready" || t["key"] == "node.kubernetes.io/unreachable") && + t["operator"] == "Exists" && t["effect"] == "NoExecute" && t["tolerationSeconds"] == float64(300) { + continue + } + filtered = append(filtered, item) + } + spec["tolerations"] = filtered + } + for _, key := range []string{"containers", "initContainers"} { + containers, ok := spec[key].([]any) + if !ok { + return errors.New("missing container specification") + } + for _, item := range containers { + c, ok := item.(map[string]any) + if !ok { + return errors.New("invalid container") + } + removeDefault(c, "imagePullPolicy", "IfNotPresent") + removeDefault(c, "terminationMessagePath", "/dev/termination-log") + removeDefault(c, "terminationMessagePolicy", "File") + for _, name := range []string{"readinessProbe", "livenessProbe", "startupProbe"} { + if probe, ok := c[name].(map[string]any); ok { + for field, value := range map[string]any{ + "timeoutSeconds": float64(1), "periodSeconds": float64(10), + "successThreshold": float64(1), "failureThreshold": float64(3), + } { + removeDefault(probe, field, value) + } + } + } + } + } + return nil +} + +func removeDefault(m map[string]any, key string, value any) { + if reflect.DeepEqual(m[key], value) { + delete(m, key) + } +} diff --git a/scripts/verify-ksail-arc-runtime/pod_test.go b/scripts/verify-ksail-arc-runtime/pod_test.go new file mode 100644 index 0000000000..f7f81ba5eb --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/pod_test.go @@ -0,0 +1,136 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestPodReadbackBindsAllRuntimeFields(t *testing.T) { + image := "ghcr.io/devantler-tech/ksail-analysis-runner@sha256:" + strings.Repeat("a", 64) + runtimeImage := "ghcr.io/devantler-tech/ksail-analysis-runner@sha256:" + strings.Repeat("b", 64) + var desired map[string]any + if err := json.Unmarshal([]byte(`{"spec":{ + "automountServiceAccountToken":false,"serviceAccountName":"no-permission", + "securityContext":{"runAsUser":1001,"runAsGroup":1001,"fsGroup":1001,"seccompProfile":{"type":"RuntimeDefault"}}, + "nodeSelector":{"analysis":"enabled"},"tolerations":[{"key":"analysis","operator":"Equal","value":"enabled","effect":"NoSchedule"}], + "volumes":[{"name":"scratch","emptyDir":{"sizeLimit":"40Gi"}}], + "restartPolicy":"Never","activeDeadlineSeconds":1200, + "containers":[{"name":"runner","image":"IMAGE","command":["/bin/sh","-ec","proof"], + "securityContext":{"privileged":false,"allowPrivilegeEscalation":false,"readOnlyRootFilesystem":true,"capabilities":{"drop":["ALL"]}}, + "resources":{"requests":{"cpu":"3","memory":"12Gi"},"limits":{"cpu":"3500m","memory":"14Gi"}}, + "volumeMounts":[{"name":"scratch","mountPath":"/home/runner"}], + "readinessProbe":{"exec":{"command":["true"]},"periodSeconds":2}}], + "initContainers":[{"name":"init","image":"IMAGE","command":["cp"],"securityContext":{"readOnlyRootFilesystem":true}}] + }}`), &desired); err != nil { + t.Fatal(err) + } + spec := desired["spec"].(map[string]any) + for _, key := range []string{"containers", "initContainers"} { + spec[key].([]any)[0].(map[string]any)["image"] = image + } + encode := func(v any) []byte { + data, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return data + } + desiredJSON := encode(desired) + actual := func() map[string]any { + var pod map[string]any + if err := json.Unmarshal(desiredJSON, &pod); err != nil { + t.Fatal(err) + } + s := pod["spec"].(map[string]any) + s["nodeName"] = "assigned-node" + s["dnsPolicy"] = "ClusterFirst" + s["schedulerName"] = "default-scheduler" + s["terminationGracePeriodSeconds"] = float64(30) + s["enableServiceLinks"] = true + s["serviceAccount"] = s["serviceAccountName"] + s["tolerations"] = append(s["tolerations"].([]any), map[string]any{ + "key": "node.kubernetes.io/not-ready", "operator": "Exists", "effect": "NoExecute", "tolerationSeconds": float64(300)}) + for _, key := range []string{"containers", "initContainers"} { + c := s[key].([]any)[0].(map[string]any) + c["imagePullPolicy"] = "IfNotPresent" + c["terminationMessagePath"] = "/dev/termination-log" + c["terminationMessagePolicy"] = "File" + } + s["containers"].([]any)[0].(map[string]any)["readinessProbe"].(map[string]any)["timeoutSeconds"] = float64(1) + pod["status"] = map[string]any{ + "containerStatuses": []any{map[string]any{"name": "runner", "ready": true, "restartCount": float64(0), "imageID": runtimeImage}}, + "initContainerStatuses": []any{map[string]any{"name": "init", "restartCount": float64(0), "imageID": runtimeImage, "state": map[string]any{"terminated": map[string]any{"exitCode": float64(0)}}}}, + } + return pod + } + t.Run("known-api-defaults-and-verified-runtime", func(t *testing.T) { + if err := verifyPod(desiredJSON, encode(actual()), runtimeImage); err != nil { + t.Fatal(err) + } + }) + t.Run("containerd-reports-requested-signed-index", func(t *testing.T) { + pod := actual() + for _, field := range []string{"containerStatuses", "initContainerStatuses"} { + pod["status"].(map[string]any)[field].([]any)[0].(map[string]any)["imageID"] = image + } + if err := verifyPod(desiredJSON, encode(pod), runtimeImage); err != nil { + t.Fatal(err) + } + }) + mutations := map[string]func(map[string]any){ + "host-network": func(s map[string]any) { s["hostNetwork"] = true }, + "host-pid": func(s map[string]any) { s["hostPID"] = true }, + "runtime-class": func(s map[string]any) { s["runtimeClassName"] = "unexpected" }, + "service-account": func(s map[string]any) { s["serviceAccountName"] = "privileged" }, + "token-mount": func(s map[string]any) { s["automountServiceAccountToken"] = true }, + "pod-security": func(s map[string]any) { s["securityContext"].(map[string]any)["runAsUser"] = float64(0) }, + "node-selector": func(s map[string]any) { s["nodeSelector"] = map[string]any{"other": "enabled"} }, + "extra-toleration": func(s map[string]any) { + s["tolerations"] = append(s["tolerations"].([]any), map[string]any{"operator": "Exists"}) + }, + "host-path": func(s map[string]any) { + s["volumes"] = append(s["volumes"].([]any), map[string]any{"name": "host", "hostPath": map[string]any{"path": "/"}}) + }, + "sidecar": func(s map[string]any) { + s["containers"] = append(s["containers"].([]any), map[string]any{"name": "sidecar", "image": "other"}) + }, + "init-image": func(s map[string]any) { s["initContainers"].([]any)[0].(map[string]any)["image"] = "other" }, + "init-security": func(s map[string]any) { + s["initContainers"].([]any)[0].(map[string]any)["securityContext"] = map[string]any{"privileged": true} + }, + "runner-image": func(s map[string]any) { s["containers"].([]any)[0].(map[string]any)["image"] = "other" }, + "runner-env": func(s map[string]any) { + s["containers"].([]any)[0].(map[string]any)["env"] = []any{map[string]any{"name": "INJECTED", "value": "unsafe"}} + }, + "runner-envfrom": func(s map[string]any) { + s["containers"].([]any)[0].(map[string]any)["envFrom"] = []any{map[string]any{"secretRef": map[string]any{"name": "unsafe"}}} + }, + "runner-mount": func(s map[string]any) { s["containers"].([]any)[0].(map[string]any)["volumeMounts"] = []any{} }, + "runner-resources": func(s map[string]any) { + s["containers"].([]any)[0].(map[string]any)["resources"].(map[string]any)["limits"] = map[string]any{"memory": "1Gi"} + }, + "runner-lifecycle": func(s map[string]any) { + s["containers"].([]any)[0].(map[string]any)["lifecycle"] = map[string]any{"postStart": map[string]any{"exec": map[string]any{"command": []any{"unsafe"}}}} + }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + pod := actual() + mutate(pod["spec"].(map[string]any)) + if verifyPod(desiredJSON, encode(pod), runtimeImage) == nil { + t.Fatal("accepted changed runtime") + } + }) + } + for _, field := range []string{"containerStatuses", "initContainerStatuses"} { + t.Run(field+"-unverified-image", func(t *testing.T) { + pod := actual() + pod["status"].(map[string]any)[field].([]any)[0].(map[string]any)["imageID"] = + "ghcr.io/devantler-tech/ksail-analysis-runner@sha256:" + strings.Repeat("c", 64) + if verifyPod(desiredJSON, encode(pod), runtimeImage) == nil { + t.Fatal("accepted unrelated runtime digest") + } + }) + } +} From 23dc193174412a083b3145bec5fe002951798091 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 01:07:47 +0200 Subject: [PATCH 09/62] fix(ci): accept variable Hetzner node suffix lengths --- scripts/tests/test-verify-ksail-arc-runtime.sh | 13 +++++++++++++ scripts/verify-ksail-arc-runtime.sh | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh index f2ed78db1d..3ca41c8e38 100644 --- a/scripts/tests/test-verify-ksail-arc-runtime.sh +++ b/scripts/tests/test-verify-ksail-arc-runtime.sh @@ -213,6 +213,19 @@ if (cd "$scratch"; PATH="$scratch/bin:$PATH" GITHUB_ACTIONS=false bash scripts/v >"$scratch/unauthorized-out" 2>"$scratch/unauthorized-error"; then exit 1; fi rg -q 'FAIL at deployment-identity' "$scratch/unauthorized-error" run_case complete-proof pass +# Hetzner formats rand.Int63 with %x, so valid suffixes contain one to sixteen +# hex digits. Exercise the complete lifecycle with the shortest legal name. +cp "$scratch/bin/kubectl" "$scratch/kubectl-original" +for suffix in a 0123456789abcdef0 nothex; do + sed "s/autoscale-ksail-analysis-0123456789abcdef/autoscale-ksail-analysis-$suffix/g" \ + "$scratch/kubectl-original" >"$scratch/bin/kubectl" + if [[ "$suffix" == a ]]; then run_case short-node-suffix pass + else run_case "invalid-node-suffix-$suffix" fail; fi +done +sed 's/autoscale-ksail-analysis-0123456789abcdef/autoscale-other-pool-0123456789abcdef/g' \ + "$scratch/kubectl-original" >"$scratch/bin/kubectl" +run_case wrong-node-pool fail +cp "$scratch/kubectl-original" "$scratch/bin/kubectl" for name in no-registration bad-signature tampered-image wider-ceiling full-quota stale-quota admission-transport mutated-init \ unknown-create replacement-race insufficient-memory unhealthy-target curl-transport allowed-egress \ forwarded-flow wrong-source observer-loss observer-diagnostics failed-node-cleanup; do diff --git a/scripts/verify-ksail-arc-runtime.sh b/scripts/verify-ksail-arc-runtime.sh index 52456953f6..315deea8b4 100644 --- a/scripts/verify-ksail-arc-runtime.sh +++ b/scripts/verify-ksail-arc-runtime.sh @@ -208,7 +208,7 @@ kc get node "$probe_node" -o json >"$scratch/node.json" jq -e --slurpfile before "$scratch/nodes.json" ' .metadata.uid as $uid | ($uid | type == "string" and length > 0) and all($before[0].items[]; .metadata.uid != $uid) and - (.metadata.name | test("^autoscale-ksail-analysis-[0-9a-f]{16}$")) and + (.metadata.name | test("^autoscale-ksail-analysis-[0-9a-f]{1,16}$")) and .metadata.labels["platform.devantler.tech/ksail-analysis"] == "enabled" and .metadata.labels["node.kubernetes.io/instance-type"] == "cx53" and .status.nodeInfo.operatingSystem == "linux" and .status.nodeInfo.architecture == "amd64" and From 7441840fc7e6970d287e0eccf9251ddff258c2a9 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 01:35:22 +0200 Subject: [PATCH 10/62] fix(ci): verify rendered ARC activation before skipping runtime proof --- .github/workflows/ci.yaml | 2 +- scripts/tests/arc-staging/activation_test.go | 233 ++++++++++++++++++ scripts/tests/arc-staging/staging_test.go | 37 +-- .../tests/test-verify-ksail-arc-runtime.sh | 47 +++- scripts/verify-ksail-arc-runtime.sh | 27 +- 5 files changed, 306 insertions(+), 40 deletions(-) create mode 100644 scripts/tests/arc-staging/activation_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a2510b5a08..cad977e723 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -146,7 +146,7 @@ jobs: - name: 🔎 Validate finite runtime-log evidence run: go test ./scripts/analyze-cri-logs - - name: 🛑 Validate inactive ARC staging boundaries + - name: 🛑 Validate bounded ARC activation boundaries run: go test ./scripts/tests/arc-staging - name: 🔬 Validate bounded ARC runtime acceptance diff --git a/scripts/tests/arc-staging/activation_test.go b/scripts/tests/arc-staging/activation_test.go new file mode 100644 index 0000000000..8232d06798 --- /dev/null +++ b/scripts/tests/arc-staging/activation_test.go @@ -0,0 +1,233 @@ +package arcstaging_test + +import ( + "fmt" + "io/fs" + "os" + "regexp" + "strings" + "testing" + "testing/fstest" + + "gopkg.in/yaml.v3" +) + +const controllerAggregate = "k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" +const runnerAggregate = "k8s/providers/hetzner/infrastructure/kustomization.yaml" +const controllerReference = "../../../../bases/infrastructure/controllers/actions-runner-controller/" +const runnerReference = "../../../bases/infrastructure/ksail-analysis-runners/" +const controllerRelease = "k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml" +const runnerRelease = "k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml" +const runtimeAction = ".github/actions/deploy-prod/action.yml" + +func TestDeploymentAggregatesKeepARCInsideActivationEnvelope(t *testing.T) { + if err := validateARCActivation(os.DirFS(repoRoot)); err != nil { + t.Fatal(err) + } +} + +func activationFixture(active bool) fstest.MapFS { + suspended, controller, runner := "true", "[]", "[]" + image := "ghcr.io/actions/actions-runner@sha256:" + strings.Repeat("a", 64) + if active { + suspended = "false" + controller = fmt.Sprintf("[%q]", controllerReference) + runner = fmt.Sprintf("[%q]", runnerReference) + image = analysisRepository + "@sha256:" + strings.Repeat("b", 64) + } + files := fstest.MapFS{} + for name, data := range map[string]string{ + controllerAggregate: "resources: " + controller + "\n", + runnerAggregate: "resources: " + runner + "\n", + controllerRelease: "spec:\n suspend: " + suspended + "\n", + runnerRelease: "spec:\n suspend: " + suspended + "\n values:\n template:\n spec:\n" + + " containers: [{image: " + image + "}]\n initContainers: [{image: " + image + "}]\n", + runtimeAction: "runs:\n steps:\n - name: Verify the deployed KSail ARC canary\n" + + " if: >-\n !cancelled() && steps.wait_flux_revision.outcome == 'success' && " + + "( steps.cluster_update.outcome == 'skipped' || steps.wait_prod_api_stability.outcome == 'success' )\n" + + " shell: bash\n env:\n PLATFORM_MANIFEST_DIGEST: ${{ steps.publish_platform_manifest.outputs.digest }}\n" + + " run: |\n export KUBECONFIG=\"${HOME}/.kube/config\"\n bash scripts/verify-ksail-arc-runtime.sh --if-active\n", + } { + files[name] = &fstest.MapFile{Data: []byte(data)} + } + return files +} + +func TestActivationEnvelopeRejectsUnboundedOrUnverifiedStates(t *testing.T) { + for _, active := range []bool{false, true} { + if err := validateARCActivation(activationFixture(active)); err != nil { + t.Fatalf("valid active=%t: %v", active, err) + } + } + tests := []struct { + name, file, from, to string + }{ + {"controller only", runnerAggregate, runnerReference, "unrelated"}, + {"runner only", controllerAggregate, controllerReference, "unrelated"}, + {"duplicate activation", runnerAggregate, "resources:", "components: [\"" + runnerReference + "\"]\nresources:"}, + {"component activation", runnerAggregate, "resources:", "components:"}, + {"legacy base activation", runnerAggregate, "resources:", "bases:"}, + {"suspended controller", controllerRelease, "suspend: false", "suspend: true"}, + {"suspended runner", runnerRelease, "suspend: false", "suspend: true"}, + {"upstream image active", runnerRelease, analysisRepository, "ghcr.io/actions/actions-runner"}, + {"mutable image", runnerRelease, "@sha256:" + strings.Repeat("b", 64), ":latest"}, + {"zero image digest", runnerRelease, strings.Repeat("b", 64), strings.Repeat("0", 64)}, + {"missing canary", runtimeAction, "Verify the deployed KSail ARC canary", "Unrelated step"}, + {"skipped canary", runtimeAction, "!cancelled() &&", "false &&"}, + {"waived canary", runtimeAction, " shell: bash", " continue-on-error: true\n shell: bash"}, + {"wrong deployment digest", runtimeAction, "steps.publish_platform_manifest.outputs.digest", "github.sha"}, + {"unexpected proof environment", runtimeAction, " env:", " env:\n PATH: /unverified"}, + {"missing suspension declaration", runnerRelease, "suspend: false", "unrelated: false"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + files := activationFixture(true) + files[test.file].Data = []byte(strings.ReplaceAll(string(files[test.file].Data), test.from, test.to)) + if err := validateARCActivation(files); err == nil { + t.Fatal("accepted invalid activation") + } + }) + } + for _, test := range []struct{ name, path string }{ + {"other cluster", "k8s/clusters/local/kustomization.yaml"}, + {"other production layer", "k8s/providers/hetzner/apps/kustomization.yaml"}, + {"alternate yaml filename", "k8s/clusters/local/kustomization.yml"}, + {"alternate canonical filename", "k8s/clusters/local/Kustomization"}, + } { + t.Run(test.name, func(t *testing.T) { + files := activationFixture(true) + files[test.path] = &fstest.MapFile{Data: []byte("resources: [\"" + runnerReference + "\"]\n")} + if err := validateARCActivation(files); err == nil { + t.Fatal("accepted activation from another aggregate") + } + }) + } + files := activationFixture(false) + files[runnerRelease].Data = []byte(strings.ReplaceAll(string(files[runnerRelease].Data), "suspend: true", "suspend: false")) + if err := validateARCActivation(files); err == nil { + t.Fatal("accepted unsuspended inactive runner") + } +} + +func validateARCActivation(files fs.FS) error { + counts := map[string]int{controllerAggregate: 0, runnerAggregate: 0} + allowed := map[string]string{controllerAggregate: controllerReference, runnerAggregate: runnerReference} + err := fs.WalkDir(files, "k8s", func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || (entry.Name() != "kustomization.yaml" && entry.Name() != "kustomization.yml" && entry.Name() != "Kustomization") || + strings.HasPrefix(name, strings.TrimSuffix(controllerRelease, "/helm-release.yaml")+"/") || + strings.HasPrefix(name, strings.TrimSuffix(runnerRelease, "/helm-release.yaml")+"/") { + return nil + } + var aggregate struct { + Resources []string `yaml:"resources"` + Components []string `yaml:"components"` + Bases []string `yaml:"bases"` + } + if err := activationYAML(files, name, &aggregate); err != nil { + return err + } + for _, reference := range append(aggregate.Components, aggregate.Bases...) { + if strings.Contains(reference, "actions-runner-controller") || strings.Contains(reference, "ksail-analysis-runners") { + return fmt.Errorf("ARC activation must be an explicit resource, not a component or legacy base: %s", name) + } + } + for _, reference := range aggregate.Resources { + if !strings.Contains(reference, "actions-runner-controller") && !strings.Contains(reference, "ksail-analysis-runners") { + continue + } + if expected, ok := allowed[name]; !ok || reference != expected { + return fmt.Errorf("ARC reference outside approved aggregate: %s", name) + } + counts[name]++ + } + return nil + }) + if err != nil { + return err + } + active := counts[controllerAggregate] == 1 && counts[runnerAggregate] == 1 + if !active && (counts[controllerAggregate] != 0 || counts[runnerAggregate] != 0) { + return fmt.Errorf("partial or duplicate ARC activation") + } + var image string + for _, name := range []string{controllerRelease, runnerRelease} { + var release struct { + Spec struct { + Suspend *bool `yaml:"suspend"` + Values struct { + Template struct { + Spec struct { + Containers []struct{ Image string } `yaml:"containers"` + InitContainers []struct{ Image string } `yaml:"initContainers"` + } `yaml:"spec"` + } `yaml:"template"` + } `yaml:"values"` + } `yaml:"spec"` + } + if err := activationYAML(files, name, &release); err != nil { + return err + } + if release.Spec.Suspend == nil || *release.Spec.Suspend == active { + return fmt.Errorf("ARC release suspension does not match aggregate activation: %s", name) + } + if name == runnerRelease { + containers := release.Spec.Values.Template.Spec.Containers + init := release.Spec.Values.Template.Spec.InitContainers + if len(containers) != 1 || len(init) != 1 || containers[0].Image != init[0].Image { + return fmt.Errorf("runner and init images must have one identical immutable pin") + } + image = containers[0].Image + } + } + repository := regexp.QuoteMeta(analysisRepository) + if !active { + repository += "|" + regexp.QuoteMeta("ghcr.io/actions/actions-runner") + } + if !regexp.MustCompile("^("+repository+")@sha256:[0-9a-f]{64}$").MatchString(image) || strings.HasSuffix(image, strings.Repeat("0", 64)) { + return fmt.Errorf("ARC image is outside the immutable publication envelope") + } + return validateARCRuntimeHook(files) +} + +func activationYAML(files fs.FS, name string, target any) error { + data, err := fs.ReadFile(files, name) + if err != nil { + return err + } + return yaml.Unmarshal(data, target) +} + +func validateARCRuntimeHook(files fs.FS) error { + var action struct { + Runs struct { + Steps []map[string]any `yaml:"steps"` + } `yaml:"runs"` + } + if err := activationYAML(files, runtimeAction, &action); err != nil { + return err + } + count := 0 + condition := "!cancelled() && steps.wait_flux_revision.outcome == 'success' && ( steps.cluster_update.outcome == 'skipped' || steps.wait_prod_api_stability.outcome == 'success' )" + command := "export KUBECONFIG=\"${HOME}/.kube/config\"\nbash scripts/verify-ksail-arc-runtime.sh --if-active" + for _, step := range action.Runs.Steps { + if step["name"] != "Verify the deployed KSail ARC canary" { + continue + } + count++ + _, waived := step["continue-on-error"] + guard, ok := step["if"].(string) + run, runOK := step["run"].(string) + env, envOK := step["env"].(map[string]any) + if waived || !ok || strings.Join(strings.Fields(guard), " ") != condition || !runOK || strings.TrimSpace(run) != command || + step["shell"] != "bash" || !envOK || len(env) != 1 || env["PLATFORM_MANIFEST_DIGEST"] != "${{ steps.publish_platform_manifest.outputs.digest }}" { + return fmt.Errorf("ARC runtime proof is skipped, waived, or bound to the wrong deployment") + } + } + if count != 1 { + return fmt.Errorf("exactly one protected ARC runtime proof is required") + } + return nil +} diff --git a/scripts/tests/arc-staging/staging_test.go b/scripts/tests/arc-staging/staging_test.go index 19e7161b80..a72e8ca8c6 100644 --- a/scripts/tests/arc-staging/staging_test.go +++ b/scripts/tests/arc-staging/staging_test.go @@ -46,14 +46,13 @@ func equal(t *testing.T, value, want any) { } } -func TestStagedReleasesAreSuspendedAndPinned(t *testing.T) { +func TestReleasesUsePinnedCharts(t *testing.T) { for _, component := range []string{ "k8s/bases/infrastructure/controllers/actions-runner-controller", "k8s/bases/infrastructure/ksail-analysis-runners", } { t.Run(filepath.Base(component), func(t *testing.T) { release := readYAML(t, component+"/helm-release.yaml") - equal(t, field(t, release, "spec", "suspend"), true) equal(t, field(t, release, "spec", "chartRef", "kind"), "OCIRepository") source := readYAML(t, component+"/oci-repository.yaml") equal(t, field(t, source, "spec", "ref", "tag"), "0.15.0") @@ -86,7 +85,8 @@ func TestRepositoryScopedPoolCannotCreateUnboundedOrPrivilegedRunners(t *testing runner := containers[0] equal(t, field(t, runner, "name"), "runner") image, ok := field(t, runner, "image").(string) - if !ok || !strings.HasPrefix(image, "ghcr.io/actions/actions-runner@sha256:") { + if !ok || (!strings.HasPrefix(image, "ghcr.io/actions/actions-runner@sha256:") && + !strings.HasPrefix(image, analysisRepository+"@sha256:")) { t.Fatal("runner image must be digest-pinned") } for _, name := range []string{"privileged", "allowPrivilegeEscalation"} { @@ -188,37 +188,6 @@ func TestStagingGuardRunsUnconditionallyOnPullRequestsAndMergeGroups(t *testing. t.Fatal("missing unconditional staging guard") } -func TestNoDeploymentAggregateActivatesARC(t *testing.T) { - err := filepath.WalkDir(filepath.Join(repoRoot, "k8s"), func(path string, entry os.DirEntry, walkErr error) error { - if walkErr != nil { - return walkErr - } - if entry.IsDir() || entry.Name() != "kustomization.yaml" || strings.Contains(path, "/actions-runner-controller/") || strings.Contains(path, "/ksail-analysis-runners/") { - return nil - } - data, err := os.ReadFile(path) - if err != nil { - return err - } - var document struct { - Resources []string `yaml:"resources"` - Components []string `yaml:"components"` - } - if err := yaml.Unmarshal(data, &document); err != nil { - return err - } - for _, reference := range append(document.Resources, document.Components...) { - if strings.Contains(reference, "actions-runner-controller") || strings.Contains(reference, "ksail-analysis-runners") { - t.Errorf("%s activates ARC through %q", path, reference) - } - } - return nil - }) - if err != nil { - t.Fatal(err) - } -} - func TestDNSAccessIsLimitedToTheDeclaredExternalDependencies(t *testing.T) { for _, path := range []string{ "k8s/bases/infrastructure/controllers/actions-runner-controller/cilium-network-policy.yaml", diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh index 3ca41c8e38..049ac3c485 100644 --- a/scripts/tests/test-verify-ksail-arc-runtime.sh +++ b/scripts/tests/test-verify-ksail-arc-runtime.sh @@ -5,10 +5,14 @@ root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) scratch=$(mktemp -d) trap 'rm -rf "$scratch"' EXIT mkdir -p "$scratch/bin" "$scratch/scripts" "$scratch/k8s/providers/hetzner/infrastructure/controllers" \ - "$scratch/k8s/bases/infrastructure/ksail-analysis-runners" + "$scratch/k8s/bases/infrastructure/ksail-analysis-runners" \ + "$scratch/k8s/bases/infrastructure/controllers/actions-runner-controller" cp "$root/scripts/verify-ksail-arc-runtime.sh" "$scratch/scripts/" cp "$root/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml" \ "$scratch/k8s/bases/infrastructure/ksail-analysis-runners/" +cp "$root/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml" \ + "$scratch/k8s/bases/infrastructure/controllers/actions-runner-controller/" +yq -i '.spec.suspend=false' "$scratch/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml" go build -o "$scratch/verifier" "$root/scripts/verify-ksail-arc-runtime" export ARC_TEST_ROOT="$scratch" printf '{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{},"layers":[]}' >"$scratch/runtime-image" @@ -20,7 +24,7 @@ digest="sha256:$(shasum -a 256 "$scratch/image" | cut -d' ' -f1)" export digest image="ghcr.io/devantler-tech/ksail-analysis-runner@$digest" image="$image" yq -i '.spec.values.template.spec.containers[0].image=strenv(image) | - .spec.values.template.spec.initContainers[0].image=strenv(image)' \ + .spec.values.template.spec.initContainers[0].image=strenv(image) | .spec.suspend=false' \ "$scratch/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml" yq -o=json '.spec.values' "$scratch/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml" \ | jq '{metadata:{generation:1,annotations:{"runner-scale-set-id":"1"}}, @@ -63,7 +67,31 @@ SH cat >"$scratch/bin/kubectl" <<'SH' #!/usr/bin/env bash set -euo pipefail +if [[ "${1:-}" == kustomize ]]; then + [[ "${ARC_TEST_CASE:-}" != source-render-failure ]] || exit 1 + case "$2" in + k8s/providers/hetzner/infrastructure/controllers) + aggregate="$ARC_TEST_ROOT/$2/kustomization.yaml" + release="$ARC_TEST_ROOT/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml" + reference='actions-runner-controller' ;; + k8s/providers/hetzner/infrastructure) + aggregate="$ARC_TEST_ROOT/$2/kustomization.yaml" + release="$ARC_TEST_ROOT/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml" + reference='ksail-analysis-runners' ;; + *) exit 0 ;; + esac + references=$(yq -o=json '(.resources // []) + (.bases // [])' "$aggregate") + if jq -e --arg component "$reference" 'any(.[]; contains($component))' <<<"$references" >/dev/null; then + case "${ARC_TEST_CASE:-}" in + rendered-suspended) yq '.spec.suspend=true' "$release" ;; + rendered-wrong-image) yq '.spec.values.template.spec.containers[0].image="unverified:latest"' "$release" ;; + *) cat "$release" ;; + esac + fi + exit 0 +fi [[ "$1" == --context && "$2" == admin@prod ]] || exit 94 +touch "$ARC_TEST_ROOT/runtime-access" shift 3 # context pair and request-timeout args="$*" case "$args" in @@ -177,7 +205,7 @@ chmod +x "$scratch/bin/"* run_case() { local name=$1 expected=$2 code=0 - rm -f "$scratch/live-pod" "$scratch/deleted" "$scratch/replacement-preserved" + rm -f "$scratch/live-pod" "$scratch/deleted" "$scratch/replacement-preserved" "$scratch/runtime-access" ( cd "$scratch" PATH="$scratch/bin:$PATH" GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform \ @@ -205,6 +233,7 @@ printf 'resources: []\n' >"$scratch/k8s/providers/hetzner/infrastructure/kustomi PATH="$scratch/bin:$PATH" bash scripts/verify-ksail-arc-runtime.sh --if-active ) >"$scratch/inactive" rg -q 'inactive source; no runtime access' "$scratch/inactive" +[[ ! -e "$scratch/runtime-access" ]] printf 'resources: ["../../../../bases/infrastructure/controllers/actions-runner-controller/"]\n' \ >"$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" printf 'resources: ["../../../bases/infrastructure/ksail-analysis-runners/"]\n' \ @@ -213,6 +242,18 @@ if (cd "$scratch"; PATH="$scratch/bin:$PATH" GITHUB_ACTIONS=false bash scripts/v >"$scratch/unauthorized-out" 2>"$scratch/unauthorized-error"; then exit 1; fi rg -q 'FAIL at deployment-identity' "$scratch/unauthorized-error" run_case complete-proof pass +cp "$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" "$scratch/controller-aggregate" +cp "$scratch/k8s/providers/hetzner/infrastructure/kustomization.yaml" "$scratch/runner-aggregate" +sed 's/^resources:/bases:/' "$scratch/controller-aggregate" >"$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" +sed 's/^resources:/bases:/' "$scratch/runner-aggregate" >"$scratch/k8s/providers/hetzner/infrastructure/kustomization.yaml" +run_case rendered-hidden-activation fail +[[ ! -e "$scratch/runtime-access" ]] +cp "$scratch/controller-aggregate" "$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" +cp "$scratch/runner-aggregate" "$scratch/k8s/providers/hetzner/infrastructure/kustomization.yaml" +for name in source-render-failure rendered-suspended rendered-wrong-image; do + run_case "$name" fail + [[ ! -e "$scratch/runtime-access" ]] +done # Hetzner formats rand.Int63 with %x, so valid suffixes contain one to sixteen # hex digits. Exercise the complete lifecycle with the shortest legal name. cp "$scratch/bin/kubectl" "$scratch/kubectl-original" diff --git a/scripts/verify-ksail-arc-runtime.sh b/scripts/verify-ksail-arc-runtime.sh index 315deea8b4..dda45d4cb3 100644 --- a/scripts/verify-ksail-arc-runtime.sh +++ b/scripts/verify-ksail-arc-runtime.sh @@ -9,11 +9,34 @@ readonly controller_overlay=k8s/providers/hetzner/infrastructure/controllers/kus readonly pool_overlay=k8s/providers/hetzner/infrastructure/kustomization.yaml controller_active=$(yq '[.resources[] | select(. == "../../../../bases/infrastructure/controllers/actions-runner-controller/")] | length' "$controller_overlay") pool_active=$(yq '[.resources[] | select(. == "../../../bases/infrastructure/ksail-analysis-runners/")] | length' "$pool_overlay") -if [[ "$controller_active" == 0 && "$pool_active" == 0 ]]; then +# Render every independently reconciled production layer before deciding to +# skip. Legacy bases, aliases and overlay patches cannot activate ARC unseen. +source_scratch=$(mktemp -d) +trap 'rm -rf "$source_scratch"' EXIT +for overlay in k8s/providers/hetzner/apps k8s/providers/hetzner/infrastructure \ + k8s/providers/hetzner/infrastructure/controllers k8s/clusters/prod/bootstrap k8s/clusters/prod; do + timeout 95s kubectl kustomize "$overlay" >>"$source_scratch/rendered.yaml" \ + 2>"$source_scratch/render-error" || fail source-render + printf '\n---\n' >>"$source_scratch/rendered.yaml" +done +yq ea -o=json -I=0 '[select(.kind == "HelmRelease" and + ((.metadata.namespace == "arc-systems" and .metadata.name == "arc-controller") or + (.metadata.namespace == "arc-ksail-analysis" and .metadata.name == "ksail-analysis-runners")))]' \ + "$source_scratch/rendered.yaml" >"$source_scratch/releases.json" +rendered_controller=$(jq '[.[] | select(.metadata.namespace == "arc-systems")] | length' "$source_scratch/releases.json") +rendered_pool=$(jq '[.[] | select(.metadata.namespace == "arc-ksail-analysis")] | length' "$source_scratch/releases.json") +if [[ "$controller_active" == 0 && "$pool_active" == 0 && "$rendered_controller" == 0 && "$rendered_pool" == 0 ]]; then printf 'ARC acceptance: inactive source; no runtime access\n' exit 0 fi -[[ "$controller_active" == 1 && "$pool_active" == 1 ]] || fail partial-activation +[[ "$controller_active" == 1 && "$pool_active" == 1 && "$rendered_controller" == 1 && "$rendered_pool" == 1 ]] || fail partial-activation +source_image=$(yq '.spec.values.template.spec.containers[0].image' k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml) +jq -e --arg image "$source_image" 'all(.[]; .spec.suspend == false) and + all(.[] | select(.metadata.namespace == "arc-ksail-analysis"); + .spec.values.template.spec.containers[0].image == $image and + .spec.values.template.spec.initContainers[0].image == $image)' "$source_scratch/releases.json" >/dev/null || fail rendered-source-state +rm -rf "$source_scratch" +trap - EXIT [[ "${GITHUB_ACTIONS:-}" == true && "${GITHUB_REPOSITORY:-}" == devantler-tech/platform ]] || fail deployment-identity [[ "${GITHUB_RUN_ID:-}" =~ ^[1-9][0-9]{0,19}$ && "${GITHUB_RUN_ATTEMPT:-}" =~ ^[1-9][0-9]{0,2}$ ]] || fail run-identity [[ "${PLATFORM_MANIFEST_DIGEST:-}" =~ ^sha256:[0-9a-f]{64}$ ]] || fail revision From c77175475d4b8087b2c268d094d669972e4e3866 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 03:38:41 +0200 Subject: [PATCH 11/62] fix(ci): await ARC registration at the deployed revision --- .github/workflows/ci.yaml | 3 +- .../tests/test-verify-ksail-arc-runtime.sh | 34 ++++++++++++++++++ scripts/verify-ksail-arc-runtime.sh | 3 ++ scripts/wait-for-ksail-arc-registration.sh | 36 +++++++++++++++++++ 4 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 scripts/wait-for-ksail-arc-registration.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cad977e723..7ed52b079a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -152,7 +152,7 @@ jobs: - name: 🔬 Validate bounded ARC runtime acceptance run: | go test -race ./scripts/verify-ksail-arc-runtime - shellcheck scripts/verify-ksail-arc-runtime.sh scripts/tests/test-verify-ksail-arc-runtime.sh + shellcheck scripts/verify-ksail-arc-runtime.sh scripts/wait-for-ksail-arc-registration.sh scripts/tests/test-verify-ksail-arc-runtime.sh bash scripts/tests/test-verify-ksail-arc-runtime.sh # Shared Wedding backup access is retired on this proof's word, so it must @@ -489,6 +489,7 @@ jobs: - 'scripts/tests/test-setup-supply-chain-tools.sh' - '.github/actions/deploy-prod/action.yml' - 'scripts/verify-ksail-arc-runtime.sh' + - 'scripts/wait-for-ksail-arc-registration.sh' - 'scripts/verify-ksail-arc-runtime/**' - 'scripts/tests/test-verify-ksail-arc-runtime.sh' - '.github/workflows/dr-rebuild.yaml' diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh index 049ac3c485..65c5486cf3 100644 --- a/scripts/tests/test-verify-ksail-arc-runtime.sh +++ b/scripts/tests/test-verify-ksail-arc-runtime.sh @@ -8,6 +8,9 @@ mkdir -p "$scratch/bin" "$scratch/scripts" "$scratch/k8s/providers/hetzner/infra "$scratch/k8s/bases/infrastructure/ksail-analysis-runners" \ "$scratch/k8s/bases/infrastructure/controllers/actions-runner-controller" cp "$root/scripts/verify-ksail-arc-runtime.sh" "$scratch/scripts/" +if [[ -e "$root/scripts/wait-for-ksail-arc-registration.sh" ]]; then + cp "$root/scripts/wait-for-ksail-arc-registration.sh" "$scratch/scripts/" +fi cp "$root/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml" \ "$scratch/k8s/bases/infrastructure/ksail-analysis-runners/" cp "$root/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml" \ @@ -37,9 +40,15 @@ SH cat >"$scratch/bin/timeout" <<'SH' #!/usr/bin/env bash export ARC_TEST_TIMEOUT=$1 +if [[ "$1" == 660s ]]; then touch "$ARC_TEST_ROOT/registration-budget"; fi shift exec "$@" SH +cat >"$scratch/bin/sleep" <<'SH' +#!/usr/bin/env bash +[[ "$1" == 10 ]] || exit 98 +case "$ARC_TEST_CASE" in no-registration|registration-timeout|stale-flux) exit 124 ;; esac +SH cat >"$scratch/bin/date" <<'SH' #!/usr/bin/env bash printf '2026-10-05T22:00:00.000000000Z\n' @@ -99,9 +108,25 @@ case "$args" in printf '%s\n' --cloud-provider=hetzner --max-nodes-total=9 --nodes=0:1:cx53:fsn1:autoscale-ksail-analysis [[ "$ARC_TEST_CASE" != wider-ceiling ]] || printf '%s\n' --max-nodes-total=10 ;; *'get kustomization '*) + count=0 + [[ ! -e "$ARC_TEST_ROOT/flux-reads" ]] || count=$(cat "$ARC_TEST_ROOT/flux-reads") + count=$((count + 1)) + printf '%s\n' "$count" >"$ARC_TEST_ROOT/flux-reads" + if [[ "$ARC_TEST_CASE" == stale-flux || ( "$ARC_TEST_CASE" == delayed-flux && "$count" -le 2 ) ]]; then + printf '{"metadata":{"generation":2},"status":{"observedGeneration":1,"lastAppliedRevision":"latest@old","conditions":[{"type":"Ready","status":"True"}]}}' + exit 0 + fi jq -n --arg revision "latest@$digest" '{metadata:{generation:1},status:{observedGeneration:1, lastAppliedRevision:$revision,conditions:[{type:"Ready",status:"True"}]}}' ;; *'get autoscalingrunnerset.actions.github.com '*) + count=0 + [[ ! -e "$ARC_TEST_ROOT/registration-reads" ]] || count=$(cat "$ARC_TEST_ROOT/registration-reads") + count=$((count + 1)) + printf '%s\n' "$count" >"$ARC_TEST_ROOT/registration-reads" + if [[ "$ARC_TEST_CASE" == registration-timeout || ( "$ARC_TEST_CASE" == delayed-registration && "$count" -eq 1 ) ]]; then + jq '.status.phase="Pending" | .status.observedGeneration=0 | .metadata.annotations["runner-scale-set-id"]="0"' "$ARC_TEST_ROOT/ars" + exit 0 + fi if [[ "$ARC_TEST_CASE" == no-registration ]]; then jq '.metadata.annotations["runner-scale-set-id"]="0"' "$ARC_TEST_ROOT/ars" else cat "$ARC_TEST_ROOT/ars"; fi ;; *'get resourcequotas '*) @@ -206,6 +231,7 @@ chmod +x "$scratch/bin/"* run_case() { local name=$1 expected=$2 code=0 rm -f "$scratch/live-pod" "$scratch/deleted" "$scratch/replacement-preserved" "$scratch/runtime-access" + rm -f "$scratch/flux-reads" "$scratch/registration-reads" "$scratch/registration-budget" ( cd "$scratch" PATH="$scratch/bin:$PATH" GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform \ @@ -242,6 +268,14 @@ if (cd "$scratch"; PATH="$scratch/bin:$PATH" GITHUB_ACTIONS=false bash scripts/v >"$scratch/unauthorized-out" 2>"$scratch/unauthorized-error"; then exit 1; fi rg -q 'FAIL at deployment-identity' "$scratch/unauthorized-error" run_case complete-proof pass +run_case delayed-flux pass +[[ $(cat "$scratch/flux-reads") -gt 2 && -e "$scratch/registration-budget" ]] +run_case delayed-registration pass +[[ $(cat "$scratch/registration-reads") -gt 2 && -e "$scratch/registration-budget" ]] +for name in registration-timeout stale-flux; do + run_case "$name" fail + [[ -e "$scratch/registration-budget" && ! -e "$scratch/live-pod" && ! -e "$scratch/deleted" ]] +done cp "$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" "$scratch/controller-aggregate" cp "$scratch/k8s/providers/hetzner/infrastructure/kustomization.yaml" "$scratch/runner-aggregate" sed 's/^resources:/bases:/' "$scratch/controller-aggregate" >"$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" diff --git a/scripts/verify-ksail-arc-runtime.sh b/scripts/verify-ksail-arc-runtime.sh index dda45d4cb3..76ccece8f0 100644 --- a/scripts/verify-ksail-arc-runtime.sh +++ b/scripts/verify-ksail-arc-runtime.sh @@ -107,6 +107,9 @@ trap 'exit 143' INT TERM stage=deployed-revision quiet bash scripts/wait-for-platform-flux-revision.sh "$PLATFORM_MANIFEST_DIGEST" +# Helm readiness precedes asynchronous ARC registration. Await the full +# revision/generation join before taking the immutable acceptance snapshots. +quiet timeout 660s bash scripts/wait-for-ksail-arc-registration.sh "$PLATFORM_MANIFEST_DIGEST" for layer in infrastructure apps; do kc -n flux-system get kustomization "$layer" -o json >"$scratch/layer.json" jq -e --arg revision "latest@$PLATFORM_MANIFEST_DIGEST" \ diff --git a/scripts/wait-for-ksail-arc-registration.sh b/scripts/wait-for-ksail-arc-registration.sh new file mode 100644 index 0000000000..6f27cd7cdf --- /dev/null +++ b/scripts/wait-for-ksail-arc-registration.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Read-only convergence polling; the caller also imposes a 660-second hard deadline. +set -euo pipefail +[[ "$#" -eq 1 && "$1" =~ ^sha256:[0-9a-f]{64}$ ]] || exit 2 +[[ "${GITHUB_ACTIONS:-}" == true && "${GITHUB_REPOSITORY:-}" == devantler-tech/platform ]] || exit 1 +readonly revision="latest@$1" +readonly deadline=$((SECONDS + 600)) +kc() { timeout 35s kubectl --context admin@prod --request-timeout=30s "$@" 2>/dev/null; } + +while ((SECONDS < deadline)); do + ready=true + for layer in infrastructure apps; do + if ! state=$(kc -n flux-system get kustomization "$layer" -o json) || + ! jq -e --arg revision "$revision" ' + .status.lastAppliedRevision == $revision and + .status.observedGeneration == .metadata.generation and + any(.status.conditions[]; .type == "Ready" and .status == "True") + ' <<<"$state" >/dev/null 2>&1; then + ready=false + fi + done + if ! state=$(kc -n arc-ksail-analysis get autoscalingrunnerset.actions.github.com ksail-code-quality -o json) || + ! jq -e ' + .status.phase == "Running" and .status.observedGeneration == .metadata.generation and + ((.metadata.annotations["runner-scale-set-id"] | tonumber) > 0) + ' <<<"$state" >/dev/null 2>&1; then + ready=false + fi + if "$ready"; then + printf 'ARC layers and registration converged at the deployed revision\n' + exit 0 + fi + sleep 10 +done +printf 'ARC layers or registration did not converge before the deadline\n' >&2 +exit 1 From f92fabf6425d2607f2c5a95b10395c53d7d4d0d4 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 03:45:39 +0200 Subject: [PATCH 12/62] docs(arc): clarify activation and retained-release recovery --- docs/operations/ksail-arc.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/operations/ksail-arc.md b/docs/operations/ksail-arc.md index ccb2ff4dad..a81b870c4d 100644 --- a/docs/operations/ksail-arc.md +++ b/docs/operations/ksail-arc.md @@ -106,11 +106,13 @@ routing managed analysis: runner pods and their registration are removed before declaring that bug fixed. For staged validation, run `go test ./scripts/tests/arc-staging` and build both -component directories directly with `kubectl kustomize`; normal local/prod trees -must continue to contain no ARC resources. The unconditional CI guard runs on -pull requests and merge groups. A deliberate activation revises that guard in -the same reviewed change, alongside its evidence; deleting the guard alone is -not activation proof. +component directories directly with `kubectl kustomize`. Inactive local/prod +trees must contain no ARC resources. The unconditional CI guard runs on pull +requests and merge groups. It accepts production activation only through both +explicit aggregate references, matching unsuspended releases and one verified +immutable KSail image pin. The protected verifier also renders every production +layer before deciding that source is inactive. Neither guard accepts a partial +activation, hidden legacy reference or waived runtime acceptance. ## Rollout and recovery @@ -134,5 +136,13 @@ Verify no listener, runner or GitHub registration remains before deleting approv temporary capacity. Remove only the ARC ExternalSecret and its materialized Secret; retain the shared App credential and its other platform consumers. +Healing to inactive main does not drain or remove installed ARC releases: their +pruning protection retains them. The orphan check must still report those +untracked resources, and the inactive verifier does not establish their runtime +cleanup. A failed or cancelled activation therefore remains on HOLD until a +reviewed protected change declares and drains the retained pool, verifies busy +jobs finish normally, and proves registration and capacity cleanup. Never remove +pruning protection, add an orphan exception or treat source absence as drain proof. + Official references: [ARC deployment and security guidance](https://docs.github.com/en/actions/how-tos/manage-runners/use-actions-runner-controller/deploy-runner-scale-sets) and [App authentication](https://docs.github.com/en/actions/how-tos/manage-runners/use-actions-runner-controller/authenticate-to-the-api). From 9fb955c158b725daa8d8c91d647adcf7c61ac919 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 04:35:49 +0200 Subject: [PATCH 13/62] feat(arc): activate verified KSail analysis runner pool --- docs/operations/ksail-arc.md | 27 ++++++++++++------- .../helm-release.yaml | 6 ++--- .../ksail-analysis-runners/helm-release.yaml | 10 +++---- .../controllers/kustomization.yaml | 2 ++ .../hetzner/infrastructure/kustomization.yaml | 2 ++ .../approved-surface.txt | 8 ++++++ 6 files changed, 37 insertions(+), 18 deletions(-) diff --git a/docs/operations/ksail-arc.md b/docs/operations/ksail-arc.md index a81b870c4d..d4a789ba52 100644 --- a/docs/operations/ksail-arc.md +++ b/docs/operations/ksail-arc.md @@ -1,10 +1,10 @@ # KSail analysis runners -The ARC controller and KSail analysis pool are prepared but inactive. Neither -component is referenced by a deployment aggregate, and both HelmReleases are -suspended. Merging their definitions registers no runner, reads no App credential -and provisions no server. This is the preparation slice of #4462, not its runtime -acceptance or a resolution of KSail #7131. +Production references the ARC controller and KSail analysis pool, with both +HelmReleases unsuspended. The protected deployment must prove registration, +isolation and cleanup at its exact published revision. Managed analysis remains +on its prior configuration until an actual registered KSail job passes. Activation +alone is not calibration or a resolution of KSail #7131. The controller chart and runner-set chart use the same immutable 0.15.0 artifacts. The runner image is digest-pinned. The controller manages runner sets only in the @@ -20,8 +20,14 @@ digest-pinned upstream runner, frozen Ubuntu package snapshot and checksum-check Go and Node archives supply the desktop compiler and headers without job-time root access. The publisher builds and exercises the image on pull requests, then publishes, attests and signs only on a push to KSail main. Both Kyverno and Talos accept -that workflow identity only for the exact analysis image repository. The staged -pool still requires a verified published digest before activation. +that workflow identity only for the exact analysis image repository. The pool pins +`sha256:1ab01644fd6f67e0b1ab0b456e10b78f58be92ac67e5e06d103cd8c12bbb119b`, +published by KSail main revision +`dbefeb5d46e01b2ffb4ac41e8d94f692a0647323` in +[run 37402941837](https://github.com/devantler-tech/ksail/actions/runs/37402941837). +That run passed the restricted compiler smoke, runtime smoke, anonymous pull and +signature gates. Independent anonymous Cosign verification also bound the digest +to that exact main publisher revision. Publication does not prove registration. The image smoke test executes the copied runner's version command and compiles a Go program against GTK and WebKit as UID/GID 1001 with a read-only root filesystem and no capabilities. @@ -105,9 +111,10 @@ routing managed analysis: KSail #7131's five consecutive successful managed runs and verify completed runner pods and their registration are removed before declaring that bug fixed. -For staged validation, run `go test ./scripts/tests/arc-staging` and build both -component directories directly with `kubectl kustomize`. Inactive local/prod -trees must contain no ARC resources. The unconditional CI guard runs on pull +For source validation, run `go test ./scripts/tests/arc-staging` and build both +component directories directly with `kubectl kustomize`. The local tree must +contain no ARC resources; production activation must stay inside the reviewed +envelope. The unconditional CI guard runs on pull requests and merge groups. It accepts production activation only through both explicit aggregate references, matching unsuspended releases and one verified immutable KSail image pin. The protected verifier also renders every production diff --git a/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml b/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml index df7e6f2e55..a1584a6961 100644 --- a/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml +++ b/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml @@ -1,5 +1,5 @@ -# Staging only: neither this component nor the runner pool is referenced by an -# aggregate. Activation is a separate reviewed change; see docs/operations/ksail-arc.md. +# Production analysis controller. The protected deploy proves registration and +# isolation before managed analysis is routed; see docs/operations/ksail-arc.md. apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -10,7 +10,7 @@ metadata: labels: helm.toolkit.fluxcd.io/crds: enabled spec: - suspend: true + suspend: false interval: 10m timeout: 10m chartRef: diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml b/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml index 710ec99d71..27c7c07a9a 100644 --- a/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml +++ b/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml @@ -1,4 +1,4 @@ -# Inactive, uncalibrated analysis-only pool. Do not route a workflow here until +# Active, uncalibrated analysis-only pool. Do not route managed analysis here until # the registration, isolation, toolchain and capacity gates in the runbook pass. apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease @@ -8,14 +8,14 @@ metadata: annotations: kustomize.toolkit.fluxcd.io/prune: disabled spec: - suspend: true + suspend: false interval: 10m timeout: 10m chartRef: kind: OCIRepository name: ksail-analysis-runners values: - # Use the controller's built-in scaler defaults during preparation. + # Use the controller's built-in scaler defaults for the bounded analysis pool. # ARC 0.15's optional tuning object is absent from the public CRD catalog. # null is explicitly supported by the pinned chart's values schema. listenerConfig: null @@ -56,7 +56,7 @@ spec: sizeLimit: 2Gi initContainers: - name: init-runner-home - image: ghcr.io/actions/actions-runner@sha256:e5496277be5d09bc968b3d64911b74e219ac4a3f2edce956a3ecf9271bea1ef4 + image: ghcr.io/devantler-tech/ksail-analysis-runner@sha256:1ab01644fd6f67e0b1ab0b456e10b78f58be92ac67e5e06d103cd8c12bbb119b command: ["/bin/sh", "-ec", "cp -R /home/runner/. /runner-data/"] securityContext: allowPrivilegeEscalation: false @@ -78,7 +78,7 @@ spec: mountPath: /runner-data containers: - name: runner - image: ghcr.io/actions/actions-runner@sha256:e5496277be5d09bc968b3d64911b74e219ac4a3f2edce956a3ecf9271bea1ef4 + image: ghcr.io/devantler-tech/ksail-analysis-runner@sha256:1ab01644fd6f67e0b1ab0b456e10b78f58be92ac67e5e06d103cd8c12bbb119b command: ["/home/runner/run.sh"] securityContext: allowPrivilegeEscalation: false diff --git a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml index f7c4269891..5a290aef4a 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml @@ -8,6 +8,8 @@ resources: # local/CI only): no VirtualMachines/DataVolumes exist on Hetzner and # virt-handler crash-looped (1000+ restarts) with zero VMs. - ../../../../bases/infrastructure/controllers/ + # Prod-only: the repository-scoped KSail analysis controller. + - ../../../../bases/infrastructure/controllers/actions-runner-controller/ # Prod-only: a minimal Alertmanager scoped to Kubescape runtime-detection # alerts. The Headlamp Kubescape plugin's "Runtime Detection > Alerts" tab # reads ONLY from a Prometheus Alertmanager (GET /api/v2/alerts), which the diff --git a/k8s/providers/hetzner/infrastructure/kustomization.yaml b/k8s/providers/hetzner/infrastructure/kustomization.yaml index 9ec781a47c..cfb3af22d4 100644 --- a/k8s/providers/hetzner/infrastructure/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/kustomization.yaml @@ -3,6 +3,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../../../bases/infrastructure/ + # Prod-only: one bounded KSail analysis runner, authenticated by the shared App. + - ../../../bases/infrastructure/ksail-analysis-runners/ # Prod-only audit-log forwarder. It consumes the `coroot-api-key` generated # from the Hetzner Coroot CR patch, so it belongs in this infrastructure # layer alongside that CR rather than in the earlier wait-gated controllers diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index 1acb037a86..955fffd1e2 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -14,6 +14,10 @@ helm.toolkit.fluxcd.io/v2|HelmRelease|actual-budget|actual-budget adc009b64f2a9da1427f0c9b80f6ea2490607a8b55ceb6e139f73de92399d5da +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-ksail-analysis|ksail-analysis-runners 3b54e0f01826bff706779c765f7eb5bace90ee625c8cea8e214d10ddb1c07476 + +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-systems|arc-controller f7bf55370e5a38911521edbafec8af715676f3d87b39ead456c7875988845637 + helm.toolkit.fluxcd.io/v2|HelmRelease|backstage|backstage 5ff19be19e3525e641b0a7a22e2a7047e1fbda16ef9ca7b4fa731ce3a61e686a helm.toolkit.fluxcd.io/v2|HelmRelease|cert-manager|cert-manager 6e91f980ce0c37a8bfa4671319b170384cffd1781e4778d84f57710a97754ca1 @@ -396,6 +400,10 @@ source.toolkit.fluxcd.io/v1|HelmRepository|vertical-pod-autoscaler|vertical-pod- source.toolkit.fluxcd.io/v1|HelmRepository|whoami|whoami 28dc88b5a5e69a086e4f7fb87be34d1b8a0cc20a13b21706aaf534a8e5d80cf8 +source.toolkit.fluxcd.io/v1|OCIRepository|arc-ksail-analysis|ksail-analysis-runners 5f89c75739fa50d998e046bc2a388ccf9c7fb4413377510e43f9f12ae4ff4c70 + +source.toolkit.fluxcd.io/v1|OCIRepository|arc-systems|arc-controller 64fb325741489bbd5ece40cf2cbf6e4aef3e87e89efea97b7b0f2ec498671fa1 + source.toolkit.fluxcd.io/v1|OCIRepository|ascoachingogvaner|ascoachingogvaner e7717c1d618e5d2dd1f3ab32d589ac06205d040e867f33db22ab9651bcb105bc source.toolkit.fluxcd.io/v1|OCIRepository|aws|aws f900dbd1860a7494381bb0fe0bd044ebedeaa1570bc9a230b7879e2b4d752163 From ad55f886e9fa78b4e546f83d2dc697cebb97c14a Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 04:48:16 +0200 Subject: [PATCH 14/62] fix(ci): keep ARC acceptance portable without ripgrep --- scripts/tests/test-verify-ksail-arc-runtime.sh | 6 +++--- scripts/verify-ksail-arc-runtime.sh | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh index 65c5486cf3..729c1acbe1 100644 --- a/scripts/tests/test-verify-ksail-arc-runtime.sh +++ b/scripts/tests/test-verify-ksail-arc-runtime.sh @@ -240,7 +240,7 @@ run_case() { ) >"$scratch/stdout" 2>"$scratch/stderr" || code=$? if [[ "$expected" == pass ]]; then [[ "$code" == 0 && -e "$scratch/deleted" ]] || { printf 'FAIL: %s (exit %s)\n' "$name" "$code" >&2; cat "$scratch/stderr" >&2; exit 1; } - rg -q '^PASS: ARC registration' "$scratch/stdout" || exit 1 + grep -q '^PASS: ARC registration' "$scratch/stdout" || exit 1 else [[ "$code" != 0 ]] || { printf 'FAIL: accepted %s\n' "$name" >&2; exit 1; } case "$name" in @@ -258,7 +258,7 @@ printf 'resources: []\n' >"$scratch/k8s/providers/hetzner/infrastructure/kustomi cd "$scratch" PATH="$scratch/bin:$PATH" bash scripts/verify-ksail-arc-runtime.sh --if-active ) >"$scratch/inactive" -rg -q 'inactive source; no runtime access' "$scratch/inactive" +grep -Fq 'inactive source; no runtime access' "$scratch/inactive" [[ ! -e "$scratch/runtime-access" ]] printf 'resources: ["../../../../bases/infrastructure/controllers/actions-runner-controller/"]\n' \ >"$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" @@ -266,7 +266,7 @@ printf 'resources: ["../../../bases/infrastructure/ksail-analysis-runners/"]\n' >"$scratch/k8s/providers/hetzner/infrastructure/kustomization.yaml" if (cd "$scratch"; PATH="$scratch/bin:$PATH" GITHUB_ACTIONS=false bash scripts/verify-ksail-arc-runtime.sh --if-active) \ >"$scratch/unauthorized-out" 2>"$scratch/unauthorized-error"; then exit 1; fi -rg -q 'FAIL at deployment-identity' "$scratch/unauthorized-error" +grep -Fq 'FAIL at deployment-identity' "$scratch/unauthorized-error" run_case complete-proof pass run_case delayed-flux pass [[ $(cat "$scratch/flux-reads") -gt 2 && -e "$scratch/registration-budget" ]] diff --git a/scripts/verify-ksail-arc-runtime.sh b/scripts/verify-ksail-arc-runtime.sh index 76ccece8f0..f06ee45b1a 100644 --- a/scripts/verify-ksail-arc-runtime.sh +++ b/scripts/verify-ksail-arc-runtime.sh @@ -173,9 +173,9 @@ fi stage=idle-pool-baseline kc -n kube-system get deployment cluster-autoscaler-hetzner-cluster-autoscaler \ -o jsonpath='{range .spec.template.spec.containers[*].args[*]}{.}{"\n"}{end}' >"$scratch/autoscaler-args" -declared_ceiling=$(rg '^--max-nodes-total=' "$scratch/autoscaler-args") -declared_pool=$(rg '^--nodes=.*:autoscale-ksail-analysis$' "$scratch/autoscaler-args") -declared_provider=$(rg '^--cloud-provider=' "$scratch/autoscaler-args") +declared_ceiling=$(grep '^--max-nodes-total=' "$scratch/autoscaler-args") +declared_pool=$(grep '^--nodes=.*:autoscale-ksail-analysis$' "$scratch/autoscaler-args") +declared_provider=$(grep '^--cloud-provider=' "$scratch/autoscaler-args") [[ "$declared_ceiling" == --max-nodes-total=9 && \ "$declared_pool" == --nodes=0:1:cx53:fsn1:autoscale-ksail-analysis && \ "$declared_provider" == --cloud-provider=hetzner ]] || fail autoscaler-boundary @@ -211,11 +211,11 @@ for negative in privileged hostpath; do if timeout 35s kubectl --context "$context" --request-timeout=30s create --dry-run=server \ -f "$scratch/negative.json" >"$scratch/denied-out" 2>"$scratch/denied-error"; then fail negative-admission; fi # An API/transport failure does not prove admission interception. - rg -q 'violates PodSecurity.*restricted|admission webhook.*denied the request' "$scratch/denied-error" || fail unproven-admission-denial + grep -Eq 'violates PodSecurity.*restricted|admission webhook.*denied the request' "$scratch/denied-error" || fail unproven-admission-denial if [[ "$negative" == privileged ]]; then - rg -qi 'privileged' "$scratch/denied-error" || fail unrelated-admission-denial + grep -qi 'privileged' "$scratch/denied-error" || fail unrelated-admission-denial else - rg -qi 'hostPath|host.path' "$scratch/denied-error" || fail unrelated-admission-denial + grep -Eqi 'hostPath|host.path' "$scratch/denied-error" || fail unrelated-admission-denial fi done created=true From 12b047291e69828ced5e054b604d37904af8a03c Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 04:54:28 +0200 Subject: [PATCH 15/62] fix(ci): track ARC acceptance entry points as executable --- scripts/tests/test-verify-ksail-arc-runtime.sh | 0 scripts/verify-ksail-arc-runtime.sh | 0 scripts/wait-for-ksail-arc-registration.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/tests/test-verify-ksail-arc-runtime.sh mode change 100644 => 100755 scripts/verify-ksail-arc-runtime.sh mode change 100644 => 100755 scripts/wait-for-ksail-arc-registration.sh diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh old mode 100644 new mode 100755 diff --git a/scripts/verify-ksail-arc-runtime.sh b/scripts/verify-ksail-arc-runtime.sh old mode 100644 new mode 100755 diff --git a/scripts/wait-for-ksail-arc-registration.sh b/scripts/wait-for-ksail-arc-registration.sh old mode 100644 new mode 100755 From f165931a76cff6a4ad67c0764cd29600a28c8eff Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 05:31:45 +0200 Subject: [PATCH 16/62] feat(ci): retain whole-job ARC memory measurements --- .github/workflows/ci.yaml | 2 +- docs/operations/ksail-arc.md | 17 ++- .../ksail-analysis-runners/helm-release.yaml | 13 ++ .../ksail-analysis-runners/job-metrics.sh | 42 +++++++ .../ksail-analysis-runners/kustomization.yaml | 11 ++ .../ksail-analysis-runners/kustomizeconfig | 8 ++ .../flux-notifications/alert.yaml | 12 ++ scripts/tests/arc-staging/job_metrics_test.go | 111 ++++++++++++++++++ scripts/tests/arc-staging/runtime_test.go | 9 +- .../tests/test-verify-ksail-arc-runtime.sh | 34 +++++- .../approved-surface.txt | 2 +- scripts/verify-ksail-arc-runtime.sh | 22 +++- 12 files changed, 270 insertions(+), 13 deletions(-) create mode 100755 k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh create mode 100644 k8s/bases/infrastructure/ksail-analysis-runners/kustomizeconfig create mode 100644 scripts/tests/arc-staging/job_metrics_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0c0b8e63c0..f3a7872528 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -152,7 +152,7 @@ jobs: - name: 🔬 Validate bounded ARC runtime acceptance run: | go test -race ./scripts/verify-ksail-arc-runtime - shellcheck scripts/verify-ksail-arc-runtime.sh scripts/wait-for-ksail-arc-registration.sh scripts/tests/test-verify-ksail-arc-runtime.sh + shellcheck scripts/verify-ksail-arc-runtime.sh scripts/wait-for-ksail-arc-registration.sh scripts/tests/test-verify-ksail-arc-runtime.sh k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh bash scripts/tests/test-verify-ksail-arc-runtime.sh # Shared Wedding backup access is retired on this proof's word, so it must diff --git a/docs/operations/ksail-arc.md b/docs/operations/ksail-arc.md index d4a789ba52..2cefa98272 100644 --- a/docs/operations/ksail-arc.md +++ b/docs/operations/ksail-arc.md @@ -69,7 +69,16 @@ routing managed analysis: combined limits are 1Gi/1.25 CPU. These fit the current default namespace quota, but are **not a calibrated memory budget**. The observed roughly 9.7GB native extractor peak is not aggregate pod memory or - proof of OOM. Measure total pod/cgroup peak and extraction duration; revise + proof of OOM. The immutable, hash-named metrics ConfigMap is mounted read-only + outside the runner application directory. Its completion hook reads only the + container's kernel cgroup memory limit, lifetime peak and OOM counters, including + the runner and its descendant processes. It requires the declared 14Gi limit, + bounded valid measurements and zero OOM events; an unknown measurement fails. + Three bounded reads complete within 15 seconds. No credentials, environment + values or JIT configuration are read or printed. The protected canary verifies + the exact immutable hook bytes and executes it. This is telemetry from the job + account, not an independent security boundary or a substitute for job success. + Measure actual managed-job peak and extraction duration; revise sizing and any narrowly justified quota change before admitting real traffic. 4. Render both pinned charts, including their CRDs and namespace-scoped RBAC. Confirm the App Secret is absent from runner volumes, the no-permission runner @@ -107,9 +116,13 @@ routing managed analysis: and read configuration back. Preserve the other setup fields. Check the required tools, GitHub proxy, network allowlist and non-root installation path; do not enable sudo or privilege escalation to make setup pass. - Preserve all root, nested-module and desktop extraction/source guards. Meet + Require the actual preflight's automatic completion hook to emit its bounded + numeric record under GitHub's Complete runner step. The smoke-step peak is + separate from that whole-job measurement. Preserve all root, nested-module and desktop extraction/source guards. Meet KSail #7131's five consecutive successful managed runs and verify completed runner pods and their registration are removed before declaring that bug fixed. + Each managed success must retain its complete-step cgroup measurement; absent + or malformed output remains HOLD and cannot count as a calibration success. For source validation, run `go test ./scripts/tests/arc-staging` and build both component directories directly with `kubectl kustomize`. The local tree must diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml b/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml index 27c7c07a9a..ec8125c127 100644 --- a/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml +++ b/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml @@ -54,6 +54,13 @@ spec: - name: runner-tmp emptyDir: sizeLimit: 2Gi + - name: runner-metrics + configMap: + name: ksail-arc-job-metrics + defaultMode: 365 # 0555: executable, read-only job hook. + items: + - key: job-metrics.sh + path: job-metrics.sh initContainers: - name: init-runner-home image: ghcr.io/devantler-tech/ksail-analysis-runner@sha256:1ab01644fd6f67e0b1ab0b456e10b78f58be92ac67e5e06d103cd8c12bbb119b @@ -80,6 +87,9 @@ spec: - name: runner image: ghcr.io/devantler-tech/ksail-analysis-runner@sha256:1ab01644fd6f67e0b1ab0b456e10b78f58be92ac67e5e06d103cd8c12bbb119b command: ["/home/runner/run.sh"] + env: + - name: ACTIONS_RUNNER_HOOK_JOB_COMPLETED + value: /etc/ksail-arc-metrics/job-metrics.sh securityContext: allowPrivilegeEscalation: false privileged: false @@ -91,6 +101,9 @@ spec: mountPath: /home/runner - name: runner-tmp mountPath: /tmp + - name: runner-metrics + mountPath: /etc/ksail-arc-metrics + readOnly: true resources: requests: cpu: "3" diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh b/k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh new file mode 100755 index 0000000000..3d546026d7 --- /dev/null +++ b/k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +arc_metrics_fail() { + printf '::error::KSail ARC whole-job cgroup measurement failed: %s\n' "$1" >&2 + return 1 +} + +# The production entrypoint uses only the container's kernel cgroup. A supplied +# fixture directory is used when this function is sourced by offline tests. +arc_job_metrics() { + local root=$1 limit peak events key value extra + local low='' high='' max='' oom='' oom_kill='' oom_group_kill='' + limit=$(timeout 5s cat "$root/memory.max" 2>/dev/null) || { arc_metrics_fail read-limit; return 1; } + peak=$(timeout 5s cat "$root/memory.peak" 2>/dev/null) || { arc_metrics_fail read-peak; return 1; } + events=$(timeout 5s cat "$root/memory.events" 2>/dev/null) || { arc_metrics_fail read-events; return 1; } + [[ "$limit" == 15032385536 ]] || { arc_metrics_fail limit; return 1; } + [[ "$peak" =~ ^(0|[1-9][0-9]{0,17})$ && "$peak" -gt 0 && "$peak" -le "$limit" ]] || { + arc_metrics_fail peak; return 1; + } + while read -r key value extra; do + [[ -z "$extra" && "$value" =~ ^(0|[1-9][0-9]{0,17})$ ]] || { arc_metrics_fail event-format; return 1; } + case "$key" in + low|high|max|oom|oom_kill|oom_group_kill) + [[ -z "${!key}" ]] || { arc_metrics_fail duplicate-event; return 1; } + printf -v "$key" '%s' "$value" ;; + *) arc_metrics_fail unknown-event; return 1 ;; + esac + done <<<"$events" + [[ -n "$low" && -n "$high" && -n "$max" && -n "$oom" && -n "$oom_kill" && -n "$oom_group_kill" ]] || { + arc_metrics_fail missing-event; return 1; + } + [[ "$oom" == 0 && "$oom_kill" == 0 && "$oom_group_kill" == 0 ]] || { arc_metrics_fail oom; return 1; } + printf 'KSail ARC whole-job cgroup: {"schemaVersion":1,"memoryMaxBytes":%s,"memoryPeakBytes":%s,"limitEvents":%s,"oomEvents":%s,"oomKills":%s,"oomGroupKills":%s}\n' \ + "$limit" "$peak" "$max" "$oom" "$oom_kill" "$oom_group_kill" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + [[ $# == 0 ]] || { arc_metrics_fail arguments; exit 1; } + arc_job_metrics /sys/fs/cgroup +fi + diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/kustomization.yaml b/k8s/bases/infrastructure/ksail-analysis-runners/kustomization.yaml index 2edc4aea69..24837960b8 100644 --- a/k8s/bases/infrastructure/ksail-analysis-runners/kustomization.yaml +++ b/k8s/bases/infrastructure/ksail-analysis-runners/kustomization.yaml @@ -1,5 +1,16 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +configurations: + - kustomizeconfig +configMapGenerator: + - name: ksail-arc-job-metrics + namespace: arc-ksail-analysis + files: + - job-metrics.sh + options: + immutable: true + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled resources: - oci-repository.yaml - helm-release.yaml diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/kustomizeconfig b/k8s/bases/infrastructure/ksail-analysis-runners/kustomizeconfig new file mode 100644 index 0000000000..1375427aaa --- /dev/null +++ b/k8s/bases/infrastructure/ksail-analysis-runners/kustomizeconfig @@ -0,0 +1,8 @@ +nameReference: + - kind: ConfigMap + version: v1 + fieldSpecs: + - kind: HelmRelease + group: helm.toolkit.fluxcd.io + version: v2 + path: spec/values/template/spec/volumes/configMap/name diff --git a/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml b/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml index dc20899363..d6051ed9d0 100644 --- a/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml +++ b/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml @@ -87,6 +87,12 @@ spec: - kind: HelmRelease name: "*" namespace: actual-budget + - kind: HelmRelease + name: "*" + namespace: arc-ksail-analysis + - kind: HelmRelease + name: "*" + namespace: arc-systems - kind: HelmRelease name: "*" namespace: backstage @@ -177,5 +183,11 @@ spec: - kind: HelmRelease name: "*" namespace: whoami + - kind: OCIRepository + name: "*" + namespace: arc-ksail-analysis + - kind: OCIRepository + name: "*" + namespace: arc-systems eventMetadata: summary: "Flux reconciliation error — prod platform" diff --git a/scripts/tests/arc-staging/job_metrics_test.go b/scripts/tests/arc-staging/job_metrics_test.go new file mode 100644 index 0000000000..1a016e68f4 --- /dev/null +++ b/scripts/tests/arc-staging/job_metrics_test.go @@ -0,0 +1,111 @@ +package arcstaging_test + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +const metricsComponent = "k8s/bases/infrastructure/ksail-analysis-runners" +const metricsScript = metricsComponent + "/job-metrics.sh" + +func TestRunnerRetainsWholeJobCgroupMeasurement(t *testing.T) { + component := readYAML(t, metricsComponent+"/kustomization.yaml") + generator := field(t, component, "configMapGenerator").([]any) + equal(t, len(generator), 1) + equal(t, field(t, generator[0], "namespace"), "arc-ksail-analysis") + equal(t, field(t, generator[0], "options", "immutable"), true) + equal(t, field(t, generator[0], "options", "annotations", "kustomize.toolkit.fluxcd.io/substitute"), "disabled") + release := readYAML(t, metricsComponent+"/helm-release.yaml") + runner := field(t, release, "spec", "values", "template", "spec", "containers").([]any)[0] + env := field(t, runner, "env").([]any) + equal(t, len(env), 1) + equal(t, field(t, env[0], "name"), "ACTIONS_RUNNER_HOOK_JOB_COMPLETED") + equal(t, field(t, env[0], "value"), "/etc/ksail-arc-metrics/job-metrics.sh") +} + +func TestJobMetricsEntrypointRejectsAnAlternateCgroup(t *testing.T) { + script := filepath.Join(repoRoot, metricsScript) + output, err := exec.Command("bash", script, t.TempDir()).CombinedOutput() + if err == nil || strings.Contains(string(output), "KSail ARC whole-job cgroup:") { + t.Fatalf("accepted a substituted production cgroup: %s", output) + } +} + +func TestWholeJobCgroupMeasurement(t *testing.T) { + for _, tc := range []struct { + name, limit, peak, events string + ok bool + }{ + {"complete", "15032385536", "9876543210", "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\noom_group_kill 0\n", true}, + {"missing limit", "", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"unbounded", "max", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"wrong cgroup", "17179869184", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"missing peak", "15032385536", "", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"zero peak", "15032385536", "0", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"exceeded", "15032385536", "15032385537", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"malformed peak", "15032385536", "1\n2", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"noncanonical peak", "15032385536", "001", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"overflow", "15032385536", "99999999999999999999999", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"missing events", "15032385536", "10", "", false}, + {"missing OOM", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom_kill 0\noom_group_kill 0\n", false}, + {"duplicate event", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"unknown event", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsecret 0\n", false}, + {"malformed event", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom nope\noom_kill 0\noom_group_kill 0\n", false}, + {"extra event data", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom 0 extra\noom_kill 0\noom_group_kill 0\n", false}, + {"OOM", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom 1\noom_kill 0\noom_group_kill 0\n", false}, + {"OOM kill", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 1\noom_group_kill 0\n", false}, + {"OOM group kill", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 1\n", false}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + for name, value := range map[string]string{"memory.max": tc.limit, "memory.peak": tc.peak, "memory.events": tc.events} { + if value != "" { + if err := os.WriteFile(filepath.Join(root, name), []byte(strings.TrimSuffix(value, "\n")+"\n"), 0600); err != nil { + t.Fatal(err) + } + } + } + script, err := filepath.Abs(filepath.Join(repoRoot, metricsScript)) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("bash", "-c", `source "$1"; arc_job_metrics "$2"`, "metrics-test", script, root) + cmd.Env = append(os.Environ(), "ACTIONS_RUNNER_INPUT_JITCONFIG=must-not-be-read", "SECRET_CANARY=must-not-be-read") + output, err := cmd.CombinedOutput() + if (err == nil) != tc.ok { + t.Fatalf("success=%t, want %t: %s", err == nil, tc.ok, output) + } + if strings.Contains(string(output), "must-not-be-read") { + t.Fatal("measurement exposed environment data") + } + if !tc.ok { + if strings.Contains(string(output), "KSail ARC whole-job cgroup:") { + t.Fatal("failed measurement emitted a success receipt") + } + return + } + var got struct { + SchemaVersion int `json:"schemaVersion"` + MemoryMaxBytes uint64 `json:"memoryMaxBytes"` + MemoryPeakBytes uint64 `json:"memoryPeakBytes"` + LimitEvents uint64 `json:"limitEvents"` + OOMEvents uint64 `json:"oomEvents"` + OOMKills uint64 `json:"oomKills"` + OOMGroupKills uint64 `json:"oomGroupKills"` + } + decoder := json.NewDecoder(strings.NewReader(strings.TrimPrefix(string(output), "KSail ARC whole-job cgroup: "))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&got); err != nil { + t.Fatal(err) + } + if got.SchemaVersion != 1 || got.MemoryMaxBytes != 15032385536 || got.MemoryPeakBytes != 9876543210 || + got.LimitEvents != 3 || got.OOMEvents != 0 || got.OOMKills != 0 || got.OOMGroupKills != 0 { + t.Fatalf("wrong measurement: %+v", got) + } + }) + } +} diff --git a/scripts/tests/arc-staging/runtime_test.go b/scripts/tests/arc-staging/runtime_test.go index 0a3ab597d2..62b6900519 100644 --- a/scripts/tests/arc-staging/runtime_test.go +++ b/scripts/tests/arc-staging/runtime_test.go @@ -47,18 +47,23 @@ func assertRunnerStorage(t *testing.T, spec any) { wantVolumes := []any{ map[string]any{"name": "runner-home", "emptyDir": map[string]any{"sizeLimit": "40Gi"}}, map[string]any{"name": "runner-tmp", "emptyDir": map[string]any{"sizeLimit": "2Gi"}}, + map[string]any{"name": "runner-metrics", "configMap": map[string]any{ + "name": "ksail-arc-job-metrics", "defaultMode": 365, + "items": []any{map[string]any{"key": "job-metrics.sh", "path": "job-metrics.sh"}}, + }}, } if !reflect.DeepEqual(field(t, spec, "volumes"), wantVolumes) { - t.Fatal("only two explicitly bounded disk emptyDirs are permitted") + t.Fatal("only two bounded disk emptyDirs and the immutable metrics hook are permitted") } runner := field(t, spec, "containers").([]any)[0] equal(t, field(t, runner, "securityContext", "readOnlyRootFilesystem"), true) wantMounts := []any{ map[string]any{"name": "runner-home", "mountPath": "/home/runner"}, map[string]any{"name": "runner-tmp", "mountPath": "/tmp"}, + map[string]any{"name": "runner-metrics", "mountPath": "/etc/ksail-arc-metrics", "readOnly": true}, } if !reflect.DeepEqual(field(t, runner, "volumeMounts"), wantMounts) { - t.Fatal("runner may mount only its disposable home and temporary storage") + t.Fatal("runner may mount only disposable storage and its read-only metrics hook") } initializers := field(t, spec, "initContainers").([]any) equal(t, len(initializers), 1) diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh index 729c1acbe1..33fd6ea0ac 100755 --- a/scripts/tests/test-verify-ksail-arc-runtime.sh +++ b/scripts/tests/test-verify-ksail-arc-runtime.sh @@ -13,6 +13,8 @@ if [[ -e "$root/scripts/wait-for-ksail-arc-registration.sh" ]]; then fi cp "$root/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml" \ "$scratch/k8s/bases/infrastructure/ksail-analysis-runners/" +cp "$root/k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh" \ + "$scratch/k8s/bases/infrastructure/ksail-analysis-runners/" cp "$root/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml" \ "$scratch/k8s/bases/infrastructure/controllers/actions-runner-controller/" yq -i '.spec.suspend=false' "$scratch/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml" @@ -32,7 +34,8 @@ image="$image" yq -i '.spec.values.template.spec.containers[0].image=strenv(imag yq -o=json '.spec.values' "$scratch/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml" \ | jq '{metadata:{generation:1,annotations:{"runner-scale-set-id":"1"}}, status:{phase:"Running",observedGeneration:1},spec:.} | - .spec.template.spec.serviceAccountName="ksail-code-quality-gha-rs-no-permission"' >"$scratch/ars" + .spec.template.spec.serviceAccountName="ksail-code-quality-gha-rs-no-permission" | + .spec.template.spec.volumes[2].configMap.name="ksail-arc-job-metrics-abc123xyz4"' >"$scratch/ars" cat >"$scratch/scripts/wait-for-platform-flux-revision.sh" <<'SH' #!/usr/bin/env bash [[ "$1" == "$digest" ]] @@ -127,8 +130,17 @@ case "$args" in jq '.status.phase="Pending" | .status.observedGeneration=0 | .metadata.annotations["runner-scale-set-id"]="0"' "$ARC_TEST_ROOT/ars" exit 0 fi - if [[ "$ARC_TEST_CASE" == no-registration ]]; then jq '.metadata.annotations["runner-scale-set-id"]="0"' "$ARC_TEST_ROOT/ars" - else cat "$ARC_TEST_ROOT/ars"; fi ;; + case "$ARC_TEST_CASE" in + no-registration) jq '.metadata.annotations["runner-scale-set-id"]="0"' "$ARC_TEST_ROOT/ars" ;; + unexpected-hook) jq '.spec.template.spec.containers[0].env[0].value="/unverified"' "$ARC_TEST_ROOT/ars" ;; + writable-metrics) jq '.spec.template.spec.containers[0].volumeMounts[2].readOnly=false' "$ARC_TEST_ROOT/ars" ;; + wrong-metrics-configmap) jq '.spec.template.spec.volumes[2].configMap.name="unverified"' "$ARC_TEST_ROOT/ars" ;; + *) cat "$ARC_TEST_ROOT/ars" ;; + esac ;; + *'get configmap ksail-arc-job-metrics-'*) + jq -cn --rawfile script "$ARC_TEST_ROOT/k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh" \ + --arg scenario "$ARC_TEST_CASE" '{metadata:{uid:"metrics-uid",annotations:{"kustomize.toolkit.fluxcd.io/substitute":"disabled"}},immutable:($scenario!="mutable-metrics"), + data:{"job-metrics.sh":(if $scenario=="tampered-metrics" then "unverified" else $script end)}}' ;; *'get resourcequotas '*) if [[ "$ARC_TEST_CASE" == full-quota ]]; then printf '{"items":[{"spec":{"hard":{"pods":"1"}},"status":{"hard":{"pods":"1"},"used":{"pods":"1"}}}]}' @@ -204,6 +216,16 @@ case "$args" in if [[ "$ARC_TEST_CASE" == allowed-egress ]]; then exit 0; fi exit 28 ;; *'exec arc-proof-'*'-- test -f /tmp/arc-proof-ready') ;; + *'exec arc-proof-'*'-- /etc/ksail-arc-metrics/job-metrics.sh') + mkdir -p "$ARC_TEST_ROOT/cgroup" + printf '15032385536\n' >"$ARC_TEST_ROOT/cgroup/memory.max" + if [[ "$ARC_TEST_CASE" == failed-metrics-hook ]]; then + printf '0\n' >"$ARC_TEST_ROOT/cgroup/memory.peak" + else printf '9876543210\n' >"$ARC_TEST_ROOT/cgroup/memory.peak"; fi + printf 'low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n' >"$ARC_TEST_ROOT/cgroup/memory.events" + source "$ARC_TEST_ROOT/k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh" + arc_job_metrics "$ARC_TEST_ROOT/cgroup" + touch "$ARC_TEST_ROOT/metrics-executed" ;; *'exec observer -c cilium-agent -- hubble observe '*) destination=192.0.2.12 port=8181 target_ns=kube-system target_pod=dns [[ "$args" != *'--to-ip 192.0.2.13'* ]] || { destination=192.0.2.13; port=6443; target_ns=''; target_pod=''; } @@ -232,6 +254,7 @@ run_case() { local name=$1 expected=$2 code=0 rm -f "$scratch/live-pod" "$scratch/deleted" "$scratch/replacement-preserved" "$scratch/runtime-access" rm -f "$scratch/flux-reads" "$scratch/registration-reads" "$scratch/registration-budget" + rm -f "$scratch/metrics-executed" ( cd "$scratch" PATH="$scratch/bin:$PATH" GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform \ @@ -239,7 +262,7 @@ run_case() { bash scripts/verify-ksail-arc-runtime.sh --if-active ) >"$scratch/stdout" 2>"$scratch/stderr" || code=$? if [[ "$expected" == pass ]]; then - [[ "$code" == 0 && -e "$scratch/deleted" ]] || { printf 'FAIL: %s (exit %s)\n' "$name" "$code" >&2; cat "$scratch/stderr" >&2; exit 1; } + [[ "$code" == 0 && -e "$scratch/deleted" && -e "$scratch/metrics-executed" ]] || { printf 'FAIL: %s (exit %s)\n' "$name" "$code" >&2; cat "$scratch/stderr" >&2; exit 1; } grep -q '^PASS: ARC registration' "$scratch/stdout" || exit 1 else [[ "$code" != 0 ]] || { printf 'FAIL: accepted %s\n' "$name" >&2; exit 1; } @@ -301,7 +324,8 @@ sed 's/autoscale-ksail-analysis-0123456789abcdef/autoscale-other-pool-0123456789 "$scratch/kubectl-original" >"$scratch/bin/kubectl" run_case wrong-node-pool fail cp "$scratch/kubectl-original" "$scratch/bin/kubectl" -for name in no-registration bad-signature tampered-image wider-ceiling full-quota stale-quota admission-transport mutated-init \ +for name in no-registration unexpected-hook writable-metrics wrong-metrics-configmap mutable-metrics tampered-metrics \ + failed-metrics-hook bad-signature tampered-image wider-ceiling full-quota stale-quota admission-transport mutated-init \ unknown-create replacement-race insufficient-memory unhealthy-target curl-transport allowed-egress \ forwarded-flow wrong-source observer-loss observer-diagnostics failed-node-cleanup; do run_case "$name" fail diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index 955fffd1e2..4d66b3e388 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -14,7 +14,7 @@ helm.toolkit.fluxcd.io/v2|HelmRelease|actual-budget|actual-budget adc009b64f2a9da1427f0c9b80f6ea2490607a8b55ceb6e139f73de92399d5da -helm.toolkit.fluxcd.io/v2|HelmRelease|arc-ksail-analysis|ksail-analysis-runners 3b54e0f01826bff706779c765f7eb5bace90ee625c8cea8e214d10ddb1c07476 +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-ksail-analysis|ksail-analysis-runners 15484b7d4dfde81d022fdc9fe595d02b4f3b415a0f60ddbe0c3617f3eac6e101 helm.toolkit.fluxcd.io/v2|HelmRelease|arc-systems|arc-controller f7bf55370e5a38911521edbafec8af715676f3d87b39ead456c7875988845637 diff --git a/scripts/verify-ksail-arc-runtime.sh b/scripts/verify-ksail-arc-runtime.sh index f06ee45b1a..fab8cfa510 100755 --- a/scripts/verify-ksail-arc-runtime.sh +++ b/scripts/verify-ksail-arc-runtime.sh @@ -120,6 +120,7 @@ done stage=registration-and-bounds kc -n "$namespace" get autoscalingrunnerset.actions.github.com ksail-code-quality -o json >"$scratch/ars.json" jq -e ' + .spec.template.spec.volumes[2].configMap.name as $metrics | .status.phase == "Running" and .status.observedGeneration == .metadata.generation and ((.metadata.annotations["runner-scale-set-id"] | tonumber) > 0) and .spec.githubConfigUrl == "https://github.com/devantler-tech/ksail" and @@ -141,14 +142,29 @@ jq -e ' .spec.template.spec.containers[0].resources.limits["ephemeral-storage"] == "48Gi" and .spec.template.spec.nodeSelector["platform.devantler.tech/ksail-analysis"] == "enabled" and .spec.template.spec.tolerations == [{"key":"platform.devantler.tech/ksail-analysis","operator":"Equal","value":"enabled","effect":"NoSchedule"}] and - .spec.template.spec.volumes == [{"name":"runner-home","emptyDir":{"sizeLimit":"40Gi"}},{"name":"runner-tmp","emptyDir":{"sizeLimit":"2Gi"}}] and + ($metrics | test("^ksail-arc-job-metrics-[a-z0-9]{10}$")) and + .spec.template.spec.volumes == [{"name":"runner-home","emptyDir":{"sizeLimit":"40Gi"}},{"name":"runner-tmp","emptyDir":{"sizeLimit":"2Gi"}}, + {"name":"runner-metrics","configMap":{"name":$metrics,"defaultMode":365,"items":[{"key":"job-metrics.sh","path":"job-metrics.sh"}]}}] and + .spec.template.spec.containers[0].env == [{"name":"ACTIONS_RUNNER_HOOK_JOB_COMPLETED","value":"/etc/ksail-arc-metrics/job-metrics.sh"}] and + .spec.template.spec.containers[0].volumeMounts == [{"name":"runner-home","mountPath":"/home/runner"},{"name":"runner-tmp","mountPath":"/tmp"}, + {"name":"runner-metrics","mountPath":"/etc/ksail-arc-metrics","readOnly":true}] and + (.spec.template.spec.initContainers[0].env // [] | length) == 0 and + .spec.template.spec.initContainers[0].volumeMounts == [{"name":"runner-home","mountPath":"/runner-data"}] and all((.spec.template.spec.containers + .spec.template.spec.initContainers)[]; .securityContext.readOnlyRootFilesystem == true and .securityContext.privileged == false and .securityContext.allowPrivilegeEscalation == false and .securityContext.capabilities.drop == ["ALL"] and - (.env // [] | length) == 0 and .envFrom == null and + .envFrom == null and (.image | test("^ghcr.io/devantler-tech/ksail-analysis-runner@sha256:[0-9a-f]{64}$"))) and .spec.template.spec.initContainers[0].image == .spec.template.spec.containers[0].image ' "$scratch/ars.json" >/dev/null +stage=immutable-job-metrics +metrics_name=$(jq -er '.spec.template.spec.volumes[2].configMap.name' "$scratch/ars.json") +kc -n "$namespace" get configmap "$metrics_name" -o json >"$scratch/metrics-config.json" +jq -e --rawfile expected k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh ' + .immutable == true and (.metadata.uid | type == "string" and length > 0) and + .metadata.annotations["kustomize.toolkit.fluxcd.io/substitute"] == "disabled" and + (.data | length) == 1 and (.binaryData // {} | length) == 0 and .data["job-metrics.sh"] == $expected +' "$scratch/metrics-config.json" >/dev/null image=$(jq -r '.spec.template.spec.containers[0].image' "$scratch/ars.json") expected_image=$(yq '.spec.values.template.spec.containers[0].image' k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml) [[ "$image" == "$expected_image" ]] || fail image-drift @@ -328,5 +344,7 @@ kc -n default get endpoints kubernetes -o json | jq -c '.subsets' >"$scratch/api jq -c '.subsets' "$scratch/api.json" >"$scratch/api-before" cmp -s "$scratch/api-before" "$scratch/api-after" || fail api-target-churn +stage=job-cgroup-measurement +quiet kc -n "$namespace" exec "$probe" -- /etc/ksail-arc-metrics/job-metrics.sh stage=complete printf 'PASS: ARC registration, restricted image, admission and two intercepted egress denials; manifest=%s\n' "$PLATFORM_MANIFEST_DIGEST" From f0e65ab48e432fd0e70c4414f0abda1db09e1383 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 05:37:23 +0200 Subject: [PATCH 17/62] style(ci): normalize ARC hook source --- k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh | 1 - scripts/validate-eks-ci-role-policy/approved-surface.txt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh b/k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh index 3d546026d7..f158d1121e 100755 --- a/k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh +++ b/k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh @@ -39,4 +39,3 @@ if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then [[ $# == 0 ]] || { arc_metrics_fail arguments; exit 1; } arc_job_metrics /sys/fs/cgroup fi - diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index 4d66b3e388..6a5bdf6b60 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -14,7 +14,7 @@ helm.toolkit.fluxcd.io/v2|HelmRelease|actual-budget|actual-budget adc009b64f2a9da1427f0c9b80f6ea2490607a8b55ceb6e139f73de92399d5da -helm.toolkit.fluxcd.io/v2|HelmRelease|arc-ksail-analysis|ksail-analysis-runners 15484b7d4dfde81d022fdc9fe595d02b4f3b415a0f60ddbe0c3617f3eac6e101 +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-ksail-analysis|ksail-analysis-runners 056994be352f2144f8405b22a395d16cba32537b1d2a67de3e30385c67446a40 helm.toolkit.fluxcd.io/v2|HelmRelease|arc-systems|arc-controller f7bf55370e5a38911521edbafec8af715676f3d87b39ead456c7875988845637 From 72e4d53d95239084767c9e604f50c61952811a51 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 05:53:46 +0200 Subject: [PATCH 18/62] fix(ci): publish the complete ARC metrics hook --- .github/workflows/ci.yaml | 5 +- docs/operations/ksail-arc.md | 5 ++ .../config-map-job-metrics.yaml | 54 ++++++++++++++ .../ksail-analysis-runners/kustomization.yaml | 25 ++++--- .../ksail-analysis-runners/kustomizeconfig | 8 --- .../generate-ksail-arc-job-metrics/main.go | 68 ++++++++++++++++++ .../main_test.go | 34 +++++++++ .../ksail-arc-job-metrics.sh | 0 .../job_metrics_publication_test.go | 72 +++++++++++++++++++ scripts/tests/arc-staging/job_metrics_test.go | 12 ++-- .../tests/test-verify-ksail-arc-runtime.sh | 7 +- .../approved-surface.txt | 2 +- scripts/verify-ksail-arc-runtime.sh | 2 +- 13 files changed, 261 insertions(+), 33 deletions(-) create mode 100644 k8s/bases/infrastructure/ksail-analysis-runners/config-map-job-metrics.yaml delete mode 100644 k8s/bases/infrastructure/ksail-analysis-runners/kustomizeconfig create mode 100644 scripts/generate-ksail-arc-job-metrics/main.go create mode 100644 scripts/generate-ksail-arc-job-metrics/main_test.go rename k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh => scripts/ksail-arc-job-metrics.sh (100%) create mode 100644 scripts/tests/arc-staging/job_metrics_publication_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1d4e4a1e72..c30ffa9a06 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -149,10 +149,13 @@ jobs: - name: 🛑 Validate bounded ARC activation boundaries run: go test ./scripts/tests/arc-staging + - name: 🧾 Validate published ARC metrics bytes + run: go test ./scripts/generate-ksail-arc-job-metrics + - name: 🔬 Validate bounded ARC runtime acceptance run: | go test -race ./scripts/verify-ksail-arc-runtime - shellcheck scripts/verify-ksail-arc-runtime.sh scripts/wait-for-ksail-arc-registration.sh scripts/tests/test-verify-ksail-arc-runtime.sh k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh + shellcheck scripts/verify-ksail-arc-runtime.sh scripts/wait-for-ksail-arc-registration.sh scripts/tests/test-verify-ksail-arc-runtime.sh scripts/ksail-arc-job-metrics.sh bash scripts/tests/test-verify-ksail-arc-runtime.sh # Shared Wedding backup access is retired on this proof's word, so it must diff --git a/docs/operations/ksail-arc.md b/docs/operations/ksail-arc.md index 2cefa98272..c4e0f48723 100644 --- a/docs/operations/ksail-arc.md +++ b/docs/operations/ksail-arc.md @@ -78,6 +78,11 @@ routing managed analysis: values or JIT configuration are read or printed. The protected canary verifies the exact immutable hook bytes and executes it. This is telemetry from the job account, not an independent security boundary or a substitute for job success. + The canonical source is `scripts/ksail-arc-job-metrics.sh`. After an edit, run + `go run ./scripts/generate-ksail-arc-job-metrics -write`; CI rejects stale + generated bytes. The published YAML contains the script and its content-hash + name, and an inline replacement binds that name to the runner mount. The + artifact render must not depend on files excluded by the publisher. Measure actual managed-job peak and extraction duration; revise sizing and any narrowly justified quota change before admitting real traffic. 4. Render both pinned charts, including their CRDs and namespace-scoped RBAC. diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/config-map-job-metrics.yaml b/k8s/bases/infrastructure/ksail-analysis-runners/config-map-job-metrics.yaml new file mode 100644 index 0000000000..3012465b39 --- /dev/null +++ b/k8s/bases/infrastructure/ksail-analysis-runners/config-map-job-metrics.yaml @@ -0,0 +1,54 @@ +# Generated by go run ./scripts/generate-ksail-arc-job-metrics -write; do not edit. +apiVersion: v1 +data: + job-metrics.sh: | + #!/usr/bin/env bash + set -euo pipefail + + arc_metrics_fail() { + printf '::error::KSail ARC whole-job cgroup measurement failed: %s\n' "$1" >&2 + return 1 + } + + # The production entrypoint uses only the container's kernel cgroup. A supplied + # fixture directory is used when this function is sourced by offline tests. + arc_job_metrics() { + local root=$1 limit peak events key value extra + local low='' high='' max='' oom='' oom_kill='' oom_group_kill='' + limit=$(timeout 5s cat "$root/memory.max" 2>/dev/null) || { arc_metrics_fail read-limit; return 1; } + peak=$(timeout 5s cat "$root/memory.peak" 2>/dev/null) || { arc_metrics_fail read-peak; return 1; } + events=$(timeout 5s cat "$root/memory.events" 2>/dev/null) || { arc_metrics_fail read-events; return 1; } + [[ "$limit" == 15032385536 ]] || { arc_metrics_fail limit; return 1; } + [[ "$peak" =~ ^(0|[1-9][0-9]{0,17})$ && "$peak" -gt 0 && "$peak" -le "$limit" ]] || { + arc_metrics_fail peak; return 1; + } + while read -r key value extra; do + [[ -z "$extra" && "$value" =~ ^(0|[1-9][0-9]{0,17})$ ]] || { arc_metrics_fail event-format; return 1; } + case "$key" in + low|high|max|oom|oom_kill|oom_group_kill) + [[ -z "${!key}" ]] || { arc_metrics_fail duplicate-event; return 1; } + printf -v "$key" '%s' "$value" ;; + *) arc_metrics_fail unknown-event; return 1 ;; + esac + done <<<"$events" + [[ -n "$low" && -n "$high" && -n "$max" && -n "$oom" && -n "$oom_kill" && -n "$oom_group_kill" ]] || { + arc_metrics_fail missing-event; return 1; + } + [[ "$oom" == 0 && "$oom_kill" == 0 && "$oom_group_kill" == 0 ]] || { arc_metrics_fail oom; return 1; } + printf 'KSail ARC whole-job cgroup: {"schemaVersion":1,"memoryMaxBytes":%s,"memoryPeakBytes":%s,"limitEvents":%s,"oomEvents":%s,"oomKills":%s,"oomGroupKills":%s}\n' \ + "$limit" "$peak" "$max" "$oom" "$oom_kill" "$oom_group_kill" + } + + if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + [[ $# == 0 ]] || { arc_metrics_fail arguments; exit 1; } + arc_job_metrics /sys/fs/cgroup + fi +immutable: true +kind: ConfigMap +metadata: + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled + labels: + app.kubernetes.io/name: ksail-arc-job-metrics + name: ksail-arc-job-metrics-d6e7291092 + namespace: arc-ksail-analysis diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/kustomization.yaml b/k8s/bases/infrastructure/ksail-analysis-runners/kustomization.yaml index 24837960b8..25d25c8460 100644 --- a/k8s/bases/infrastructure/ksail-analysis-runners/kustomization.yaml +++ b/k8s/bases/infrastructure/ksail-analysis-runners/kustomization.yaml @@ -1,17 +1,20 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -configurations: - - kustomizeconfig -configMapGenerator: - - name: ksail-arc-job-metrics - namespace: arc-ksail-analysis - files: - - job-metrics.sh - options: - immutable: true - annotations: - kustomize.toolkit.fluxcd.io/substitute: disabled +replacements: + - source: + kind: ConfigMap + version: v1 + namespace: arc-ksail-analysis + fieldPath: metadata.name + targets: + - select: + kind: HelmRelease + name: ksail-analysis-runners + namespace: arc-ksail-analysis + fieldPaths: + - spec.values.template.spec.volumes.[name=runner-metrics].configMap.name resources: + - config-map-job-metrics.yaml - oci-repository.yaml - helm-release.yaml - external-secret.yaml diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/kustomizeconfig b/k8s/bases/infrastructure/ksail-analysis-runners/kustomizeconfig deleted file mode 100644 index 1375427aaa..0000000000 --- a/k8s/bases/infrastructure/ksail-analysis-runners/kustomizeconfig +++ /dev/null @@ -1,8 +0,0 @@ -nameReference: - - kind: ConfigMap - version: v1 - fieldSpecs: - - kind: HelmRelease - group: helm.toolkit.fluxcd.io - version: v2 - path: spec/values/template/spec/volumes/configMap/name diff --git a/scripts/generate-ksail-arc-job-metrics/main.go b/scripts/generate-ksail-arc-job-metrics/main.go new file mode 100644 index 0000000000..d8e052b326 --- /dev/null +++ b/scripts/generate-ksail-arc-job-metrics/main.go @@ -0,0 +1,68 @@ +// Generate the public metrics hook as a manifest the KSail publisher includes. +package main + +import ( + "crypto/sha256" + "flag" + "fmt" + "os" + + "sigs.k8s.io/yaml" +) + +const ( + scriptPath = "scripts/ksail-arc-job-metrics.sh" + manifestPath = "k8s/bases/infrastructure/ksail-analysis-runners/config-map-job-metrics.yaml" +) + +func generate(script []byte) ([]byte, error) { + if len(script) == 0 || len(script) > 64*1024 { + return nil, fmt.Errorf("metrics hook must contain 1–65536 bytes") + } + sum := sha256.Sum256(script) + manifest := map[string]any{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]any{ + "name": fmt.Sprintf("ksail-arc-job-metrics-%x", sum[:5]), + "namespace": "arc-ksail-analysis", + "labels": map[string]string{"app.kubernetes.io/name": "ksail-arc-job-metrics"}, + "annotations": map[string]string{"kustomize.toolkit.fluxcd.io/substitute": "disabled"}, + }, + "immutable": true, + "data": map[string]string{"job-metrics.sh": string(script)}, + } + data, err := yaml.Marshal(manifest) + if err != nil { + return nil, err + } + return append([]byte("# Generated by go run ./scripts/generate-ksail-arc-job-metrics -write; do not edit.\n"), data...), nil +} + +func run() error { + write := flag.Bool("write", false, "update the generated manifest") + flag.Parse() + if flag.NArg() != 0 { + return fmt.Errorf("unexpected positional arguments") + } + script, err := os.ReadFile(scriptPath) + if err != nil { + return err + } + data, err := generate(script) + if err != nil { + return err + } + if *write { + return os.WriteFile(manifestPath, data, 0644) + } + _, err = os.Stdout.Write(data) + return err +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/scripts/generate-ksail-arc-job-metrics/main_test.go b/scripts/generate-ksail-arc-job-metrics/main_test.go new file mode 100644 index 0000000000..c7b9803a85 --- /dev/null +++ b/scripts/generate-ksail-arc-job-metrics/main_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "testing" +) + +func TestPublishedMetricsManifestIsCurrent(t *testing.T) { + script, err := os.ReadFile(filepath.Join("../..", scriptPath)) + if err != nil { + t.Fatal(err) + } + expected, err := generate(script) + if err != nil { + t.Fatal(err) + } + published, err := os.ReadFile(filepath.Join("../..", manifestPath)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(published, expected) { + t.Fatal("published hook differs from its source; run go run ./scripts/generate-ksail-arc-job-metrics -write") + } +} + +func TestMetricsManifestRejectsEmptyOrOversizedSource(t *testing.T) { + for _, size := range []int{0, 65537} { + if _, err := generate(make([]byte, size)); err == nil { + t.Fatalf("accepted %d bytes", size) + } + } +} diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh b/scripts/ksail-arc-job-metrics.sh similarity index 100% rename from k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh rename to scripts/ksail-arc-job-metrics.sh diff --git a/scripts/tests/arc-staging/job_metrics_publication_test.go b/scripts/tests/arc-staging/job_metrics_publication_test.go new file mode 100644 index 0000000000..c8713fd582 --- /dev/null +++ b/scripts/tests/arc-staging/job_metrics_publication_test.go @@ -0,0 +1,72 @@ +package arcstaging_test + +import ( + "bytes" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestMetricsHookSurvivesPublishedArtifactRender(t *testing.T) { + // The real publisher includes manifest files, not arbitrary checkout files. + // Render that artifact boundary so a checkout-only dependency cannot pass. + component := filepath.Join(repoRoot, metricsComponent) + published := t.TempDir() + files, err := os.ReadDir(component) + if err != nil { + t.Fatal(err) + } + for _, file := range files { + switch strings.ToLower(filepath.Ext(file.Name())) { + case ".yaml", ".yml", ".json": + data, err := os.ReadFile(filepath.Join(component, file.Name())) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(published, file.Name()), data, 0600); err != nil { + t.Fatal(err) + } + } + } + rendered, err := exec.Command("kubectl", "kustomize", published).CombinedOutput() + if err != nil { + t.Fatalf("published metrics dependencies cannot render: %v\n%s", err, rendered) + } + script, err := os.ReadFile(filepath.Join(repoRoot, metricsScript)) + if err != nil { + t.Fatal(err) + } + decoder := yaml.NewDecoder(bytes.NewReader(rendered)) + var config, release map[string]any + for { + var object map[string]any + if err := decoder.Decode(&object); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + if object["kind"] == "ConfigMap" { + if config != nil { + t.Fatal("ambiguous metrics ConfigMap") + } + config = object + } + if object["kind"] == "HelmRelease" { + release = object + } + } + if config == nil || release == nil { + t.Fatal("published artifact lacks the metrics ConfigMap or runner release") + } + equal(t, field(t, config, "immutable"), true) + equal(t, field(t, config, "metadata", "annotations", "kustomize.toolkit.fluxcd.io/substitute"), "disabled") + equal(t, field(t, config, "data", "job-metrics.sh"), string(script)) + volumes := field(t, release, "spec", "values", "template", "spec", "volumes").([]any) + equal(t, field(t, volumes[2], "configMap", "name"), field(t, config, "metadata", "name")) + equal(t, field(t, volumes[2], "configMap", "defaultMode"), 365) +} diff --git a/scripts/tests/arc-staging/job_metrics_test.go b/scripts/tests/arc-staging/job_metrics_test.go index 1a016e68f4..4c20740cf4 100644 --- a/scripts/tests/arc-staging/job_metrics_test.go +++ b/scripts/tests/arc-staging/job_metrics_test.go @@ -10,15 +10,13 @@ import ( ) const metricsComponent = "k8s/bases/infrastructure/ksail-analysis-runners" -const metricsScript = metricsComponent + "/job-metrics.sh" +const metricsScript = "scripts/ksail-arc-job-metrics.sh" func TestRunnerRetainsWholeJobCgroupMeasurement(t *testing.T) { - component := readYAML(t, metricsComponent+"/kustomization.yaml") - generator := field(t, component, "configMapGenerator").([]any) - equal(t, len(generator), 1) - equal(t, field(t, generator[0], "namespace"), "arc-ksail-analysis") - equal(t, field(t, generator[0], "options", "immutable"), true) - equal(t, field(t, generator[0], "options", "annotations", "kustomize.toolkit.fluxcd.io/substitute"), "disabled") + config := readYAML(t, metricsComponent+"/config-map-job-metrics.yaml") + equal(t, field(t, config, "metadata", "namespace"), "arc-ksail-analysis") + equal(t, field(t, config, "immutable"), true) + equal(t, field(t, config, "metadata", "annotations", "kustomize.toolkit.fluxcd.io/substitute"), "disabled") release := readYAML(t, metricsComponent+"/helm-release.yaml") runner := field(t, release, "spec", "values", "template", "spec", "containers").([]any)[0] env := field(t, runner, "env").([]any) diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh index 33fd6ea0ac..9df94a0f17 100755 --- a/scripts/tests/test-verify-ksail-arc-runtime.sh +++ b/scripts/tests/test-verify-ksail-arc-runtime.sh @@ -13,8 +13,7 @@ if [[ -e "$root/scripts/wait-for-ksail-arc-registration.sh" ]]; then fi cp "$root/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml" \ "$scratch/k8s/bases/infrastructure/ksail-analysis-runners/" -cp "$root/k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh" \ - "$scratch/k8s/bases/infrastructure/ksail-analysis-runners/" +cp "$root/scripts/ksail-arc-job-metrics.sh" "$scratch/scripts/" cp "$root/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml" \ "$scratch/k8s/bases/infrastructure/controllers/actions-runner-controller/" yq -i '.spec.suspend=false' "$scratch/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml" @@ -138,7 +137,7 @@ case "$args" in *) cat "$ARC_TEST_ROOT/ars" ;; esac ;; *'get configmap ksail-arc-job-metrics-'*) - jq -cn --rawfile script "$ARC_TEST_ROOT/k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh" \ + jq -cn --rawfile script "$ARC_TEST_ROOT/scripts/ksail-arc-job-metrics.sh" \ --arg scenario "$ARC_TEST_CASE" '{metadata:{uid:"metrics-uid",annotations:{"kustomize.toolkit.fluxcd.io/substitute":"disabled"}},immutable:($scenario!="mutable-metrics"), data:{"job-metrics.sh":(if $scenario=="tampered-metrics" then "unverified" else $script end)}}' ;; *'get resourcequotas '*) @@ -223,7 +222,7 @@ case "$args" in printf '0\n' >"$ARC_TEST_ROOT/cgroup/memory.peak" else printf '9876543210\n' >"$ARC_TEST_ROOT/cgroup/memory.peak"; fi printf 'low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n' >"$ARC_TEST_ROOT/cgroup/memory.events" - source "$ARC_TEST_ROOT/k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh" + source "$ARC_TEST_ROOT/scripts/ksail-arc-job-metrics.sh" arc_job_metrics "$ARC_TEST_ROOT/cgroup" touch "$ARC_TEST_ROOT/metrics-executed" ;; *'exec observer -c cilium-agent -- hubble observe '*) diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index 6a5bdf6b60..99e9bd6a4a 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -14,7 +14,7 @@ helm.toolkit.fluxcd.io/v2|HelmRelease|actual-budget|actual-budget adc009b64f2a9da1427f0c9b80f6ea2490607a8b55ceb6e139f73de92399d5da -helm.toolkit.fluxcd.io/v2|HelmRelease|arc-ksail-analysis|ksail-analysis-runners 056994be352f2144f8405b22a395d16cba32537b1d2a67de3e30385c67446a40 +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-ksail-analysis|ksail-analysis-runners 732d3152fa99bd142ee7469b9625b7daaef5964e24cabbadd62486413742ce03 helm.toolkit.fluxcd.io/v2|HelmRelease|arc-systems|arc-controller f7bf55370e5a38911521edbafec8af715676f3d87b39ead456c7875988845637 diff --git a/scripts/verify-ksail-arc-runtime.sh b/scripts/verify-ksail-arc-runtime.sh index fab8cfa510..80c15a75c1 100755 --- a/scripts/verify-ksail-arc-runtime.sh +++ b/scripts/verify-ksail-arc-runtime.sh @@ -160,7 +160,7 @@ jq -e ' stage=immutable-job-metrics metrics_name=$(jq -er '.spec.template.spec.volumes[2].configMap.name' "$scratch/ars.json") kc -n "$namespace" get configmap "$metrics_name" -o json >"$scratch/metrics-config.json" -jq -e --rawfile expected k8s/bases/infrastructure/ksail-analysis-runners/job-metrics.sh ' +jq -e --rawfile expected scripts/ksail-arc-job-metrics.sh ' .immutable == true and (.metadata.uid | type == "string" and length > 0) and .metadata.annotations["kustomize.toolkit.fluxcd.io/substitute"] == "disabled" and (.data | length) == 1 and (.binaryData // {} | length) == 0 and .data["job-metrics.sh"] == $expected From cb8a31f5fd8afea0e809cd75cf099a37d0270e79 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 10:33:30 +0200 Subject: [PATCH 19/62] fix(arc): enable the KSail-only organization runner group --- .github/actions/deploy-prod/action.yml | 11 +++ .github/workflows/ci.yaml | 5 ++ docs/github-management.md | 17 ++++ k8s/bases/apps/github-config/role.yaml | 1 + .../restrict-github-runner-group.yaml | 78 +++++++++++++++++++ .../cluster-policies/kustomization.yaml | 1 + .../managed-resource-activation-policy.yaml | 2 + ...st-verify-ksail-runner-group-capability.sh | 41 ++++++++++ .../verify-ksail-runner-group-capability.sh | 50 ++++++++++++ .../all-repositories/kyverno-test.yaml | 21 +++++ .../all-repositories/resources.yaml | 24 ++++++ .../all-repositories/user-info.yaml | 6 ++ .../all-repositories/values.yaml | 14 ++++ .../different-name/kyverno-test.yaml | 21 +++++ .../different-name/resources.yaml | 24 ++++++ .../different-name/user-info.yaml | 6 ++ .../different-name/values.yaml | 14 ++++ .../empty-selection/kyverno-test.yaml | 21 +++++ .../empty-selection/resources.yaml | 22 ++++++ .../empty-selection/user-info.yaml | 6 ++ .../empty-selection/values.yaml | 14 ++++ .../foreign-external-name/kyverno-test.yaml | 21 +++++ .../foreign-external-name/resources.yaml | 27 +++++++ .../foreign-external-name/user-info.yaml | 6 ++ .../foreign-external-name/values.yaml | 14 ++++ .../init-provider-override/kyverno-test.yaml | 21 +++++ .../init-provider-override/resources.yaml | 30 +++++++ .../init-provider-override/user-info.yaml | 6 ++ .../init-provider-override/values.yaml | 14 ++++ .../ksail-only/kyverno-test.yaml | 21 +++++ .../ksail-only/resources.yaml | 24 ++++++ .../ksail-only/user-info.yaml | 6 ++ .../ksail-only/values.yaml | 14 ++++ .../lookalike-provider/kyverno-test.yaml | 18 +++++ .../lookalike-provider/resources.yaml | 27 +++++++ .../lookalike-provider/user-info.yaml | 6 ++ .../lookalike-provider/values.yaml | 14 ++++ .../missing-selection/kyverno-test.yaml | 21 +++++ .../missing-selection/resources.yaml | 21 +++++ .../missing-selection/user-info.yaml | 6 ++ .../missing-selection/values.yaml | 14 ++++ .../other-object-name/kyverno-test.yaml | 21 +++++ .../other-object-name/resources.yaml | 24 ++++++ .../other-object-name/user-info.yaml | 6 ++ .../other-object-name/values.yaml | 14 ++++ .../other-provider/kyverno-test.yaml | 21 +++++ .../other-provider/resources.yaml | 24 ++++++ .../other-provider/user-info.yaml | 6 ++ .../other-provider/values.yaml | 14 ++++ .../other-repository/kyverno-test.yaml | 21 +++++ .../other-repository/resources.yaml | 24 ++++++ .../other-repository/user-info.yaml | 6 ++ .../other-repository/values.yaml | 14 ++++ .../kyverno-test.yaml | 18 +++++ .../resources.yaml | 27 +++++++ .../user-info.yaml | 6 ++ .../values.yaml | 14 ++++ .../second-repository/kyverno-test.yaml | 21 +++++ .../second-repository/resources.yaml | 25 ++++++ .../second-repository/user-info.yaml | 6 ++ .../second-repository/values.yaml | 14 ++++ .../kyverno-test.yaml | 21 +++++ .../resources.yaml | 21 +++++ .../user-info.yaml | 6 ++ .../unrestricted-public-default/values.yaml | 14 ++++ .../update-rebound/kyverno-test.yaml | 18 +++++ .../update-rebound/resources.yaml | 27 +++++++ .../update-rebound/user-info.yaml | 6 ++ .../update-rebound/values.yaml | 14 ++++ .../update-unchanged/kyverno-test.yaml | 18 +++++ .../update-unchanged/resources.yaml | 27 +++++++ .../update-unchanged/user-info.yaml | 6 ++ .../update-unchanged/values.yaml | 14 ++++ .../wildcard-external-name/kyverno-test.yaml | 21 +++++ .../wildcard-external-name/resources.yaml | 27 +++++++ .../wildcard-external-name/user-info.yaml | 6 ++ .../wildcard-external-name/values.yaml | 14 ++++ .../wildcard-name/kyverno-test.yaml | 21 +++++ .../wildcard-name/resources.yaml | 24 ++++++ .../wildcard-name/user-info.yaml | 6 ++ .../wildcard-name/values.yaml | 14 ++++ 81 files changed, 1381 insertions(+) create mode 100644 k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml create mode 100755 scripts/tests/test-verify-ksail-runner-group-capability.sh create mode 100755 scripts/verify-ksail-runner-group-capability.sh create mode 100644 tests/restrict-github-runner-group/all-repositories/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/all-repositories/resources.yaml create mode 100644 tests/restrict-github-runner-group/all-repositories/user-info.yaml create mode 100644 tests/restrict-github-runner-group/all-repositories/values.yaml create mode 100644 tests/restrict-github-runner-group/different-name/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/different-name/resources.yaml create mode 100644 tests/restrict-github-runner-group/different-name/user-info.yaml create mode 100644 tests/restrict-github-runner-group/different-name/values.yaml create mode 100644 tests/restrict-github-runner-group/empty-selection/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/empty-selection/resources.yaml create mode 100644 tests/restrict-github-runner-group/empty-selection/user-info.yaml create mode 100644 tests/restrict-github-runner-group/empty-selection/values.yaml create mode 100644 tests/restrict-github-runner-group/foreign-external-name/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/foreign-external-name/resources.yaml create mode 100644 tests/restrict-github-runner-group/foreign-external-name/user-info.yaml create mode 100644 tests/restrict-github-runner-group/foreign-external-name/values.yaml create mode 100644 tests/restrict-github-runner-group/init-provider-override/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/init-provider-override/resources.yaml create mode 100644 tests/restrict-github-runner-group/init-provider-override/user-info.yaml create mode 100644 tests/restrict-github-runner-group/init-provider-override/values.yaml create mode 100644 tests/restrict-github-runner-group/ksail-only/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/ksail-only/resources.yaml create mode 100644 tests/restrict-github-runner-group/ksail-only/user-info.yaml create mode 100644 tests/restrict-github-runner-group/ksail-only/values.yaml create mode 100644 tests/restrict-github-runner-group/lookalike-provider/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/lookalike-provider/resources.yaml create mode 100644 tests/restrict-github-runner-group/lookalike-provider/user-info.yaml create mode 100644 tests/restrict-github-runner-group/lookalike-provider/values.yaml create mode 100644 tests/restrict-github-runner-group/missing-selection/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/missing-selection/resources.yaml create mode 100644 tests/restrict-github-runner-group/missing-selection/user-info.yaml create mode 100644 tests/restrict-github-runner-group/missing-selection/values.yaml create mode 100644 tests/restrict-github-runner-group/other-object-name/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/other-object-name/resources.yaml create mode 100644 tests/restrict-github-runner-group/other-object-name/user-info.yaml create mode 100644 tests/restrict-github-runner-group/other-object-name/values.yaml create mode 100644 tests/restrict-github-runner-group/other-provider/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/other-provider/resources.yaml create mode 100644 tests/restrict-github-runner-group/other-provider/user-info.yaml create mode 100644 tests/restrict-github-runner-group/other-provider/values.yaml create mode 100644 tests/restrict-github-runner-group/other-repository/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/other-repository/resources.yaml create mode 100644 tests/restrict-github-runner-group/other-repository/user-info.yaml create mode 100644 tests/restrict-github-runner-group/other-repository/values.yaml create mode 100644 tests/restrict-github-runner-group/provider-records-created-group/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/provider-records-created-group/resources.yaml create mode 100644 tests/restrict-github-runner-group/provider-records-created-group/user-info.yaml create mode 100644 tests/restrict-github-runner-group/provider-records-created-group/values.yaml create mode 100644 tests/restrict-github-runner-group/second-repository/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/second-repository/resources.yaml create mode 100644 tests/restrict-github-runner-group/second-repository/user-info.yaml create mode 100644 tests/restrict-github-runner-group/second-repository/values.yaml create mode 100644 tests/restrict-github-runner-group/unrestricted-public-default/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/unrestricted-public-default/resources.yaml create mode 100644 tests/restrict-github-runner-group/unrestricted-public-default/user-info.yaml create mode 100644 tests/restrict-github-runner-group/unrestricted-public-default/values.yaml create mode 100644 tests/restrict-github-runner-group/update-rebound/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/update-rebound/resources.yaml create mode 100644 tests/restrict-github-runner-group/update-rebound/user-info.yaml create mode 100644 tests/restrict-github-runner-group/update-rebound/values.yaml create mode 100644 tests/restrict-github-runner-group/update-unchanged/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/update-unchanged/resources.yaml create mode 100644 tests/restrict-github-runner-group/update-unchanged/user-info.yaml create mode 100644 tests/restrict-github-runner-group/update-unchanged/values.yaml create mode 100644 tests/restrict-github-runner-group/wildcard-external-name/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/wildcard-external-name/resources.yaml create mode 100644 tests/restrict-github-runner-group/wildcard-external-name/user-info.yaml create mode 100644 tests/restrict-github-runner-group/wildcard-external-name/values.yaml create mode 100644 tests/restrict-github-runner-group/wildcard-name/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/wildcard-name/resources.yaml create mode 100644 tests/restrict-github-runner-group/wildcard-name/user-info.yaml create mode 100644 tests/restrict-github-runner-group/wildcard-name/values.yaml diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index 230326a286..f44704bf0e 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -554,6 +554,17 @@ runs: shell: bash run: ./scripts/verify-gateway-routes-applied.sh + - name: Verify the KSail-only runner-group capability + if: >- + !cancelled() && + steps.wait_flux_revision.outcome == 'success' && + ( + steps.cluster_update.outcome == 'skipped' || + steps.wait_prod_api_stability.outcome == 'success' + ) + shell: bash + run: bash scripts/verify-ksail-runner-group-capability.sh + - name: Verify the deployed data product UI release # The protected deployment must prove current Apps/chart/Helm identities, # all current Pods and public endpoints before its candidate can merge. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 787667d246..4fec2cbd2e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -149,6 +149,11 @@ jobs: - name: 🛑 Validate inactive ARC staging boundaries run: go test ./scripts/tests/arc-staging + - name: Validate KSail runner-group capability acceptance + run: | + shellcheck scripts/verify-ksail-runner-group-capability.sh scripts/tests/test-verify-ksail-runner-group-capability.sh + bash scripts/tests/test-verify-ksail-runner-group-capability.sh + # Shared Wedding backup access is retired on this proof's word, so it must # pass only on an AccessDenied for every access, after a positive control # and against an object the shared credential showed exists. These cases diff --git a/docs/github-management.md b/docs/github-management.md index 8df1818e87..685d8258c8 100644 --- a/docs/github-management.md +++ b/docs/github-management.md @@ -126,6 +126,23 @@ reconcile, then batch the rest. where an external object already exists. Each active MRD costs the apiserver ~3 MiB; activate only what is used. +## KSail analysis runner group + +The existing GitHub App's organization runner permission supports the dedicated +`ksail-code-quality` runner group. The namespaced RunnerGroup kind is explicitly +activated, and only the `github-config` tenant Role receives its managed-resource +verbs. The declaration lives in `.github` and selects only KSail repository +`737584922`; public-repository use is necessary because KSail is public, while +the one-item selection keeps every other repository excluded. + +Admission requires this exact group name, the existing `default` ProviderConfig, +selected visibility, and the KSail-only repository set. It refuses initialization +overrides and tenant changes to the group's remote identity. The protected +deployment proves the installed CRD and tenant permissions, admits the permitted +declaration using server dry-run, and observes explicit admission refusals for +organization-wide access and foreign identity adoption. ARC activation and +managed-analysis routing require separate registered-runner acceptance. + ## Safety rails - **Never `Delete`** — every managed resource omits `Delete` from diff --git a/k8s/bases/apps/github-config/role.yaml b/k8s/bases/apps/github-config/role.yaml index 5d1165f997..1902e348b0 100644 --- a/k8s/bases/apps/github-config/role.yaml +++ b/k8s/bases/apps/github-config/role.yaml @@ -75,6 +75,7 @@ rules: - actions.github.m.upbound.io resources: - repositorypermissions + - runnergroups verbs: - get - list diff --git a/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml b/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml new file mode 100644 index 0000000000..3ba8e71160 --- /dev/null +++ b/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml @@ -0,0 +1,78 @@ +# The GitHub tenant can manage only the KSail analysis runner group. The +# organization's existing App already grants runner management; this policy +# confines the new managed-resource surface without changing that App. +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: restrict-github-runner-group + annotations: + policies.kyverno.io/title: Restrict GitHub Analysis Runner Group + policies.kyverno.io/category: Security, GitHub + policies.kyverno.io/severity: high + policies.kyverno.io/subject: RunnerGroup + policies.kyverno.io/minversion: 1.13.0 +spec: + background: false + rules: + - name: ksail-only-runner-group + match: + any: + - resources: + kinds: + - actions.github.m.upbound.io/*/RunnerGroup + namespaces: + - github-config + operations: [CREATE, UPDATE] + validate: + failureAction: Enforce + allowExistingViolations: false + message: >- + The analysis runner group must be ksail-code-quality, use the existing + default ProviderConfig, and select only KSail. initProvider overrides, + default or organization-wide groups, and additional repositories are refused. + deny: + conditions: + all: + # JMESPath compares literal names and the complete numeric set; + # Kyverno wildcard-aware string operators cannot express this boundary. + - key: "{{ request.object.metadata.name == 'ksail-code-quality' && request.object.spec.forProvider.name == 'ksail-code-quality' && request.object.spec.forProvider.visibility == 'selected' && request.object.spec.forProvider.selectedRepositoryIds == `[737584922]` && request.object.spec.forProvider.allowsPublicRepositories == `true` && request.object.spec.forProvider.restrictedToWorkflows == `false` && not_null(request.object.spec.forProvider.selectedWorkflows, `[]`) == `[]` && not_null(request.object.spec.initProvider, `{}`) == `{}` && request.object.spec.providerConfigRef.name == 'default' && not_null(request.object.spec.providerConfigRef.kind, 'ProviderConfig') == 'ProviderConfig' }}" + operator: Equals + value: false + - name: tenant-cannot-rebind-group + match: + any: + - resources: + kinds: + - actions.github.m.upbound.io/*/RunnerGroup + namespaces: + - github-config + operations: [CREATE, UPDATE] + # The provider records the identity it creates. Tenant writes may retain + # that identity, but cannot adopt or redirect a different existing group. + exclude: + any: + - subjects: + - kind: ServiceAccount + name: provider-upjet-github-* + namespace: crossplane-system + context: + - name: externalName + variable: + jmesPath: request.object.metadata.annotations."crossplane.io/external-name" + default: "" + - name: previousExternalName + variable: + jmesPath: request.oldObject.metadata.annotations."crossplane.io/external-name" + default: "" + validate: + failureAction: Enforce + allowExistingViolations: false + message: >- + Only the provider may establish the runner group's remote identity. + The tenant may leave it unset or preserve the existing identity. + deny: + conditions: + all: + - key: "{{ externalName == '' || externalName == previousExternalName }}" + operator: Equals + value: false diff --git a/k8s/bases/infrastructure/cluster-policies/kustomization.yaml b/k8s/bases/infrastructure/cluster-policies/kustomization.yaml index 3b74201d38..720ace22d7 100644 --- a/k8s/bases/infrastructure/cluster-policies/kustomization.yaml +++ b/k8s/bases/infrastructure/cluster-policies/kustomization.yaml @@ -15,6 +15,7 @@ resources: - best-practices/propagate-reloader-to-flagger-primary.yaml - best-practices/restrict-github-team-external-identity.yaml - best-practices/restrict-github-team-management.yaml + - best-practices/restrict-github-runner-group.yaml - best-practices/restrict-homepage-service-groups.yaml - best-practices/restrict-tenant-issuer-refs.yaml - best-practices/restrict-tenant-network-policies.yaml diff --git a/k8s/providers/hetzner/infrastructure/crossplane/managed-resource-activation-policy.yaml b/k8s/providers/hetzner/infrastructure/crossplane/managed-resource-activation-policy.yaml index 5036948f1f..3c058fe501 100644 --- a/k8s/providers/hetzner/infrastructure/crossplane/managed-resource-activation-policy.yaml +++ b/k8s/providers/hetzner/infrastructure/crossplane/managed-resource-activation-policy.yaml @@ -44,3 +44,5 @@ spec: # provider-upjet-github release embedding terraform-provider-github >= 6.11.0; # the pinned v0.20.0 embeds 6.13.0. - repositorypermissions.actions.github.m.upbound.io + # Dedicated organization runner group, restricted to KSail by admission. + - runnergroups.actions.github.m.upbound.io diff --git a/scripts/tests/test-verify-ksail-runner-group-capability.sh b/scripts/tests/test-verify-ksail-runner-group-capability.sh new file mode 100755 index 0000000000..c0e68711f3 --- /dev/null +++ b/scripts/tests/test-verify-ksail-runner-group-capability.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/bin" +cat >"$scratch/bin/kubectl" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +case "$*" in + *'get crd runnergroups.actions.github.m.upbound.io'*) + [[ "$GROUP_CASE" != missing-kind ]] || exit 1 + jq -n --arg scenario "$GROUP_CASE" '{spec:{group:"actions.github.m.upbound.io",scope:"Namespaced",names:{plural:"runnergroups"},versions:[{name:"v1alpha1",served:true}]},status:{conditions:[{type:"Established",status:(if $scenario=="stale-kind" then "False" else "True" end)},{type:"NamesAccepted",status:"True"}]}}' ;; + *'get role github-config-managed-resources'*) + printf '{"rules":[{"apiGroups":["actions.github.m.upbound.io"],"resources":["repositorypermissions","runnergroups"],"verbs":["get","list","watch","create","update","patch","delete"]}]}' ;; + *'auth can-i create runnergroups.actions.github.m.upbound.io'*|*'auth can-i patch runnergroups.actions.github.m.upbound.io'*) + [[ "$GROUP_CASE" != no-tenant-grant ]] || { printf 'no\n'; exit 1; } + printf 'yes\n' ;; + *'apply --dry-run=server'*) + [[ "$*" == *'--as=system:serviceaccount:github-config:github-config'* ]] || exit 91 + input=''; while (( $# )); do if [[ "$1" == -f ]]; then input="$2"; break; fi; shift; done + [[ -n "$input" && -f "$input" ]] || exit 92 + if jq -e '.spec.forProvider.visibility=="selected" and .spec.forProvider.selectedRepositoryIds==[737584922] and (.metadata.annotations["crossplane.io/external-name"] // "")==""' "$input" >/dev/null; then exit 0; fi + [[ "$GROUP_CASE" != waived-policy ]] || exit 0 + if [[ "$GROUP_CASE" == unrelated-denial ]]; then echo 'forbidden by unrelated policy' >&2; exit 1; fi + if jq -e '(.metadata.annotations["crossplane.io/external-name"] // "")!=""' "$input" >/dev/null; then + echo 'blocked by restrict-github-runner-group/tenant-cannot-rebind-group' >&2 + else echo 'blocked by restrict-github-runner-group/ksail-only-runner-group' >&2; fi + exit 1 ;; + *) exit 93 ;; +esac +SH +chmod 700 "$scratch/bin/kubectl" +for scenario in normal missing-kind stale-kind no-tenant-grant waived-policy unrelated-denial; do + if PATH="$scratch/bin:$PATH" GROUP_CASE="$scenario" GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform \ + bash "$root/scripts/verify-ksail-runner-group-capability.sh" >"$scratch/out" 2>"$scratch/err"; then + [[ "$scenario" == normal ]] || { echo "accepted $scenario"; exit 1; } + grep -q '^PASS: KSail-only runner-group' "$scratch/out" + else [[ "$scenario" != normal ]] || { cat "$scratch/err"; exit 1; }; fi +done +printf 'PASS: live runner-group capability acceptance controls\n' diff --git a/scripts/verify-ksail-runner-group-capability.sh b/scripts/verify-ksail-runner-group-capability.sh new file mode 100755 index 0000000000..d0b7534181 --- /dev/null +++ b/scripts/verify-ksail-runner-group-capability.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Protected, metadata-only readback and server dry-run admission acceptance. +set -euo pipefail +umask 077 +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +policy="$root/k8s/providers/hetzner/infrastructure/crossplane/managed-resource-activation-policy.yaml" +fail() { printf 'Runner-group capability: FAIL at %s\n' "$1" >&2; exit 1; } +activation=$(yq -o=json '.spec.activate' "$policy" 2>/dev/null) || fail activation-source +enabled=$(jq -er 'if type=="array" and all(.[]; type=="string") then + map(select(.=="runnergroups.actions.github.m.upbound.io"))|length + else error("invalid activation policy") end' <<<"$activation" 2>/dev/null) || fail activation-source +[[ "$enabled" == 0 || "$enabled" == 1 ]] || fail activation-source +if [[ "$enabled" == 0 ]]; then + printf 'Runner-group capability inactive; no runtime access\n' + exit 0 +fi +[[ "${GITHUB_ACTIONS:-}" == true && "${GITHUB_REPOSITORY:-}" == devantler-tech/platform ]] || exit 1 +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +kc() { timeout 35s kubectl --context admin@prod --request-timeout=30s "$@"; } +kc get crd runnergroups.actions.github.m.upbound.io -o json >"$scratch/crd.json" 2>"$scratch/error" || fail kind-read +jq -e '.spec.group=="actions.github.m.upbound.io" and .spec.scope=="Namespaced" and .spec.names.plural=="runnergroups" and + any(.spec.versions[]; .name=="v1alpha1" and .served==true) and + any(.status.conditions[]; .type=="Established" and .status=="True") and + any(.status.conditions[]; .type=="NamesAccepted" and .status=="True")' "$scratch/crd.json" >/dev/null || fail kind-established +kc -n github-config get role github-config-managed-resources -o json >"$scratch/role.json" 2>"$scratch/error" || fail role-read +jq -e 'any(.rules[]; .apiGroups==["actions.github.m.upbound.io"] and + (.resources|index("runnergroups"))!=null and (.verbs|index("create"))!=null and (.verbs|index("patch"))!=null) and + all(.rules[]; (.resources|index("*"))==null)' "$scratch/role.json" >/dev/null || fail role-bounds +readonly caller=system:serviceaccount:github-config:github-config +for verb in create patch; do + answer=$(kc --as="$caller" -n github-config auth can-i "$verb" runnergroups.actions.github.m.upbound.io 2>"$scratch/error") || fail tenant-grant + [[ "$answer" == yes ]] || fail tenant-grant +done +# Server dry-run evaluates the actual CRD, tenant RBAC and installed admission +# policy, including when the separately reconciled group already exists. +yq -o=json '.' "$root/tests/restrict-github-runner-group/ksail-only/resources.yaml" >"$scratch/group.json" +kc --as="$caller" -n github-config apply --dry-run=server --server-side --field-manager=ksail-group-acceptance \ + --force-conflicts -f "$scratch/group.json" >/dev/null 2>"$scratch/error" || fail admitted-ksail-group +for scenario in all-repositories wildcard-external-name; do + yq -o=json '.' "$root/tests/restrict-github-runner-group/$scenario/resources.yaml" >"$scratch/group.json" + if kc --as="$caller" -n github-config apply --dry-run=server --server-side --field-manager=ksail-group-acceptance \ + --force-conflicts -f "$scratch/group.json" >/dev/null 2>"$scratch/error"; then fail intercepted-denial; fi + rule=ksail-only-runner-group + [[ "$scenario" != wildcard-external-name ]] || rule=tenant-cannot-rebind-group + if ! grep -Fq 'restrict-github-runner-group' "$scratch/error" || ! grep -Fq "$rule" "$scratch/error"; then + fail authenticated-denial + fi +done +printf 'PASS: KSail-only runner-group CRD, tenant RBAC and intercepted admission denials\n' diff --git a/tests/restrict-github-runner-group/all-repositories/kyverno-test.yaml b/tests/restrict-github-runner-group/all-repositories/kyverno-test.yaml new file mode 100644 index 0000000000..33e06474c3 --- /dev/null +++ b/tests/restrict-github-runner-group/all-repositories/kyverno-test.yaml @@ -0,0 +1,21 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: ksail-runner-group-all-repositories +policies: + - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +resources: + - resources.yaml +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass diff --git a/tests/restrict-github-runner-group/all-repositories/resources.yaml b/tests/restrict-github-runner-group/all-repositories/resources.yaml new file mode 100644 index 0000000000..b651bc9bcb --- /dev/null +++ b/tests/restrict-github-runner-group/all-repositories/resources.yaml @@ -0,0 +1,24 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config" + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "all", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/all-repositories/user-info.yaml b/tests/restrict-github-runner-group/all-repositories/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/all-repositories/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/all-repositories/values.yaml b/tests/restrict-github-runner-group/all-repositories/values.yaml new file mode 100644 index 0000000000..3b02a285e6 --- /dev/null +++ b/tests/restrict-github-runner-group/all-repositories/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/different-name/kyverno-test.yaml b/tests/restrict-github-runner-group/different-name/kyverno-test.yaml new file mode 100644 index 0000000000..d22f0e599f --- /dev/null +++ b/tests/restrict-github-runner-group/different-name/kyverno-test.yaml @@ -0,0 +1,21 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: ksail-runner-group-different-name +policies: + - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +resources: + - resources.yaml +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass diff --git a/tests/restrict-github-runner-group/different-name/resources.yaml b/tests/restrict-github-runner-group/different-name/resources.yaml new file mode 100644 index 0000000000..8b961b56e7 --- /dev/null +++ b/tests/restrict-github-runner-group/different-name/resources.yaml @@ -0,0 +1,24 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config" + }, + "spec": { + "forProvider": { + "name": "Default", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/different-name/user-info.yaml b/tests/restrict-github-runner-group/different-name/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/different-name/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/different-name/values.yaml b/tests/restrict-github-runner-group/different-name/values.yaml new file mode 100644 index 0000000000..3b02a285e6 --- /dev/null +++ b/tests/restrict-github-runner-group/different-name/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/empty-selection/kyverno-test.yaml b/tests/restrict-github-runner-group/empty-selection/kyverno-test.yaml new file mode 100644 index 0000000000..2767cb957e --- /dev/null +++ b/tests/restrict-github-runner-group/empty-selection/kyverno-test.yaml @@ -0,0 +1,21 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: ksail-runner-group-empty-selection +policies: + - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +resources: + - resources.yaml +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass diff --git a/tests/restrict-github-runner-group/empty-selection/resources.yaml b/tests/restrict-github-runner-group/empty-selection/resources.yaml new file mode 100644 index 0000000000..30c98b8fde --- /dev/null +++ b/tests/restrict-github-runner-group/empty-selection/resources.yaml @@ -0,0 +1,22 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config" + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/empty-selection/user-info.yaml b/tests/restrict-github-runner-group/empty-selection/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/empty-selection/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/empty-selection/values.yaml b/tests/restrict-github-runner-group/empty-selection/values.yaml new file mode 100644 index 0000000000..3b02a285e6 --- /dev/null +++ b/tests/restrict-github-runner-group/empty-selection/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/foreign-external-name/kyverno-test.yaml b/tests/restrict-github-runner-group/foreign-external-name/kyverno-test.yaml new file mode 100644 index 0000000000..0506dfb73c --- /dev/null +++ b/tests/restrict-github-runner-group/foreign-external-name/kyverno-test.yaml @@ -0,0 +1,21 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: ksail-runner-group-foreign-external-name +policies: + - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +resources: + - resources.yaml +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail diff --git a/tests/restrict-github-runner-group/foreign-external-name/resources.yaml b/tests/restrict-github-runner-group/foreign-external-name/resources.yaml new file mode 100644 index 0000000000..1f3d6368cf --- /dev/null +++ b/tests/restrict-github-runner-group/foreign-external-name/resources.yaml @@ -0,0 +1,27 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config", + "annotations": { + "crossplane.io/external-name": "1" + } + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/foreign-external-name/user-info.yaml b/tests/restrict-github-runner-group/foreign-external-name/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/foreign-external-name/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/foreign-external-name/values.yaml b/tests/restrict-github-runner-group/foreign-external-name/values.yaml new file mode 100644 index 0000000000..3b02a285e6 --- /dev/null +++ b/tests/restrict-github-runner-group/foreign-external-name/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/init-provider-override/kyverno-test.yaml b/tests/restrict-github-runner-group/init-provider-override/kyverno-test.yaml new file mode 100644 index 0000000000..4245f7554c --- /dev/null +++ b/tests/restrict-github-runner-group/init-provider-override/kyverno-test.yaml @@ -0,0 +1,21 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: ksail-runner-group-init-provider-override +policies: + - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +resources: + - resources.yaml +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass diff --git a/tests/restrict-github-runner-group/init-provider-override/resources.yaml b/tests/restrict-github-runner-group/init-provider-override/resources.yaml new file mode 100644 index 0000000000..a6c07a3b5e --- /dev/null +++ b/tests/restrict-github-runner-group/init-provider-override/resources.yaml @@ -0,0 +1,30 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config" + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + }, + "initProvider": { + "visibility": "all", + "selectedRepositoryIds": [ + 1 + ] + } + } +} diff --git a/tests/restrict-github-runner-group/init-provider-override/user-info.yaml b/tests/restrict-github-runner-group/init-provider-override/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/init-provider-override/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/init-provider-override/values.yaml b/tests/restrict-github-runner-group/init-provider-override/values.yaml new file mode 100644 index 0000000000..3b02a285e6 --- /dev/null +++ b/tests/restrict-github-runner-group/init-provider-override/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/ksail-only/kyverno-test.yaml b/tests/restrict-github-runner-group/ksail-only/kyverno-test.yaml new file mode 100644 index 0000000000..8ecc91f0f9 --- /dev/null +++ b/tests/restrict-github-runner-group/ksail-only/kyverno-test.yaml @@ -0,0 +1,21 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: ksail-runner-group-ksail-only +policies: + - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +resources: + - resources.yaml +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass diff --git a/tests/restrict-github-runner-group/ksail-only/resources.yaml b/tests/restrict-github-runner-group/ksail-only/resources.yaml new file mode 100644 index 0000000000..1b250b3341 --- /dev/null +++ b/tests/restrict-github-runner-group/ksail-only/resources.yaml @@ -0,0 +1,24 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config" + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/ksail-only/user-info.yaml b/tests/restrict-github-runner-group/ksail-only/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/ksail-only/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/ksail-only/values.yaml b/tests/restrict-github-runner-group/ksail-only/values.yaml new file mode 100644 index 0000000000..3b02a285e6 --- /dev/null +++ b/tests/restrict-github-runner-group/ksail-only/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/lookalike-provider/kyverno-test.yaml b/tests/restrict-github-runner-group/lookalike-provider/kyverno-test.yaml new file mode 100644 index 0000000000..9e576b90cd --- /dev/null +++ b/tests/restrict-github-runner-group/lookalike-provider/kyverno-test.yaml @@ -0,0 +1,18 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: {name: ksail-group-lookalike-provider} +policies: [../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml] +resources: [resources.yaml] +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail diff --git a/tests/restrict-github-runner-group/lookalike-provider/resources.yaml b/tests/restrict-github-runner-group/lookalike-provider/resources.yaml new file mode 100644 index 0000000000..2e956bf0d1 --- /dev/null +++ b/tests/restrict-github-runner-group/lookalike-provider/resources.yaml @@ -0,0 +1,27 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config", + "annotations": { + "crossplane.io/external-name": "27" + } + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/lookalike-provider/user-info.yaml b/tests/restrict-github-runner-group/lookalike-provider/user-info.yaml new file mode 100644 index 0000000000..a9c4cd9ddd --- /dev/null +++ b/tests/restrict-github-runner-group/lookalike-provider/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: caller} +userInfo: + username: system:serviceaccount:github-config:provider-upjet-github-abcdef + groups: [system:serviceaccounts, system:authenticated] diff --git a/tests/restrict-github-runner-group/lookalike-provider/values.yaml b/tests/restrict-github-runner-group/lookalike-provider/values.yaml new file mode 100644 index 0000000000..ab04cbf8be --- /dev/null +++ b/tests/restrict-github-runner-group/lookalike-provider/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: UPDATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/missing-selection/kyverno-test.yaml b/tests/restrict-github-runner-group/missing-selection/kyverno-test.yaml new file mode 100644 index 0000000000..9df906b154 --- /dev/null +++ b/tests/restrict-github-runner-group/missing-selection/kyverno-test.yaml @@ -0,0 +1,21 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: ksail-runner-group-missing-selection +policies: + - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +resources: + - resources.yaml +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass diff --git a/tests/restrict-github-runner-group/missing-selection/resources.yaml b/tests/restrict-github-runner-group/missing-selection/resources.yaml new file mode 100644 index 0000000000..4186c41f9a --- /dev/null +++ b/tests/restrict-github-runner-group/missing-selection/resources.yaml @@ -0,0 +1,21 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config" + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/missing-selection/user-info.yaml b/tests/restrict-github-runner-group/missing-selection/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/missing-selection/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/missing-selection/values.yaml b/tests/restrict-github-runner-group/missing-selection/values.yaml new file mode 100644 index 0000000000..3b02a285e6 --- /dev/null +++ b/tests/restrict-github-runner-group/missing-selection/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/other-object-name/kyverno-test.yaml b/tests/restrict-github-runner-group/other-object-name/kyverno-test.yaml new file mode 100644 index 0000000000..3b937ebdc5 --- /dev/null +++ b/tests/restrict-github-runner-group/other-object-name/kyverno-test.yaml @@ -0,0 +1,21 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: ksail-runner-group-other-object-name +policies: + - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +resources: + - resources.yaml +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/other] + kind: RunnerGroup + result: fail + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/other] + kind: RunnerGroup + result: pass diff --git a/tests/restrict-github-runner-group/other-object-name/resources.yaml b/tests/restrict-github-runner-group/other-object-name/resources.yaml new file mode 100644 index 0000000000..83a7125a13 --- /dev/null +++ b/tests/restrict-github-runner-group/other-object-name/resources.yaml @@ -0,0 +1,24 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "other", + "namespace": "github-config" + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/other-object-name/user-info.yaml b/tests/restrict-github-runner-group/other-object-name/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/other-object-name/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/other-object-name/values.yaml b/tests/restrict-github-runner-group/other-object-name/values.yaml new file mode 100644 index 0000000000..2ed21fc8ae --- /dev/null +++ b/tests/restrict-github-runner-group/other-object-name/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: other + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/other-provider/kyverno-test.yaml b/tests/restrict-github-runner-group/other-provider/kyverno-test.yaml new file mode 100644 index 0000000000..2b881acf60 --- /dev/null +++ b/tests/restrict-github-runner-group/other-provider/kyverno-test.yaml @@ -0,0 +1,21 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: ksail-runner-group-other-provider +policies: + - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +resources: + - resources.yaml +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass diff --git a/tests/restrict-github-runner-group/other-provider/resources.yaml b/tests/restrict-github-runner-group/other-provider/resources.yaml new file mode 100644 index 0000000000..c64ddb5417 --- /dev/null +++ b/tests/restrict-github-runner-group/other-provider/resources.yaml @@ -0,0 +1,24 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config" + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "other", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/other-provider/user-info.yaml b/tests/restrict-github-runner-group/other-provider/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/other-provider/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/other-provider/values.yaml b/tests/restrict-github-runner-group/other-provider/values.yaml new file mode 100644 index 0000000000..3b02a285e6 --- /dev/null +++ b/tests/restrict-github-runner-group/other-provider/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/other-repository/kyverno-test.yaml b/tests/restrict-github-runner-group/other-repository/kyverno-test.yaml new file mode 100644 index 0000000000..af012afc17 --- /dev/null +++ b/tests/restrict-github-runner-group/other-repository/kyverno-test.yaml @@ -0,0 +1,21 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: ksail-runner-group-other-repository +policies: + - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +resources: + - resources.yaml +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass diff --git a/tests/restrict-github-runner-group/other-repository/resources.yaml b/tests/restrict-github-runner-group/other-repository/resources.yaml new file mode 100644 index 0000000000..a89b31f760 --- /dev/null +++ b/tests/restrict-github-runner-group/other-repository/resources.yaml @@ -0,0 +1,24 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config" + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 1 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/other-repository/user-info.yaml b/tests/restrict-github-runner-group/other-repository/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/other-repository/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/other-repository/values.yaml b/tests/restrict-github-runner-group/other-repository/values.yaml new file mode 100644 index 0000000000..3b02a285e6 --- /dev/null +++ b/tests/restrict-github-runner-group/other-repository/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/provider-records-created-group/kyverno-test.yaml b/tests/restrict-github-runner-group/provider-records-created-group/kyverno-test.yaml new file mode 100644 index 0000000000..ab4d26ca55 --- /dev/null +++ b/tests/restrict-github-runner-group/provider-records-created-group/kyverno-test.yaml @@ -0,0 +1,18 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: {name: ksail-group-provider-records-created-group} +policies: [../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml] +resources: [resources.yaml] +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: skip diff --git a/tests/restrict-github-runner-group/provider-records-created-group/resources.yaml b/tests/restrict-github-runner-group/provider-records-created-group/resources.yaml new file mode 100644 index 0000000000..2e956bf0d1 --- /dev/null +++ b/tests/restrict-github-runner-group/provider-records-created-group/resources.yaml @@ -0,0 +1,27 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config", + "annotations": { + "crossplane.io/external-name": "27" + } + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/provider-records-created-group/user-info.yaml b/tests/restrict-github-runner-group/provider-records-created-group/user-info.yaml new file mode 100644 index 0000000000..cabb2b4346 --- /dev/null +++ b/tests/restrict-github-runner-group/provider-records-created-group/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: caller} +userInfo: + username: system:serviceaccount:crossplane-system:provider-upjet-github-abcdef + groups: [system:serviceaccounts, system:authenticated] diff --git a/tests/restrict-github-runner-group/provider-records-created-group/values.yaml b/tests/restrict-github-runner-group/provider-records-created-group/values.yaml new file mode 100644 index 0000000000..ab04cbf8be --- /dev/null +++ b/tests/restrict-github-runner-group/provider-records-created-group/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: UPDATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/second-repository/kyverno-test.yaml b/tests/restrict-github-runner-group/second-repository/kyverno-test.yaml new file mode 100644 index 0000000000..b8051d1f93 --- /dev/null +++ b/tests/restrict-github-runner-group/second-repository/kyverno-test.yaml @@ -0,0 +1,21 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: ksail-runner-group-second-repository +policies: + - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +resources: + - resources.yaml +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass diff --git a/tests/restrict-github-runner-group/second-repository/resources.yaml b/tests/restrict-github-runner-group/second-repository/resources.yaml new file mode 100644 index 0000000000..b2fbc6a496 --- /dev/null +++ b/tests/restrict-github-runner-group/second-repository/resources.yaml @@ -0,0 +1,25 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config" + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922, + 1 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/second-repository/user-info.yaml b/tests/restrict-github-runner-group/second-repository/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/second-repository/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/second-repository/values.yaml b/tests/restrict-github-runner-group/second-repository/values.yaml new file mode 100644 index 0000000000..3b02a285e6 --- /dev/null +++ b/tests/restrict-github-runner-group/second-repository/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/unrestricted-public-default/kyverno-test.yaml b/tests/restrict-github-runner-group/unrestricted-public-default/kyverno-test.yaml new file mode 100644 index 0000000000..3662ae2050 --- /dev/null +++ b/tests/restrict-github-runner-group/unrestricted-public-default/kyverno-test.yaml @@ -0,0 +1,21 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: ksail-runner-group-unrestricted-public-default +policies: + - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +resources: + - resources.yaml +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass diff --git a/tests/restrict-github-runner-group/unrestricted-public-default/resources.yaml b/tests/restrict-github-runner-group/unrestricted-public-default/resources.yaml new file mode 100644 index 0000000000..b0141489fc --- /dev/null +++ b/tests/restrict-github-runner-group/unrestricted-public-default/resources.yaml @@ -0,0 +1,21 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config" + }, + "spec": { + "forProvider": { + "name": "Default", + "visibility": "all", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/unrestricted-public-default/user-info.yaml b/tests/restrict-github-runner-group/unrestricted-public-default/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/unrestricted-public-default/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/unrestricted-public-default/values.yaml b/tests/restrict-github-runner-group/unrestricted-public-default/values.yaml new file mode 100644 index 0000000000..3b02a285e6 --- /dev/null +++ b/tests/restrict-github-runner-group/unrestricted-public-default/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/update-rebound/kyverno-test.yaml b/tests/restrict-github-runner-group/update-rebound/kyverno-test.yaml new file mode 100644 index 0000000000..7021e6e92e --- /dev/null +++ b/tests/restrict-github-runner-group/update-rebound/kyverno-test.yaml @@ -0,0 +1,18 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: {name: ksail-group-update-rebound} +policies: [../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml] +resources: [resources.yaml] +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail diff --git a/tests/restrict-github-runner-group/update-rebound/resources.yaml b/tests/restrict-github-runner-group/update-rebound/resources.yaml new file mode 100644 index 0000000000..2e956bf0d1 --- /dev/null +++ b/tests/restrict-github-runner-group/update-rebound/resources.yaml @@ -0,0 +1,27 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config", + "annotations": { + "crossplane.io/external-name": "27" + } + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/update-rebound/user-info.yaml b/tests/restrict-github-runner-group/update-rebound/user-info.yaml new file mode 100644 index 0000000000..52c0c6a8e8 --- /dev/null +++ b/tests/restrict-github-runner-group/update-rebound/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: caller} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:authenticated] diff --git a/tests/restrict-github-runner-group/update-rebound/values.yaml b/tests/restrict-github-runner-group/update-rebound/values.yaml new file mode 100644 index 0000000000..915744a3ad --- /dev/null +++ b/tests/restrict-github-runner-group/update-rebound/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: UPDATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "1" diff --git a/tests/restrict-github-runner-group/update-unchanged/kyverno-test.yaml b/tests/restrict-github-runner-group/update-unchanged/kyverno-test.yaml new file mode 100644 index 0000000000..bb5d12f74c --- /dev/null +++ b/tests/restrict-github-runner-group/update-unchanged/kyverno-test.yaml @@ -0,0 +1,18 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: {name: ksail-group-update-unchanged} +policies: [../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml] +resources: [resources.yaml] +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass diff --git a/tests/restrict-github-runner-group/update-unchanged/resources.yaml b/tests/restrict-github-runner-group/update-unchanged/resources.yaml new file mode 100644 index 0000000000..2e956bf0d1 --- /dev/null +++ b/tests/restrict-github-runner-group/update-unchanged/resources.yaml @@ -0,0 +1,27 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config", + "annotations": { + "crossplane.io/external-name": "27" + } + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/update-unchanged/user-info.yaml b/tests/restrict-github-runner-group/update-unchanged/user-info.yaml new file mode 100644 index 0000000000..52c0c6a8e8 --- /dev/null +++ b/tests/restrict-github-runner-group/update-unchanged/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: caller} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:authenticated] diff --git a/tests/restrict-github-runner-group/update-unchanged/values.yaml b/tests/restrict-github-runner-group/update-unchanged/values.yaml new file mode 100644 index 0000000000..190c83925f --- /dev/null +++ b/tests/restrict-github-runner-group/update-unchanged/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: UPDATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "27" diff --git a/tests/restrict-github-runner-group/wildcard-external-name/kyverno-test.yaml b/tests/restrict-github-runner-group/wildcard-external-name/kyverno-test.yaml new file mode 100644 index 0000000000..998d604f01 --- /dev/null +++ b/tests/restrict-github-runner-group/wildcard-external-name/kyverno-test.yaml @@ -0,0 +1,21 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: ksail-runner-group-wildcard-external-name +policies: + - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +resources: + - resources.yaml +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail diff --git a/tests/restrict-github-runner-group/wildcard-external-name/resources.yaml b/tests/restrict-github-runner-group/wildcard-external-name/resources.yaml new file mode 100644 index 0000000000..c5db6acb0a --- /dev/null +++ b/tests/restrict-github-runner-group/wildcard-external-name/resources.yaml @@ -0,0 +1,27 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config", + "annotations": { + "crossplane.io/external-name": "*" + } + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/wildcard-external-name/user-info.yaml b/tests/restrict-github-runner-group/wildcard-external-name/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/wildcard-external-name/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/wildcard-external-name/values.yaml b/tests/restrict-github-runner-group/wildcard-external-name/values.yaml new file mode 100644 index 0000000000..3b02a285e6 --- /dev/null +++ b/tests/restrict-github-runner-group/wildcard-external-name/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/wildcard-name/kyverno-test.yaml b/tests/restrict-github-runner-group/wildcard-name/kyverno-test.yaml new file mode 100644 index 0000000000..a2b3851e59 --- /dev/null +++ b/tests/restrict-github-runner-group/wildcard-name/kyverno-test.yaml @@ -0,0 +1,21 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: ksail-runner-group-wildcard-name +policies: + - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +resources: + - resources.yaml +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass diff --git a/tests/restrict-github-runner-group/wildcard-name/resources.yaml b/tests/restrict-github-runner-group/wildcard-name/resources.yaml new file mode 100644 index 0000000000..1b24ff0b5b --- /dev/null +++ b/tests/restrict-github-runner-group/wildcard-name/resources.yaml @@ -0,0 +1,24 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config" + }, + "spec": { + "forProvider": { + "name": "*", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/wildcard-name/user-info.yaml b/tests/restrict-github-runner-group/wildcard-name/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/wildcard-name/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/wildcard-name/values.yaml b/tests/restrict-github-runner-group/wildcard-name/values.yaml new file mode 100644 index 0000000000..3b02a285e6 --- /dev/null +++ b/tests/restrict-github-runner-group/wildcard-name/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" From c58577475ebffd2d618c8e6856a37b942923cd33 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 10:35:04 +0200 Subject: [PATCH 20/62] fix(arc): require the namespaced runner-group provider --- .../restrict-github-runner-group.yaml | 2 +- .../cluster-provider-kind/kyverno-test.yaml | 18 ++++++++++++++ .../cluster-provider-kind/resources.yaml | 24 +++++++++++++++++++ .../cluster-provider-kind/user-info.yaml | 6 +++++ .../cluster-provider-kind/values.yaml | 14 +++++++++++ .../missing-provider-kind/kyverno-test.yaml | 18 ++++++++++++++ .../missing-provider-kind/resources.yaml | 23 ++++++++++++++++++ .../missing-provider-kind/user-info.yaml | 6 +++++ .../missing-provider-kind/values.yaml | 14 +++++++++++ 9 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 tests/restrict-github-runner-group/cluster-provider-kind/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/cluster-provider-kind/resources.yaml create mode 100644 tests/restrict-github-runner-group/cluster-provider-kind/user-info.yaml create mode 100644 tests/restrict-github-runner-group/cluster-provider-kind/values.yaml create mode 100644 tests/restrict-github-runner-group/missing-provider-kind/kyverno-test.yaml create mode 100644 tests/restrict-github-runner-group/missing-provider-kind/resources.yaml create mode 100644 tests/restrict-github-runner-group/missing-provider-kind/user-info.yaml create mode 100644 tests/restrict-github-runner-group/missing-provider-kind/values.yaml diff --git a/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml b/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml index 3ba8e71160..9b49c4f3a8 100644 --- a/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +++ b/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml @@ -35,7 +35,7 @@ spec: all: # JMESPath compares literal names and the complete numeric set; # Kyverno wildcard-aware string operators cannot express this boundary. - - key: "{{ request.object.metadata.name == 'ksail-code-quality' && request.object.spec.forProvider.name == 'ksail-code-quality' && request.object.spec.forProvider.visibility == 'selected' && request.object.spec.forProvider.selectedRepositoryIds == `[737584922]` && request.object.spec.forProvider.allowsPublicRepositories == `true` && request.object.spec.forProvider.restrictedToWorkflows == `false` && not_null(request.object.spec.forProvider.selectedWorkflows, `[]`) == `[]` && not_null(request.object.spec.initProvider, `{}`) == `{}` && request.object.spec.providerConfigRef.name == 'default' && not_null(request.object.spec.providerConfigRef.kind, 'ProviderConfig') == 'ProviderConfig' }}" + - key: "{{ request.object.metadata.name == 'ksail-code-quality' && request.object.spec.forProvider.name == 'ksail-code-quality' && request.object.spec.forProvider.visibility == 'selected' && request.object.spec.forProvider.selectedRepositoryIds == `[737584922]` && request.object.spec.forProvider.allowsPublicRepositories == `true` && request.object.spec.forProvider.restrictedToWorkflows == `false` && not_null(request.object.spec.forProvider.selectedWorkflows, `[]`) == `[]` && not_null(request.object.spec.initProvider, `{}`) == `{}` && request.object.spec.providerConfigRef.name == 'default' && request.object.spec.providerConfigRef.kind == 'ProviderConfig' }}" operator: Equals value: false - name: tenant-cannot-rebind-group diff --git a/tests/restrict-github-runner-group/cluster-provider-kind/kyverno-test.yaml b/tests/restrict-github-runner-group/cluster-provider-kind/kyverno-test.yaml new file mode 100644 index 0000000000..a0748627b6 --- /dev/null +++ b/tests/restrict-github-runner-group/cluster-provider-kind/kyverno-test.yaml @@ -0,0 +1,18 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: {name: ksail-group-cluster-provider-kind} +policies: [../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml] +resources: [resources.yaml] +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass diff --git a/tests/restrict-github-runner-group/cluster-provider-kind/resources.yaml b/tests/restrict-github-runner-group/cluster-provider-kind/resources.yaml new file mode 100644 index 0000000000..a98de81a11 --- /dev/null +++ b/tests/restrict-github-runner-group/cluster-provider-kind/resources.yaml @@ -0,0 +1,24 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config" + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default", + "kind": "ClusterProviderConfig" + } + } +} diff --git a/tests/restrict-github-runner-group/cluster-provider-kind/user-info.yaml b/tests/restrict-github-runner-group/cluster-provider-kind/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/cluster-provider-kind/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/cluster-provider-kind/values.yaml b/tests/restrict-github-runner-group/cluster-provider-kind/values.yaml new file mode 100644 index 0000000000..3b02a285e6 --- /dev/null +++ b/tests/restrict-github-runner-group/cluster-provider-kind/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/missing-provider-kind/kyverno-test.yaml b/tests/restrict-github-runner-group/missing-provider-kind/kyverno-test.yaml new file mode 100644 index 0000000000..9404e78c40 --- /dev/null +++ b/tests/restrict-github-runner-group/missing-provider-kind/kyverno-test.yaml @@ -0,0 +1,18 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: {name: ksail-group-missing-provider-kind} +policies: [../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml] +resources: [resources.yaml] +variables: values.yaml +userinfo: user-info.yaml +results: + - policy: restrict-github-runner-group + rule: ksail-only-runner-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: fail + - policy: restrict-github-runner-group + rule: tenant-cannot-rebind-group + resources: [github-config/ksail-code-quality] + kind: RunnerGroup + result: pass diff --git a/tests/restrict-github-runner-group/missing-provider-kind/resources.yaml b/tests/restrict-github-runner-group/missing-provider-kind/resources.yaml new file mode 100644 index 0000000000..6cba09edaf --- /dev/null +++ b/tests/restrict-github-runner-group/missing-provider-kind/resources.yaml @@ -0,0 +1,23 @@ +{ + "apiVersion": "actions.github.m.upbound.io/v1alpha1", + "kind": "RunnerGroup", + "metadata": { + "name": "ksail-code-quality", + "namespace": "github-config" + }, + "spec": { + "forProvider": { + "name": "ksail-code-quality", + "visibility": "selected", + "allowsPublicRepositories": true, + "restrictedToWorkflows": false, + "selectedRepositoryIds": [ + 737584922 + ], + "selectedWorkflows": [] + }, + "providerConfigRef": { + "name": "default" + } + } +} diff --git a/tests/restrict-github-runner-group/missing-provider-kind/user-info.yaml b/tests/restrict-github-runner-group/missing-provider-kind/user-info.yaml new file mode 100644 index 0000000000..bd3387aeeb --- /dev/null +++ b/tests/restrict-github-runner-group/missing-provider-kind/user-info.yaml @@ -0,0 +1,6 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: UserInfo +metadata: {name: tenant} +userInfo: + username: system:serviceaccount:github-config:github-config + groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/missing-provider-kind/values.yaml b/tests/restrict-github-runner-group/missing-provider-kind/values.yaml new file mode 100644 index 0000000000..3b02a285e6 --- /dev/null +++ b/tests/restrict-github-runner-group/missing-provider-kind/values.yaml @@ -0,0 +1,14 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Values +metadata: {name: values} +globalValues: + request.operation: CREATE +policies: + - name: restrict-github-runner-group + resources: + - name: ksail-code-quality + values: + request.oldObject: + metadata: + annotations: + crossplane.io/external-name: "" From 807315eb4d3f17ec2977fe477f6f5e53fd79c73b Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 10:40:27 +0200 Subject: [PATCH 21/62] fix(arc): preserve the runner-group enforcement actions --- tests/policy-failure-actions.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/policy-failure-actions.json b/tests/policy-failure-actions.json index bc65a4bbbf..3f969c6032 100644 --- a/tests/policy-failure-actions.json +++ b/tests/policy-failure-actions.json @@ -5,6 +5,8 @@ "disallow-latest-tag/require-image-tag": "Audit", "disallow-latest-tag/validate-image-tag": "Audit", "restrict-github-team-external-identity/teams-external-name-is-an-approved-identity": "Enforce", + "restrict-github-runner-group/ksail-only-runner-group": "Enforce", + "restrict-github-runner-group/tenant-cannot-rebind-group": "Enforce", "restrict-github-team-management/teams-allow-listed": "Enforce", "restrict-github-team-management/teams-bind-provider-identity-to-object-name": "Enforce", "restrict-github-team-management/teams-not-nested-or-directory-linked": "Enforce", From d0f047baa04a069d1241728d463cd4a3e2549051 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 10:53:53 +0200 Subject: [PATCH 22/62] fix: approve the confined runner-group authorization surface --- scripts/validate-eks-ci-role-policy/approved-surface.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index 1acb037a86..7c0d8cd894 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -284,7 +284,7 @@ rbac.authorization.k8s.io/v1|RoleBinding|world-at-ruin|world-at-ruin-zone-trial- rbac.authorization.k8s.io/v1|Role|aws|aws-managed-resources bcbd97adfa055719ff19364d7341e4952fcb0566f1c1ba16d073b6f1ae4108b0 -rbac.authorization.k8s.io/v1|Role|github-config|github-config-managed-resources acb7434cc52dd53923f047d301c10d0e85ebfe5bb8c3ef8de67de24817e3e6b0 +rbac.authorization.k8s.io/v1|Role|github-config|github-config-managed-resources 90264d4cbeeb5b50f83b74657d924049d6c9110fa95b04072187e77ae1769f44 rbac.authorization.k8s.io/v1|Role|kubelet-serving-cert-approver|leader-election:kubelet-serving-cert-approver b73fcea5da2f0aad8552ff6a168372a6d47e9f4fb967c7df741ba89708240c73 From 2b61cd72eac780a512cfdb7a48a7ce10619fc6ca Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 10:57:31 +0200 Subject: [PATCH 23/62] fix: retain suspended ARC resources in the production inventory --- docs/operations/ksail-arc.md | 19 ++++++++++--------- .../helm-release.yaml | 4 ++-- .../controllers/kustomization.yaml | 3 +++ .../hetzner/infrastructure/kustomization.yaml | 3 +++ scripts/tests/arc-staging/staging_test.go | 19 ++++++++++++++++++- 5 files changed, 36 insertions(+), 12 deletions(-) diff --git a/docs/operations/ksail-arc.md b/docs/operations/ksail-arc.md index 55ab9e78f5..46722a5ddb 100644 --- a/docs/operations/ksail-arc.md +++ b/docs/operations/ksail-arc.md @@ -1,10 +1,11 @@ # KSail analysis runners -The ARC controller and KSail analysis pool are prepared but inactive. Neither -component is referenced by a deployment aggregate, and both HelmReleases are -suspended. Merging their definitions registers no runner, reads no App credential -and provisions no server. This is the preparation slice of #4462, not its runtime -acceptance or a resolution of KSail #7131. +Production declares the ARC controller, analysis pool and protected namespaces +in its Flux inventory, with both HelmReleases suspended. This preserves ownership +of retained resources while registration is repaired. Suspension stops Helm +reconciliation; it does not drain an installed scale set. Registration, execution +and cleanup still require the separate activation proofs below and do not resolve +KSail #7131 until managed calibration completes. The controller chart and runner-set chart use the same immutable 0.15.0 artifacts. The runner image is digest-pinned. The controller manages runner sets only in the @@ -84,19 +85,19 @@ Activation requires a separate reviewed change and all of these proofs: runner pods and their registration are removed before declaring that bug fixed. For staged validation, run `go test ./scripts/tests/arc-staging` and build both -component directories directly with `kubectl kustomize`; normal local/prod trees -must continue to contain no ARC resources. The unconditional CI guard runs on +component directories directly with `kubectl kustomize`. Local trees exclude ARC; +production retains its suspended releases and protected namespaces. The unconditional CI guard runs on pull requests and merge groups. A deliberate activation revises that guard in the same reviewed change, alongside its evidence; deleting the guard alone is not activation proof. ## Rollout and recovery -After the gates above are approved, first reference and unsuspend the controller +After the gates above are approved, first unsuspend the controller in the controller layer and verify it is healthy. That component creates both namespaces before the chart installs its namespace-scoped RBAC. The controller's network policy also covers the listener, which ARC creates in that namespace. -Only then reference the pool in +Only then unsuspend the pool in the infrastructure layer, where its external secret store already exists. Keep the unique analysis label out of ordinary build/test/provider workflows. diff --git a/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml b/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml index df7e6f2e55..7f74316c4e 100644 --- a/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml +++ b/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml @@ -1,5 +1,5 @@ -# Staging only: neither this component nor the runner pool is referenced by an -# aggregate. Activation is a separate reviewed change; see docs/operations/ksail-arc.md. +# The production inventory retains this suspended release and both protected +# namespaces. Activation is a separate reviewed change; see docs/operations/ksail-arc.md. apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml index f7c4269891..a47f5cfc03 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml @@ -8,6 +8,9 @@ resources: # local/CI only): no VirtualMachines/DataVolumes exist on Hetzner and # virt-handler crash-looped (1000+ restarts) with zero VMs. - ../../../../bases/infrastructure/controllers/ + # Retain the protected ARC namespaces and release in Flux inventory while + # registration is repaired. The declared HelmRelease remains suspended. + - ../../../../bases/infrastructure/controllers/actions-runner-controller/ # Prod-only: a minimal Alertmanager scoped to Kubescape runtime-detection # alerts. The Headlamp Kubescape plugin's "Runtime Detection > Alerts" tab # reads ONLY from a Prometheus Alertmanager (GET /api/v2/alerts), which the diff --git a/k8s/providers/hetzner/infrastructure/kustomization.yaml b/k8s/providers/hetzner/infrastructure/kustomization.yaml index 9ec781a47c..dc08e99bb7 100644 --- a/k8s/providers/hetzner/infrastructure/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/kustomization.yaml @@ -3,6 +3,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../../../bases/infrastructure/ + # Re-adopt the retained release without activating a runner. Suspension does + # not drain an installed scale set; runtime acceptance remains a separate gate. + - ../../../bases/infrastructure/ksail-analysis-runners/ # Prod-only audit-log forwarder. It consumes the `coroot-api-key` generated # from the Hetzner Coroot CR patch, so it belongs in this infrastructure # layer alongside that CR rather than in the earlier wait-gated controllers diff --git a/scripts/tests/arc-staging/staging_test.go b/scripts/tests/arc-staging/staging_test.go index 19e7161b80..1bc4ebd3cb 100644 --- a/scripts/tests/arc-staging/staging_test.go +++ b/scripts/tests/arc-staging/staging_test.go @@ -189,6 +189,11 @@ func TestStagingGuardRunsUnconditionallyOnPullRequestsAndMergeGroups(t *testing. } func TestNoDeploymentAggregateActivatesARC(t *testing.T) { + allowed := map[string]string{ + "k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml": "../../../../bases/infrastructure/controllers/actions-runner-controller/", + "k8s/providers/hetzner/infrastructure/kustomization.yaml": "../../../bases/infrastructure/ksail-analysis-runners/", + } + found := map[string]bool{} err := filepath.WalkDir(filepath.Join(repoRoot, "k8s"), func(path string, entry os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr @@ -209,7 +214,14 @@ func TestNoDeploymentAggregateActivatesARC(t *testing.T) { } for _, reference := range append(document.Resources, document.Components...) { if strings.Contains(reference, "actions-runner-controller") || strings.Contains(reference, "ksail-analysis-runners") { - t.Errorf("%s activates ARC through %q", path, reference) + relative, err := filepath.Rel(repoRoot, path) + if err != nil { + return err + } + if allowed[relative] != reference || found[relative] { + t.Errorf("%s has an unexpected or duplicate ARC reference %q", path, reference) + } + found[relative] = true } } return nil @@ -217,6 +229,11 @@ func TestNoDeploymentAggregateActivatesARC(t *testing.T) { if err != nil { t.Fatal(err) } + for aggregate := range allowed { + if !found[aggregate] { + t.Errorf("%s omits the retained suspended ARC inventory", aggregate) + } + } } func TestDNSAccessIsLimitedToTheDeclaredExternalDependencies(t *testing.T) { From e5d0b8d4080dcbf286431349a1021302e1bb44f1 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 10:59:09 +0200 Subject: [PATCH 24/62] fix: register ARC in the verified KSail-only organization group --- .github/workflows/ci.yaml | 5 +- docs/operations/ksail-arc.md | 13 ++-- .../ksail-analysis-runners/helm-release.yaml | 3 +- scripts/tests/arc-staging/staging_test.go | 16 +++-- .../tests/test-verify-ksail-arc-runtime.sh | 12 +++- .../test-wait-for-ksail-arc-registration.sh | 61 +++++++++++++++++++ scripts/verify-ksail-arc-runtime.sh | 10 ++- scripts/wait-for-ksail-arc-registration.sh | 33 +++++++++- 8 files changed, 137 insertions(+), 16 deletions(-) create mode 100755 scripts/tests/test-wait-for-ksail-arc-registration.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4fc2cd5aa0..e70871b507 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -147,7 +147,10 @@ jobs: run: go test ./scripts/analyze-cri-logs - name: 🛑 Validate bounded ARC activation boundaries - run: go test ./scripts/tests/arc-staging + run: | + go test ./scripts/tests/arc-staging + shellcheck scripts/tests/test-wait-for-ksail-arc-registration.sh + bash scripts/tests/test-wait-for-ksail-arc-registration.sh - name: 🧾 Validate published ARC metrics bytes run: go test ./scripts/generate-ksail-arc-job-metrics diff --git a/docs/operations/ksail-arc.md b/docs/operations/ksail-arc.md index c4e0f48723..0fa6bb360d 100644 --- a/docs/operations/ksail-arc.md +++ b/docs/operations/ksail-arc.md @@ -48,9 +48,14 @@ routing managed analysis: `arc-ksail-app` credential-reader ServiceAccount. The OpenBao role grants read on that exact credential path; the shared ESO role keeps its existing access. Job runners use the chart's separate no-permission identity. Verify the - current installation covers KSail and grants repository - Administration read/write before activation. The pool's registration URL - remains KSail-only; the existing App installation also serves other platform + current installation grants organization self-hosted runners read/write. + The GitHub tenant declares the dedicated `ksail-code-quality` organization + runner group with selected visibility and KSail as its sole repository. + Activation requires the native provider's current-generation Ready and + Synced observations to confirm that exact scope, remote organization and + group identity. ARC registers at `https://github.com/devantler-tech` with + the explicit `ksail-code-quality` group; its actual registration annotations + must confirm that group. The existing App also serves other platform consumers. No new App, credential copy, key rotation or permission expansion is required. Do not print, check in or mount the private key in a job runner. 2. Use the declared `autoscale-ksail-analysis` CX53 pool for isolated analysis @@ -116,7 +121,7 @@ routing managed analysis: `enable_arc=true`. Join that actual registered job to its restricted runner Pod, signed image, allowed network path and cleanup before changing managed Code Quality. The image's version command and the platform probe do not prove - an authenticated runner job. Read the repository-assigned registration, + an authenticated runner job. Read the registration in the verified KSail-only group, then change managed Code Quality to the verified `ksail-code-quality` label and read configuration back. Preserve the other setup fields. Check the required tools, GitHub proxy, network allowlist and non-root installation diff --git a/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml b/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml index ec8125c127..d41ce451ee 100644 --- a/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml +++ b/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml @@ -19,8 +19,9 @@ spec: # ARC 0.15's optional tuning object is absent from the public CRD catalog. # null is explicitly supported by the pinned chart's values schema. listenerConfig: null - githubConfigUrl: https://github.com/devantler-tech/ksail + githubConfigUrl: https://github.com/devantler-tech githubConfigSecret: arc-ksail-app + runnerGroup: ksail-code-quality runnerScaleSetName: ksail-code-quality minRunners: 0 maxRunners: 1 diff --git a/scripts/tests/arc-staging/staging_test.go b/scripts/tests/arc-staging/staging_test.go index a72e8ca8c6..383cbfcc7b 100644 --- a/scripts/tests/arc-staging/staging_test.go +++ b/scripts/tests/arc-staging/staging_test.go @@ -64,10 +64,11 @@ func TestReleasesUsePinnedCharts(t *testing.T) { } } -func TestRepositoryScopedPoolCannotCreateUnboundedOrPrivilegedRunners(t *testing.T) { +func TestKSailOnlyOrganizationGroupCannotCreateUnboundedOrPrivilegedRunners(t *testing.T) { release := readYAML(t, "k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml") values := field(t, release, "spec", "values") - equal(t, field(t, values, "githubConfigUrl"), "https://github.com/devantler-tech/ksail") + equal(t, field(t, values, "githubConfigUrl"), "https://github.com/devantler-tech") + equal(t, field(t, values, "runnerGroup"), "ksail-code-quality") equal(t, field(t, values, "githubConfigSecret"), "arc-ksail-app") equal(t, field(t, values, "runnerScaleSetName"), "ksail-code-quality") equal(t, field(t, values, "minRunners"), 0) @@ -178,11 +179,14 @@ func TestStagingGuardRunsUnconditionallyOnPullRequestsAndMergeGroups(t *testing. } for _, step := range field(t, changes, "steps").([]any) { mapping := step.(map[string]any) - if mapping["run"] == "go test ./scripts/tests/arc-staging" { - if _, conditional := mapping["if"]; conditional { - t.Fatal("staging guard cannot be conditional") + run, _ := mapping["run"].(string) + for _, line := range strings.Split(run, "\n") { + if strings.TrimSpace(line) == "go test ./scripts/tests/arc-staging" { + if _, conditional := mapping["if"]; conditional { + t.Fatal("staging guard cannot be conditional") + } + return } - return } } t.Fatal("missing unconditional staging guard") diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh index 9df94a0f17..c5703e0963 100755 --- a/scripts/tests/test-verify-ksail-arc-runtime.sh +++ b/scripts/tests/test-verify-ksail-arc-runtime.sh @@ -31,7 +31,9 @@ image="$image" yq -i '.spec.values.template.spec.containers[0].image=strenv(imag .spec.values.template.spec.initContainers[0].image=strenv(image) | .spec.suspend=false' \ "$scratch/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml" yq -o=json '.spec.values' "$scratch/k8s/bases/infrastructure/ksail-analysis-runners/helm-release.yaml" \ - | jq '{metadata:{generation:1,annotations:{"runner-scale-set-id":"1"}}, + | jq '{metadata:{generation:1,annotations:{"runner-scale-set-id":"1", + "actions.github.com/runner-group-name":"ksail-code-quality", + "actions.github.com/runner-scale-set-name":"ksail-code-quality"}}, status:{phase:"Running",observedGeneration:1},spec:.} | .spec.template.spec.serviceAccountName="ksail-code-quality-gha-rs-no-permission" | .spec.template.spec.volumes[2].configMap.name="ksail-arc-job-metrics-abc123xyz4"' >"$scratch/ars" @@ -120,6 +122,14 @@ case "$args" in fi jq -n --arg revision "latest@$digest" '{metadata:{generation:1},status:{observedGeneration:1, lastAppliedRevision:$revision,conditions:[{type:"Ready",status:"True"}]}}' ;; + *'get runnergroups.actions.github.m.upbound.io ksail-code-quality '*) + jq -cn '{apiVersion:"actions.github.m.upbound.io/v1alpha1",kind:"RunnerGroup", + metadata:{name:"ksail-code-quality",namespace:"github-config",uid:"group-uid",generation:2,annotations:{"crossplane.io/external-name":"27"}}, + spec:{providerConfigRef:{name:"default",kind:"ProviderConfig"},forProvider:{name:"ksail-code-quality",visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:false,selectedRepositoryIds:[737584922],selectedWorkflows:[]}}, + status:{atProvider:{id:"27",name:"ksail-code-quality",default:false,inherited:false,visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:false,selectedRepositoryIds:[737584922],selectedWorkflows:[], + runnersUrl:"https://api.github.com/orgs/devantler-tech/actions/runner-groups/27/runners", + selectedRepositoriesUrl:"https://api.github.com/orgs/devantler-tech/actions/runner-groups/27/repositories"}, + conditions:[{type:"Ready",status:"True",observedGeneration:2},{type:"Synced",status:"True",observedGeneration:2}]}}' ;; *'get autoscalingrunnerset.actions.github.com '*) count=0 [[ ! -e "$ARC_TEST_ROOT/registration-reads" ]] || count=$(cat "$ARC_TEST_ROOT/registration-reads") diff --git a/scripts/tests/test-wait-for-ksail-arc-registration.sh b/scripts/tests/test-wait-for-ksail-arc-registration.sh new file mode 100755 index 0000000000..0d50a0179e --- /dev/null +++ b/scripts/tests/test-wait-for-ksail-arc-registration.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/bin" +cat >"$scratch/bin/timeout" <<'SH' +#!/usr/bin/env bash +shift +exec "$@" +SH +cat >"$scratch/bin/sleep" <<'SH' +#!/usr/bin/env bash +exit 124 +SH +cat >"$scratch/bin/kubectl" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +[[ "$*" == *'--context admin@prod'* ]] || exit 92 +case "$*" in + *'get kustomization '*) + jq -cn --arg revision "latest@$GROUP_DIGEST" '{metadata:{generation:1},status:{observedGeneration:1,lastAppliedRevision:$revision,conditions:[{type:"Ready",status:"True"}]}}' ;; + *'get runnergroups.actions.github.m.upbound.io ksail-code-quality '*) + [[ "$GROUP_CASE" != unreadable-group ]] || exit 1 + jq -cn --arg scenario "$GROUP_CASE" '{ + apiVersion:"actions.github.m.upbound.io/v1alpha1",kind:"RunnerGroup", + metadata:{name:"ksail-code-quality",namespace:"github-config",uid:"00000000-0000-0000-0000-000000000002",generation:2,annotations:{"crossplane.io/external-name":"27"}}, + spec:{providerConfigRef:{name:"default",kind:"ProviderConfig"},forProvider:{name:"ksail-code-quality",visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:false,selectedRepositoryIds:[737584922],selectedWorkflows:[]}}, + status:{atProvider:{id:"27",name:"ksail-code-quality",default:false,inherited:false,visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:false,selectedRepositoryIds:[737584922],selectedWorkflows:[], + runnersUrl:"https://api.github.com/orgs/devantler-tech/actions/runner-groups/27/runners", + selectedRepositoriesUrl:"https://api.github.com/orgs/devantler-tech/actions/runner-groups/27/repositories"}, + conditions:[{type:"Ready",status:"True",observedGeneration:2},{type:"Synced",status:"True",observedGeneration:2}]} + } | + if $scenario=="all-repositories" then .status.atProvider.visibility="all" + elif $scenario=="extra-repository" then .status.atProvider.selectedRepositoryIds+=[1] + elif $scenario=="default-group" then .status.atProvider.default=true + elif $scenario=="inherited-group" then .status.atProvider.inherited=true + elif $scenario=="wrong-remote-org" then .status.atProvider.runnersUrl="https://api.github.com/orgs/other/actions/runner-groups/27/runners" + elif $scenario=="stale-observation" then .status.conditions[1].observedGeneration=1 + elif $scenario=="missing-observation" then del(.status.conditions[1].observedGeneration) + elif $scenario=="unbound-id" then .metadata.annotations["crossplane.io/external-name"]="1" + else . end' ;; + *'get autoscalingrunnerset.actions.github.com ksail-code-quality '*) + jq -cn --arg scenario "$GROUP_CASE" '{metadata:{generation:1,annotations:{"runner-scale-set-id":"12","actions.github.com/runner-group-name":"ksail-code-quality","actions.github.com/runner-scale-set-name":"ksail-code-quality"}}, + spec:{githubConfigUrl:"https://github.com/devantler-tech",runnerGroup:"ksail-code-quality",runnerScaleSetName:"ksail-code-quality"}, + status:{phase:"Running",observedGeneration:1}} | + if $scenario=="wrong-registered-group" then .metadata.annotations["actions.github.com/runner-group-name"]="Default" else . end' ;; + *) exit 93 ;; +esac +SH +chmod 700 "$scratch/bin/"* +for scenario in normal all-repositories extra-repository default-group inherited-group wrong-remote-org \ + stale-observation missing-observation unbound-id unreadable-group wrong-registered-group; do + if PATH="$scratch/bin:$PATH" GROUP_DIGEST="sha256:$(printf 'a%.0s' {1..64})" GROUP_CASE="$scenario" \ + GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform \ + bash "$root/scripts/wait-for-ksail-arc-registration.sh" "sha256:$(printf 'a%.0s' {1..64})" \ + >"$scratch/out" 2>"$scratch/err"; then + [[ "$scenario" == normal ]] || { echo "accepted $scenario"; exit 1; } + else [[ "$scenario" != normal ]] || { cat "$scratch/err"; exit 1; }; fi +done +printf 'PASS: organization registration requires an observed KSail-only group\n' diff --git a/scripts/verify-ksail-arc-runtime.sh b/scripts/verify-ksail-arc-runtime.sh index 80c15a75c1..00f8531c20 100755 --- a/scripts/verify-ksail-arc-runtime.sh +++ b/scripts/verify-ksail-arc-runtime.sh @@ -109,7 +109,10 @@ stage=deployed-revision quiet bash scripts/wait-for-platform-flux-revision.sh "$PLATFORM_MANIFEST_DIGEST" # Helm readiness precedes asynchronous ARC registration. Await the full # revision/generation join before taking the immutable acceptance snapshots. -quiet timeout 660s bash scripts/wait-for-ksail-arc-registration.sh "$PLATFORM_MANIFEST_DIGEST" +stage=registration-convergence +if ! quiet timeout 660s bash scripts/wait-for-ksail-arc-registration.sh "$PLATFORM_MANIFEST_DIGEST"; then + fail registration-convergence +fi for layer in infrastructure apps; do kc -n flux-system get kustomization "$layer" -o json >"$scratch/layer.json" jq -e --arg revision "latest@$PLATFORM_MANIFEST_DIGEST" \ @@ -123,7 +126,10 @@ jq -e ' .spec.template.spec.volumes[2].configMap.name as $metrics | .status.phase == "Running" and .status.observedGeneration == .metadata.generation and ((.metadata.annotations["runner-scale-set-id"] | tonumber) > 0) and - .spec.githubConfigUrl == "https://github.com/devantler-tech/ksail" and + .spec.githubConfigUrl == "https://github.com/devantler-tech" and + .spec.runnerGroup == "ksail-code-quality" and + .metadata.annotations["actions.github.com/runner-group-name"] == "ksail-code-quality" and + .metadata.annotations["actions.github.com/runner-scale-set-name"] == "ksail-code-quality" and .spec.githubConfigSecret == "arc-ksail-app" and .spec.runnerScaleSetName == "ksail-code-quality" and .spec.minRunners == 0 and .spec.maxRunners == 1 and .spec.template.spec.serviceAccountName == "ksail-code-quality-gha-rs-no-permission" and diff --git a/scripts/wait-for-ksail-arc-registration.sh b/scripts/wait-for-ksail-arc-registration.sh index 6f27cd7cdf..d72d08f9c9 100755 --- a/scripts/wait-for-ksail-arc-registration.sh +++ b/scripts/wait-for-ksail-arc-registration.sh @@ -19,10 +19,41 @@ while ((SECONDS < deadline)); do ready=false fi done + # The provider observation comes from the existing organization's App. + # Desired selection alone cannot establish the remote access boundary. + if ! state=$(kc -n github-config get runnergroups.actions.github.m.upbound.io ksail-code-quality -o json) || + ! jq -e ' + .metadata.generation as $generation | + .status.atProvider.id as $id | + .apiVersion == "actions.github.m.upbound.io/v1alpha1" and .kind == "RunnerGroup" and + .metadata.name == "ksail-code-quality" and .metadata.namespace == "github-config" and + (.metadata.uid | type == "string" and length > 0) and .metadata.deletionTimestamp == null and + ($generation | type == "number" and . > 0) and + ($id | type == "string" and test("^[1-9][0-9]{0,18}$")) and + .metadata.annotations["crossplane.io/external-name"] == $id and + .spec.providerConfigRef == {"name":"default","kind":"ProviderConfig"} and + .spec.forProvider.name == "ksail-code-quality" and .spec.forProvider.visibility == "selected" and + .spec.forProvider.selectedRepositoryIds == [737584922] and + .spec.forProvider.allowsPublicRepositories == true and + .status.atProvider.name == "ksail-code-quality" and .status.atProvider.visibility == "selected" and + .status.atProvider.selectedRepositoryIds == [737584922] and + .status.atProvider.allowsPublicRepositories == true and + .status.atProvider.default == false and .status.atProvider.inherited == false and + .status.atProvider.runnersUrl == ("https://api.github.com/orgs/devantler-tech/actions/runner-groups/" + $id + "/runners") and + .status.atProvider.selectedRepositoriesUrl == ("https://api.github.com/orgs/devantler-tech/actions/runner-groups/" + $id + "/repositories") and + any(.status.conditions[]; .type == "Ready" and .status == "True" and .observedGeneration == $generation) and + any(.status.conditions[]; .type == "Synced" and .status == "True" and .observedGeneration == $generation) + ' <<<"$state" >/dev/null 2>&1; then + ready=false + fi if ! state=$(kc -n arc-ksail-analysis get autoscalingrunnerset.actions.github.com ksail-code-quality -o json) || ! jq -e ' .status.phase == "Running" and .status.observedGeneration == .metadata.generation and - ((.metadata.annotations["runner-scale-set-id"] | tonumber) > 0) + ((.metadata.annotations["runner-scale-set-id"] | tonumber) > 0) and + .spec.githubConfigUrl == "https://github.com/devantler-tech" and + .spec.runnerGroup == "ksail-code-quality" and .spec.runnerScaleSetName == "ksail-code-quality" and + .metadata.annotations["actions.github.com/runner-group-name"] == "ksail-code-quality" and + .metadata.annotations["actions.github.com/runner-scale-set-name"] == "ksail-code-quality" ' <<<"$state" >/dev/null 2>&1; then ready=false fi From a214de10133afd3c17faad9d8bbbcf54a5ce0808 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 11:22:50 +0200 Subject: [PATCH 25/62] fix: confine ARC recovery to suspended resource ownership --- .github/actions/deploy-prod/action.yml | 11 --- .github/workflows/ci.yaml | 5 -- docs/github-management.md | 17 ---- k8s/bases/apps/github-config/role.yaml | 1 - .../restrict-github-runner-group.yaml | 78 ------------------- .../cluster-policies/kustomization.yaml | 1 - .../managed-resource-activation-policy.yaml | 2 - ...st-verify-ksail-runner-group-capability.sh | 41 ---------- .../approved-surface.txt | 10 ++- .../verify-ksail-runner-group-capability.sh | 50 ------------ tests/policy-failure-actions.json | 2 - .../all-repositories/kyverno-test.yaml | 21 ----- .../all-repositories/resources.yaml | 24 ------ .../all-repositories/user-info.yaml | 6 -- .../all-repositories/values.yaml | 14 ---- .../cluster-provider-kind/kyverno-test.yaml | 18 ----- .../cluster-provider-kind/resources.yaml | 24 ------ .../cluster-provider-kind/user-info.yaml | 6 -- .../cluster-provider-kind/values.yaml | 14 ---- .../different-name/kyverno-test.yaml | 21 ----- .../different-name/resources.yaml | 24 ------ .../different-name/user-info.yaml | 6 -- .../different-name/values.yaml | 14 ---- .../empty-selection/kyverno-test.yaml | 21 ----- .../empty-selection/resources.yaml | 22 ------ .../empty-selection/user-info.yaml | 6 -- .../empty-selection/values.yaml | 14 ---- .../foreign-external-name/kyverno-test.yaml | 21 ----- .../foreign-external-name/resources.yaml | 27 ------- .../foreign-external-name/user-info.yaml | 6 -- .../foreign-external-name/values.yaml | 14 ---- .../init-provider-override/kyverno-test.yaml | 21 ----- .../init-provider-override/resources.yaml | 30 ------- .../init-provider-override/user-info.yaml | 6 -- .../init-provider-override/values.yaml | 14 ---- .../ksail-only/kyverno-test.yaml | 21 ----- .../ksail-only/resources.yaml | 24 ------ .../ksail-only/user-info.yaml | 6 -- .../ksail-only/values.yaml | 14 ---- .../lookalike-provider/kyverno-test.yaml | 18 ----- .../lookalike-provider/resources.yaml | 27 ------- .../lookalike-provider/user-info.yaml | 6 -- .../lookalike-provider/values.yaml | 14 ---- .../missing-provider-kind/kyverno-test.yaml | 18 ----- .../missing-provider-kind/resources.yaml | 23 ------ .../missing-provider-kind/user-info.yaml | 6 -- .../missing-provider-kind/values.yaml | 14 ---- .../missing-selection/kyverno-test.yaml | 21 ----- .../missing-selection/resources.yaml | 21 ----- .../missing-selection/user-info.yaml | 6 -- .../missing-selection/values.yaml | 14 ---- .../other-object-name/kyverno-test.yaml | 21 ----- .../other-object-name/resources.yaml | 24 ------ .../other-object-name/user-info.yaml | 6 -- .../other-object-name/values.yaml | 14 ---- .../other-provider/kyverno-test.yaml | 21 ----- .../other-provider/resources.yaml | 24 ------ .../other-provider/user-info.yaml | 6 -- .../other-provider/values.yaml | 14 ---- .../other-repository/kyverno-test.yaml | 21 ----- .../other-repository/resources.yaml | 24 ------ .../other-repository/user-info.yaml | 6 -- .../other-repository/values.yaml | 14 ---- .../kyverno-test.yaml | 18 ----- .../resources.yaml | 27 ------- .../user-info.yaml | 6 -- .../values.yaml | 14 ---- .../second-repository/kyverno-test.yaml | 21 ----- .../second-repository/resources.yaml | 25 ------ .../second-repository/user-info.yaml | 6 -- .../second-repository/values.yaml | 14 ---- .../kyverno-test.yaml | 21 ----- .../resources.yaml | 21 ----- .../user-info.yaml | 6 -- .../unrestricted-public-default/values.yaml | 14 ---- .../update-rebound/kyverno-test.yaml | 18 ----- .../update-rebound/resources.yaml | 27 ------- .../update-rebound/user-info.yaml | 6 -- .../update-rebound/values.yaml | 14 ---- .../update-unchanged/kyverno-test.yaml | 18 ----- .../update-unchanged/resources.yaml | 27 ------- .../update-unchanged/user-info.yaml | 6 -- .../update-unchanged/values.yaml | 14 ---- .../wildcard-external-name/kyverno-test.yaml | 21 ----- .../wildcard-external-name/resources.yaml | 27 ------- .../wildcard-external-name/user-info.yaml | 6 -- .../wildcard-external-name/values.yaml | 14 ---- .../wildcard-name/kyverno-test.yaml | 21 ----- .../wildcard-name/resources.yaml | 24 ------ .../wildcard-name/user-info.yaml | 6 -- .../wildcard-name/values.yaml | 14 ---- 91 files changed, 9 insertions(+), 1507 deletions(-) delete mode 100644 k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml delete mode 100755 scripts/tests/test-verify-ksail-runner-group-capability.sh delete mode 100755 scripts/verify-ksail-runner-group-capability.sh delete mode 100644 tests/restrict-github-runner-group/all-repositories/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/all-repositories/resources.yaml delete mode 100644 tests/restrict-github-runner-group/all-repositories/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/all-repositories/values.yaml delete mode 100644 tests/restrict-github-runner-group/cluster-provider-kind/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/cluster-provider-kind/resources.yaml delete mode 100644 tests/restrict-github-runner-group/cluster-provider-kind/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/cluster-provider-kind/values.yaml delete mode 100644 tests/restrict-github-runner-group/different-name/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/different-name/resources.yaml delete mode 100644 tests/restrict-github-runner-group/different-name/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/different-name/values.yaml delete mode 100644 tests/restrict-github-runner-group/empty-selection/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/empty-selection/resources.yaml delete mode 100644 tests/restrict-github-runner-group/empty-selection/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/empty-selection/values.yaml delete mode 100644 tests/restrict-github-runner-group/foreign-external-name/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/foreign-external-name/resources.yaml delete mode 100644 tests/restrict-github-runner-group/foreign-external-name/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/foreign-external-name/values.yaml delete mode 100644 tests/restrict-github-runner-group/init-provider-override/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/init-provider-override/resources.yaml delete mode 100644 tests/restrict-github-runner-group/init-provider-override/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/init-provider-override/values.yaml delete mode 100644 tests/restrict-github-runner-group/ksail-only/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/ksail-only/resources.yaml delete mode 100644 tests/restrict-github-runner-group/ksail-only/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/ksail-only/values.yaml delete mode 100644 tests/restrict-github-runner-group/lookalike-provider/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/lookalike-provider/resources.yaml delete mode 100644 tests/restrict-github-runner-group/lookalike-provider/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/lookalike-provider/values.yaml delete mode 100644 tests/restrict-github-runner-group/missing-provider-kind/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/missing-provider-kind/resources.yaml delete mode 100644 tests/restrict-github-runner-group/missing-provider-kind/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/missing-provider-kind/values.yaml delete mode 100644 tests/restrict-github-runner-group/missing-selection/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/missing-selection/resources.yaml delete mode 100644 tests/restrict-github-runner-group/missing-selection/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/missing-selection/values.yaml delete mode 100644 tests/restrict-github-runner-group/other-object-name/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/other-object-name/resources.yaml delete mode 100644 tests/restrict-github-runner-group/other-object-name/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/other-object-name/values.yaml delete mode 100644 tests/restrict-github-runner-group/other-provider/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/other-provider/resources.yaml delete mode 100644 tests/restrict-github-runner-group/other-provider/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/other-provider/values.yaml delete mode 100644 tests/restrict-github-runner-group/other-repository/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/other-repository/resources.yaml delete mode 100644 tests/restrict-github-runner-group/other-repository/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/other-repository/values.yaml delete mode 100644 tests/restrict-github-runner-group/provider-records-created-group/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/provider-records-created-group/resources.yaml delete mode 100644 tests/restrict-github-runner-group/provider-records-created-group/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/provider-records-created-group/values.yaml delete mode 100644 tests/restrict-github-runner-group/second-repository/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/second-repository/resources.yaml delete mode 100644 tests/restrict-github-runner-group/second-repository/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/second-repository/values.yaml delete mode 100644 tests/restrict-github-runner-group/unrestricted-public-default/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/unrestricted-public-default/resources.yaml delete mode 100644 tests/restrict-github-runner-group/unrestricted-public-default/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/unrestricted-public-default/values.yaml delete mode 100644 tests/restrict-github-runner-group/update-rebound/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/update-rebound/resources.yaml delete mode 100644 tests/restrict-github-runner-group/update-rebound/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/update-rebound/values.yaml delete mode 100644 tests/restrict-github-runner-group/update-unchanged/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/update-unchanged/resources.yaml delete mode 100644 tests/restrict-github-runner-group/update-unchanged/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/update-unchanged/values.yaml delete mode 100644 tests/restrict-github-runner-group/wildcard-external-name/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/wildcard-external-name/resources.yaml delete mode 100644 tests/restrict-github-runner-group/wildcard-external-name/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/wildcard-external-name/values.yaml delete mode 100644 tests/restrict-github-runner-group/wildcard-name/kyverno-test.yaml delete mode 100644 tests/restrict-github-runner-group/wildcard-name/resources.yaml delete mode 100644 tests/restrict-github-runner-group/wildcard-name/user-info.yaml delete mode 100644 tests/restrict-github-runner-group/wildcard-name/values.yaml diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index f44704bf0e..230326a286 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -554,17 +554,6 @@ runs: shell: bash run: ./scripts/verify-gateway-routes-applied.sh - - name: Verify the KSail-only runner-group capability - if: >- - !cancelled() && - steps.wait_flux_revision.outcome == 'success' && - ( - steps.cluster_update.outcome == 'skipped' || - steps.wait_prod_api_stability.outcome == 'success' - ) - shell: bash - run: bash scripts/verify-ksail-runner-group-capability.sh - - name: Verify the deployed data product UI release # The protected deployment must prove current Apps/chart/Helm identities, # all current Pods and public endpoints before its candidate can merge. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5ab8e941d2..b42c3c6853 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -149,11 +149,6 @@ jobs: - name: 🛑 Validate inactive ARC staging boundaries run: go test ./scripts/tests/arc-staging - - name: Validate KSail runner-group capability acceptance - run: | - shellcheck scripts/verify-ksail-runner-group-capability.sh scripts/tests/test-verify-ksail-runner-group-capability.sh - bash scripts/tests/test-verify-ksail-runner-group-capability.sh - # Shared Wedding backup access is retired on this proof's word, so it must # pass only on an AccessDenied for every access, after a positive control # and against an object the shared credential showed exists. These cases diff --git a/docs/github-management.md b/docs/github-management.md index 33bbff01d9..0e919d0d5f 100644 --- a/docs/github-management.md +++ b/docs/github-management.md @@ -134,23 +134,6 @@ reconcile, then batch the rest. where an external object already exists. Each active MRD costs the apiserver ~3 MiB; activate only what is used. -## KSail analysis runner group - -The existing GitHub App's organization runner permission supports the dedicated -`ksail-code-quality` runner group. The namespaced RunnerGroup kind is explicitly -activated, and only the `github-config` tenant Role receives its managed-resource -verbs. The declaration lives in `.github` and selects only KSail repository -`737584922`; public-repository use is necessary because KSail is public, while -the one-item selection keeps every other repository excluded. - -Admission requires this exact group name, the existing `default` ProviderConfig, -selected visibility, and the KSail-only repository set. It refuses initialization -overrides and tenant changes to the group's remote identity. The protected -deployment proves the installed CRD and tenant permissions, admits the permitted -declaration using server dry-run, and observes explicit admission refusals for -organization-wide access and foreign identity adoption. ARC activation and -managed-analysis routing require separate registered-runner acceptance. - ## Safety rails - **Never `Delete`** — every managed resource omits `Delete` from diff --git a/k8s/bases/apps/github-config/role.yaml b/k8s/bases/apps/github-config/role.yaml index 1902e348b0..5d1165f997 100644 --- a/k8s/bases/apps/github-config/role.yaml +++ b/k8s/bases/apps/github-config/role.yaml @@ -75,7 +75,6 @@ rules: - actions.github.m.upbound.io resources: - repositorypermissions - - runnergroups verbs: - get - list diff --git a/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml b/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml deleted file mode 100644 index 9b49c4f3a8..0000000000 --- a/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# The GitHub tenant can manage only the KSail analysis runner group. The -# organization's existing App already grants runner management; this policy -# confines the new managed-resource surface without changing that App. -apiVersion: kyverno.io/v1 -kind: ClusterPolicy -metadata: - name: restrict-github-runner-group - annotations: - policies.kyverno.io/title: Restrict GitHub Analysis Runner Group - policies.kyverno.io/category: Security, GitHub - policies.kyverno.io/severity: high - policies.kyverno.io/subject: RunnerGroup - policies.kyverno.io/minversion: 1.13.0 -spec: - background: false - rules: - - name: ksail-only-runner-group - match: - any: - - resources: - kinds: - - actions.github.m.upbound.io/*/RunnerGroup - namespaces: - - github-config - operations: [CREATE, UPDATE] - validate: - failureAction: Enforce - allowExistingViolations: false - message: >- - The analysis runner group must be ksail-code-quality, use the existing - default ProviderConfig, and select only KSail. initProvider overrides, - default or organization-wide groups, and additional repositories are refused. - deny: - conditions: - all: - # JMESPath compares literal names and the complete numeric set; - # Kyverno wildcard-aware string operators cannot express this boundary. - - key: "{{ request.object.metadata.name == 'ksail-code-quality' && request.object.spec.forProvider.name == 'ksail-code-quality' && request.object.spec.forProvider.visibility == 'selected' && request.object.spec.forProvider.selectedRepositoryIds == `[737584922]` && request.object.spec.forProvider.allowsPublicRepositories == `true` && request.object.spec.forProvider.restrictedToWorkflows == `false` && not_null(request.object.spec.forProvider.selectedWorkflows, `[]`) == `[]` && not_null(request.object.spec.initProvider, `{}`) == `{}` && request.object.spec.providerConfigRef.name == 'default' && request.object.spec.providerConfigRef.kind == 'ProviderConfig' }}" - operator: Equals - value: false - - name: tenant-cannot-rebind-group - match: - any: - - resources: - kinds: - - actions.github.m.upbound.io/*/RunnerGroup - namespaces: - - github-config - operations: [CREATE, UPDATE] - # The provider records the identity it creates. Tenant writes may retain - # that identity, but cannot adopt or redirect a different existing group. - exclude: - any: - - subjects: - - kind: ServiceAccount - name: provider-upjet-github-* - namespace: crossplane-system - context: - - name: externalName - variable: - jmesPath: request.object.metadata.annotations."crossplane.io/external-name" - default: "" - - name: previousExternalName - variable: - jmesPath: request.oldObject.metadata.annotations."crossplane.io/external-name" - default: "" - validate: - failureAction: Enforce - allowExistingViolations: false - message: >- - Only the provider may establish the runner group's remote identity. - The tenant may leave it unset or preserve the existing identity. - deny: - conditions: - all: - - key: "{{ externalName == '' || externalName == previousExternalName }}" - operator: Equals - value: false diff --git a/k8s/bases/infrastructure/cluster-policies/kustomization.yaml b/k8s/bases/infrastructure/cluster-policies/kustomization.yaml index 720ace22d7..3b74201d38 100644 --- a/k8s/bases/infrastructure/cluster-policies/kustomization.yaml +++ b/k8s/bases/infrastructure/cluster-policies/kustomization.yaml @@ -15,7 +15,6 @@ resources: - best-practices/propagate-reloader-to-flagger-primary.yaml - best-practices/restrict-github-team-external-identity.yaml - best-practices/restrict-github-team-management.yaml - - best-practices/restrict-github-runner-group.yaml - best-practices/restrict-homepage-service-groups.yaml - best-practices/restrict-tenant-issuer-refs.yaml - best-practices/restrict-tenant-network-policies.yaml diff --git a/k8s/providers/hetzner/infrastructure/crossplane/managed-resource-activation-policy.yaml b/k8s/providers/hetzner/infrastructure/crossplane/managed-resource-activation-policy.yaml index 3c058fe501..5036948f1f 100644 --- a/k8s/providers/hetzner/infrastructure/crossplane/managed-resource-activation-policy.yaml +++ b/k8s/providers/hetzner/infrastructure/crossplane/managed-resource-activation-policy.yaml @@ -44,5 +44,3 @@ spec: # provider-upjet-github release embedding terraform-provider-github >= 6.11.0; # the pinned v0.20.0 embeds 6.13.0. - repositorypermissions.actions.github.m.upbound.io - # Dedicated organization runner group, restricted to KSail by admission. - - runnergroups.actions.github.m.upbound.io diff --git a/scripts/tests/test-verify-ksail-runner-group-capability.sh b/scripts/tests/test-verify-ksail-runner-group-capability.sh deleted file mode 100755 index c0e68711f3..0000000000 --- a/scripts/tests/test-verify-ksail-runner-group-capability.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) -scratch=$(mktemp -d) -trap 'rm -rf "$scratch"' EXIT -mkdir -p "$scratch/bin" -cat >"$scratch/bin/kubectl" <<'SH' -#!/usr/bin/env bash -set -euo pipefail -case "$*" in - *'get crd runnergroups.actions.github.m.upbound.io'*) - [[ "$GROUP_CASE" != missing-kind ]] || exit 1 - jq -n --arg scenario "$GROUP_CASE" '{spec:{group:"actions.github.m.upbound.io",scope:"Namespaced",names:{plural:"runnergroups"},versions:[{name:"v1alpha1",served:true}]},status:{conditions:[{type:"Established",status:(if $scenario=="stale-kind" then "False" else "True" end)},{type:"NamesAccepted",status:"True"}]}}' ;; - *'get role github-config-managed-resources'*) - printf '{"rules":[{"apiGroups":["actions.github.m.upbound.io"],"resources":["repositorypermissions","runnergroups"],"verbs":["get","list","watch","create","update","patch","delete"]}]}' ;; - *'auth can-i create runnergroups.actions.github.m.upbound.io'*|*'auth can-i patch runnergroups.actions.github.m.upbound.io'*) - [[ "$GROUP_CASE" != no-tenant-grant ]] || { printf 'no\n'; exit 1; } - printf 'yes\n' ;; - *'apply --dry-run=server'*) - [[ "$*" == *'--as=system:serviceaccount:github-config:github-config'* ]] || exit 91 - input=''; while (( $# )); do if [[ "$1" == -f ]]; then input="$2"; break; fi; shift; done - [[ -n "$input" && -f "$input" ]] || exit 92 - if jq -e '.spec.forProvider.visibility=="selected" and .spec.forProvider.selectedRepositoryIds==[737584922] and (.metadata.annotations["crossplane.io/external-name"] // "")==""' "$input" >/dev/null; then exit 0; fi - [[ "$GROUP_CASE" != waived-policy ]] || exit 0 - if [[ "$GROUP_CASE" == unrelated-denial ]]; then echo 'forbidden by unrelated policy' >&2; exit 1; fi - if jq -e '(.metadata.annotations["crossplane.io/external-name"] // "")!=""' "$input" >/dev/null; then - echo 'blocked by restrict-github-runner-group/tenant-cannot-rebind-group' >&2 - else echo 'blocked by restrict-github-runner-group/ksail-only-runner-group' >&2; fi - exit 1 ;; - *) exit 93 ;; -esac -SH -chmod 700 "$scratch/bin/kubectl" -for scenario in normal missing-kind stale-kind no-tenant-grant waived-policy unrelated-denial; do - if PATH="$scratch/bin:$PATH" GROUP_CASE="$scenario" GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform \ - bash "$root/scripts/verify-ksail-runner-group-capability.sh" >"$scratch/out" 2>"$scratch/err"; then - [[ "$scenario" == normal ]] || { echo "accepted $scenario"; exit 1; } - grep -q '^PASS: KSail-only runner-group' "$scratch/out" - else [[ "$scenario" != normal ]] || { cat "$scratch/err"; exit 1; }; fi -done -printf 'PASS: live runner-group capability acceptance controls\n' diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index 7c0d8cd894..e3ec6d9d1e 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -14,6 +14,10 @@ helm.toolkit.fluxcd.io/v2|HelmRelease|actual-budget|actual-budget adc009b64f2a9da1427f0c9b80f6ea2490607a8b55ceb6e139f73de92399d5da +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-ksail-analysis|ksail-analysis-runners 6d362a8f340a13e3f71293e341a762a590c3fab61eb81e2167bdc1975c67748a + +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-systems|arc-controller 655a911346c6ea07b14e8561759f735e643090a03454688186e8f31ac898cbf6 + helm.toolkit.fluxcd.io/v2|HelmRelease|backstage|backstage 5ff19be19e3525e641b0a7a22e2a7047e1fbda16ef9ca7b4fa731ce3a61e686a helm.toolkit.fluxcd.io/v2|HelmRelease|cert-manager|cert-manager 6e91f980ce0c37a8bfa4671319b170384cffd1781e4778d84f57710a97754ca1 @@ -284,7 +288,7 @@ rbac.authorization.k8s.io/v1|RoleBinding|world-at-ruin|world-at-ruin-zone-trial- rbac.authorization.k8s.io/v1|Role|aws|aws-managed-resources bcbd97adfa055719ff19364d7341e4952fcb0566f1c1ba16d073b6f1ae4108b0 -rbac.authorization.k8s.io/v1|Role|github-config|github-config-managed-resources 90264d4cbeeb5b50f83b74657d924049d6c9110fa95b04072187e77ae1769f44 +rbac.authorization.k8s.io/v1|Role|github-config|github-config-managed-resources acb7434cc52dd53923f047d301c10d0e85ebfe5bb8c3ef8de67de24817e3e6b0 rbac.authorization.k8s.io/v1|Role|kubelet-serving-cert-approver|leader-election:kubelet-serving-cert-approver b73fcea5da2f0aad8552ff6a168372a6d47e9f4fb967c7df741ba89708240c73 @@ -396,6 +400,10 @@ source.toolkit.fluxcd.io/v1|HelmRepository|vertical-pod-autoscaler|vertical-pod- source.toolkit.fluxcd.io/v1|HelmRepository|whoami|whoami 28dc88b5a5e69a086e4f7fb87be34d1b8a0cc20a13b21706aaf534a8e5d80cf8 +source.toolkit.fluxcd.io/v1|OCIRepository|arc-ksail-analysis|ksail-analysis-runners 5f89c75739fa50d998e046bc2a388ccf9c7fb4413377510e43f9f12ae4ff4c70 + +source.toolkit.fluxcd.io/v1|OCIRepository|arc-systems|arc-controller 64fb325741489bbd5ece40cf2cbf6e4aef3e87e89efea97b7b0f2ec498671fa1 + source.toolkit.fluxcd.io/v1|OCIRepository|ascoachingogvaner|ascoachingogvaner e7717c1d618e5d2dd1f3ab32d589ac06205d040e867f33db22ab9651bcb105bc source.toolkit.fluxcd.io/v1|OCIRepository|aws|aws f900dbd1860a7494381bb0fe0bd044ebedeaa1570bc9a230b7879e2b4d752163 diff --git a/scripts/verify-ksail-runner-group-capability.sh b/scripts/verify-ksail-runner-group-capability.sh deleted file mode 100755 index d0b7534181..0000000000 --- a/scripts/verify-ksail-runner-group-capability.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -# Protected, metadata-only readback and server dry-run admission acceptance. -set -euo pipefail -umask 077 -root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) -policy="$root/k8s/providers/hetzner/infrastructure/crossplane/managed-resource-activation-policy.yaml" -fail() { printf 'Runner-group capability: FAIL at %s\n' "$1" >&2; exit 1; } -activation=$(yq -o=json '.spec.activate' "$policy" 2>/dev/null) || fail activation-source -enabled=$(jq -er 'if type=="array" and all(.[]; type=="string") then - map(select(.=="runnergroups.actions.github.m.upbound.io"))|length - else error("invalid activation policy") end' <<<"$activation" 2>/dev/null) || fail activation-source -[[ "$enabled" == 0 || "$enabled" == 1 ]] || fail activation-source -if [[ "$enabled" == 0 ]]; then - printf 'Runner-group capability inactive; no runtime access\n' - exit 0 -fi -[[ "${GITHUB_ACTIONS:-}" == true && "${GITHUB_REPOSITORY:-}" == devantler-tech/platform ]] || exit 1 -scratch=$(mktemp -d) -trap 'rm -rf "$scratch"' EXIT -kc() { timeout 35s kubectl --context admin@prod --request-timeout=30s "$@"; } -kc get crd runnergroups.actions.github.m.upbound.io -o json >"$scratch/crd.json" 2>"$scratch/error" || fail kind-read -jq -e '.spec.group=="actions.github.m.upbound.io" and .spec.scope=="Namespaced" and .spec.names.plural=="runnergroups" and - any(.spec.versions[]; .name=="v1alpha1" and .served==true) and - any(.status.conditions[]; .type=="Established" and .status=="True") and - any(.status.conditions[]; .type=="NamesAccepted" and .status=="True")' "$scratch/crd.json" >/dev/null || fail kind-established -kc -n github-config get role github-config-managed-resources -o json >"$scratch/role.json" 2>"$scratch/error" || fail role-read -jq -e 'any(.rules[]; .apiGroups==["actions.github.m.upbound.io"] and - (.resources|index("runnergroups"))!=null and (.verbs|index("create"))!=null and (.verbs|index("patch"))!=null) and - all(.rules[]; (.resources|index("*"))==null)' "$scratch/role.json" >/dev/null || fail role-bounds -readonly caller=system:serviceaccount:github-config:github-config -for verb in create patch; do - answer=$(kc --as="$caller" -n github-config auth can-i "$verb" runnergroups.actions.github.m.upbound.io 2>"$scratch/error") || fail tenant-grant - [[ "$answer" == yes ]] || fail tenant-grant -done -# Server dry-run evaluates the actual CRD, tenant RBAC and installed admission -# policy, including when the separately reconciled group already exists. -yq -o=json '.' "$root/tests/restrict-github-runner-group/ksail-only/resources.yaml" >"$scratch/group.json" -kc --as="$caller" -n github-config apply --dry-run=server --server-side --field-manager=ksail-group-acceptance \ - --force-conflicts -f "$scratch/group.json" >/dev/null 2>"$scratch/error" || fail admitted-ksail-group -for scenario in all-repositories wildcard-external-name; do - yq -o=json '.' "$root/tests/restrict-github-runner-group/$scenario/resources.yaml" >"$scratch/group.json" - if kc --as="$caller" -n github-config apply --dry-run=server --server-side --field-manager=ksail-group-acceptance \ - --force-conflicts -f "$scratch/group.json" >/dev/null 2>"$scratch/error"; then fail intercepted-denial; fi - rule=ksail-only-runner-group - [[ "$scenario" != wildcard-external-name ]] || rule=tenant-cannot-rebind-group - if ! grep -Fq 'restrict-github-runner-group' "$scratch/error" || ! grep -Fq "$rule" "$scratch/error"; then - fail authenticated-denial - fi -done -printf 'PASS: KSail-only runner-group CRD, tenant RBAC and intercepted admission denials\n' diff --git a/tests/policy-failure-actions.json b/tests/policy-failure-actions.json index 3f969c6032..bc65a4bbbf 100644 --- a/tests/policy-failure-actions.json +++ b/tests/policy-failure-actions.json @@ -5,8 +5,6 @@ "disallow-latest-tag/require-image-tag": "Audit", "disallow-latest-tag/validate-image-tag": "Audit", "restrict-github-team-external-identity/teams-external-name-is-an-approved-identity": "Enforce", - "restrict-github-runner-group/ksail-only-runner-group": "Enforce", - "restrict-github-runner-group/tenant-cannot-rebind-group": "Enforce", "restrict-github-team-management/teams-allow-listed": "Enforce", "restrict-github-team-management/teams-bind-provider-identity-to-object-name": "Enforce", "restrict-github-team-management/teams-not-nested-or-directory-linked": "Enforce", diff --git a/tests/restrict-github-runner-group/all-repositories/kyverno-test.yaml b/tests/restrict-github-runner-group/all-repositories/kyverno-test.yaml deleted file mode 100644 index 33e06474c3..0000000000 --- a/tests/restrict-github-runner-group/all-repositories/kyverno-test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: - name: ksail-runner-group-all-repositories -policies: - - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml -resources: - - resources.yaml -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass diff --git a/tests/restrict-github-runner-group/all-repositories/resources.yaml b/tests/restrict-github-runner-group/all-repositories/resources.yaml deleted file mode 100644 index b651bc9bcb..0000000000 --- a/tests/restrict-github-runner-group/all-repositories/resources.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config" - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "all", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/all-repositories/user-info.yaml b/tests/restrict-github-runner-group/all-repositories/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/all-repositories/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/all-repositories/values.yaml b/tests/restrict-github-runner-group/all-repositories/values.yaml deleted file mode 100644 index 3b02a285e6..0000000000 --- a/tests/restrict-github-runner-group/all-repositories/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/cluster-provider-kind/kyverno-test.yaml b/tests/restrict-github-runner-group/cluster-provider-kind/kyverno-test.yaml deleted file mode 100644 index a0748627b6..0000000000 --- a/tests/restrict-github-runner-group/cluster-provider-kind/kyverno-test.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: {name: ksail-group-cluster-provider-kind} -policies: [../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml] -resources: [resources.yaml] -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass diff --git a/tests/restrict-github-runner-group/cluster-provider-kind/resources.yaml b/tests/restrict-github-runner-group/cluster-provider-kind/resources.yaml deleted file mode 100644 index a98de81a11..0000000000 --- a/tests/restrict-github-runner-group/cluster-provider-kind/resources.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config" - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ClusterProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/cluster-provider-kind/user-info.yaml b/tests/restrict-github-runner-group/cluster-provider-kind/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/cluster-provider-kind/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/cluster-provider-kind/values.yaml b/tests/restrict-github-runner-group/cluster-provider-kind/values.yaml deleted file mode 100644 index 3b02a285e6..0000000000 --- a/tests/restrict-github-runner-group/cluster-provider-kind/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/different-name/kyverno-test.yaml b/tests/restrict-github-runner-group/different-name/kyverno-test.yaml deleted file mode 100644 index d22f0e599f..0000000000 --- a/tests/restrict-github-runner-group/different-name/kyverno-test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: - name: ksail-runner-group-different-name -policies: - - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml -resources: - - resources.yaml -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass diff --git a/tests/restrict-github-runner-group/different-name/resources.yaml b/tests/restrict-github-runner-group/different-name/resources.yaml deleted file mode 100644 index 8b961b56e7..0000000000 --- a/tests/restrict-github-runner-group/different-name/resources.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config" - }, - "spec": { - "forProvider": { - "name": "Default", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/different-name/user-info.yaml b/tests/restrict-github-runner-group/different-name/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/different-name/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/different-name/values.yaml b/tests/restrict-github-runner-group/different-name/values.yaml deleted file mode 100644 index 3b02a285e6..0000000000 --- a/tests/restrict-github-runner-group/different-name/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/empty-selection/kyverno-test.yaml b/tests/restrict-github-runner-group/empty-selection/kyverno-test.yaml deleted file mode 100644 index 2767cb957e..0000000000 --- a/tests/restrict-github-runner-group/empty-selection/kyverno-test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: - name: ksail-runner-group-empty-selection -policies: - - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml -resources: - - resources.yaml -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass diff --git a/tests/restrict-github-runner-group/empty-selection/resources.yaml b/tests/restrict-github-runner-group/empty-selection/resources.yaml deleted file mode 100644 index 30c98b8fde..0000000000 --- a/tests/restrict-github-runner-group/empty-selection/resources.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config" - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/empty-selection/user-info.yaml b/tests/restrict-github-runner-group/empty-selection/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/empty-selection/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/empty-selection/values.yaml b/tests/restrict-github-runner-group/empty-selection/values.yaml deleted file mode 100644 index 3b02a285e6..0000000000 --- a/tests/restrict-github-runner-group/empty-selection/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/foreign-external-name/kyverno-test.yaml b/tests/restrict-github-runner-group/foreign-external-name/kyverno-test.yaml deleted file mode 100644 index 0506dfb73c..0000000000 --- a/tests/restrict-github-runner-group/foreign-external-name/kyverno-test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: - name: ksail-runner-group-foreign-external-name -policies: - - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml -resources: - - resources.yaml -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail diff --git a/tests/restrict-github-runner-group/foreign-external-name/resources.yaml b/tests/restrict-github-runner-group/foreign-external-name/resources.yaml deleted file mode 100644 index 1f3d6368cf..0000000000 --- a/tests/restrict-github-runner-group/foreign-external-name/resources.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config", - "annotations": { - "crossplane.io/external-name": "1" - } - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/foreign-external-name/user-info.yaml b/tests/restrict-github-runner-group/foreign-external-name/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/foreign-external-name/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/foreign-external-name/values.yaml b/tests/restrict-github-runner-group/foreign-external-name/values.yaml deleted file mode 100644 index 3b02a285e6..0000000000 --- a/tests/restrict-github-runner-group/foreign-external-name/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/init-provider-override/kyverno-test.yaml b/tests/restrict-github-runner-group/init-provider-override/kyverno-test.yaml deleted file mode 100644 index 4245f7554c..0000000000 --- a/tests/restrict-github-runner-group/init-provider-override/kyverno-test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: - name: ksail-runner-group-init-provider-override -policies: - - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml -resources: - - resources.yaml -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass diff --git a/tests/restrict-github-runner-group/init-provider-override/resources.yaml b/tests/restrict-github-runner-group/init-provider-override/resources.yaml deleted file mode 100644 index a6c07a3b5e..0000000000 --- a/tests/restrict-github-runner-group/init-provider-override/resources.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config" - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - }, - "initProvider": { - "visibility": "all", - "selectedRepositoryIds": [ - 1 - ] - } - } -} diff --git a/tests/restrict-github-runner-group/init-provider-override/user-info.yaml b/tests/restrict-github-runner-group/init-provider-override/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/init-provider-override/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/init-provider-override/values.yaml b/tests/restrict-github-runner-group/init-provider-override/values.yaml deleted file mode 100644 index 3b02a285e6..0000000000 --- a/tests/restrict-github-runner-group/init-provider-override/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/ksail-only/kyverno-test.yaml b/tests/restrict-github-runner-group/ksail-only/kyverno-test.yaml deleted file mode 100644 index 8ecc91f0f9..0000000000 --- a/tests/restrict-github-runner-group/ksail-only/kyverno-test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: - name: ksail-runner-group-ksail-only -policies: - - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml -resources: - - resources.yaml -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass diff --git a/tests/restrict-github-runner-group/ksail-only/resources.yaml b/tests/restrict-github-runner-group/ksail-only/resources.yaml deleted file mode 100644 index 1b250b3341..0000000000 --- a/tests/restrict-github-runner-group/ksail-only/resources.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config" - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/ksail-only/user-info.yaml b/tests/restrict-github-runner-group/ksail-only/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/ksail-only/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/ksail-only/values.yaml b/tests/restrict-github-runner-group/ksail-only/values.yaml deleted file mode 100644 index 3b02a285e6..0000000000 --- a/tests/restrict-github-runner-group/ksail-only/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/lookalike-provider/kyverno-test.yaml b/tests/restrict-github-runner-group/lookalike-provider/kyverno-test.yaml deleted file mode 100644 index 9e576b90cd..0000000000 --- a/tests/restrict-github-runner-group/lookalike-provider/kyverno-test.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: {name: ksail-group-lookalike-provider} -policies: [../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml] -resources: [resources.yaml] -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail diff --git a/tests/restrict-github-runner-group/lookalike-provider/resources.yaml b/tests/restrict-github-runner-group/lookalike-provider/resources.yaml deleted file mode 100644 index 2e956bf0d1..0000000000 --- a/tests/restrict-github-runner-group/lookalike-provider/resources.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config", - "annotations": { - "crossplane.io/external-name": "27" - } - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/lookalike-provider/user-info.yaml b/tests/restrict-github-runner-group/lookalike-provider/user-info.yaml deleted file mode 100644 index a9c4cd9ddd..0000000000 --- a/tests/restrict-github-runner-group/lookalike-provider/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: caller} -userInfo: - username: system:serviceaccount:github-config:provider-upjet-github-abcdef - groups: [system:serviceaccounts, system:authenticated] diff --git a/tests/restrict-github-runner-group/lookalike-provider/values.yaml b/tests/restrict-github-runner-group/lookalike-provider/values.yaml deleted file mode 100644 index ab04cbf8be..0000000000 --- a/tests/restrict-github-runner-group/lookalike-provider/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: UPDATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/missing-provider-kind/kyverno-test.yaml b/tests/restrict-github-runner-group/missing-provider-kind/kyverno-test.yaml deleted file mode 100644 index 9404e78c40..0000000000 --- a/tests/restrict-github-runner-group/missing-provider-kind/kyverno-test.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: {name: ksail-group-missing-provider-kind} -policies: [../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml] -resources: [resources.yaml] -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass diff --git a/tests/restrict-github-runner-group/missing-provider-kind/resources.yaml b/tests/restrict-github-runner-group/missing-provider-kind/resources.yaml deleted file mode 100644 index 6cba09edaf..0000000000 --- a/tests/restrict-github-runner-group/missing-provider-kind/resources.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config" - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default" - } - } -} diff --git a/tests/restrict-github-runner-group/missing-provider-kind/user-info.yaml b/tests/restrict-github-runner-group/missing-provider-kind/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/missing-provider-kind/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/missing-provider-kind/values.yaml b/tests/restrict-github-runner-group/missing-provider-kind/values.yaml deleted file mode 100644 index 3b02a285e6..0000000000 --- a/tests/restrict-github-runner-group/missing-provider-kind/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/missing-selection/kyverno-test.yaml b/tests/restrict-github-runner-group/missing-selection/kyverno-test.yaml deleted file mode 100644 index 9df906b154..0000000000 --- a/tests/restrict-github-runner-group/missing-selection/kyverno-test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: - name: ksail-runner-group-missing-selection -policies: - - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml -resources: - - resources.yaml -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass diff --git a/tests/restrict-github-runner-group/missing-selection/resources.yaml b/tests/restrict-github-runner-group/missing-selection/resources.yaml deleted file mode 100644 index 4186c41f9a..0000000000 --- a/tests/restrict-github-runner-group/missing-selection/resources.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config" - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/missing-selection/user-info.yaml b/tests/restrict-github-runner-group/missing-selection/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/missing-selection/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/missing-selection/values.yaml b/tests/restrict-github-runner-group/missing-selection/values.yaml deleted file mode 100644 index 3b02a285e6..0000000000 --- a/tests/restrict-github-runner-group/missing-selection/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/other-object-name/kyverno-test.yaml b/tests/restrict-github-runner-group/other-object-name/kyverno-test.yaml deleted file mode 100644 index 3b937ebdc5..0000000000 --- a/tests/restrict-github-runner-group/other-object-name/kyverno-test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: - name: ksail-runner-group-other-object-name -policies: - - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml -resources: - - resources.yaml -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/other] - kind: RunnerGroup - result: fail - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/other] - kind: RunnerGroup - result: pass diff --git a/tests/restrict-github-runner-group/other-object-name/resources.yaml b/tests/restrict-github-runner-group/other-object-name/resources.yaml deleted file mode 100644 index 83a7125a13..0000000000 --- a/tests/restrict-github-runner-group/other-object-name/resources.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "other", - "namespace": "github-config" - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/other-object-name/user-info.yaml b/tests/restrict-github-runner-group/other-object-name/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/other-object-name/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/other-object-name/values.yaml b/tests/restrict-github-runner-group/other-object-name/values.yaml deleted file mode 100644 index 2ed21fc8ae..0000000000 --- a/tests/restrict-github-runner-group/other-object-name/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: other - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/other-provider/kyverno-test.yaml b/tests/restrict-github-runner-group/other-provider/kyverno-test.yaml deleted file mode 100644 index 2b881acf60..0000000000 --- a/tests/restrict-github-runner-group/other-provider/kyverno-test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: - name: ksail-runner-group-other-provider -policies: - - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml -resources: - - resources.yaml -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass diff --git a/tests/restrict-github-runner-group/other-provider/resources.yaml b/tests/restrict-github-runner-group/other-provider/resources.yaml deleted file mode 100644 index c64ddb5417..0000000000 --- a/tests/restrict-github-runner-group/other-provider/resources.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config" - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "other", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/other-provider/user-info.yaml b/tests/restrict-github-runner-group/other-provider/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/other-provider/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/other-provider/values.yaml b/tests/restrict-github-runner-group/other-provider/values.yaml deleted file mode 100644 index 3b02a285e6..0000000000 --- a/tests/restrict-github-runner-group/other-provider/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/other-repository/kyverno-test.yaml b/tests/restrict-github-runner-group/other-repository/kyverno-test.yaml deleted file mode 100644 index af012afc17..0000000000 --- a/tests/restrict-github-runner-group/other-repository/kyverno-test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: - name: ksail-runner-group-other-repository -policies: - - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml -resources: - - resources.yaml -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass diff --git a/tests/restrict-github-runner-group/other-repository/resources.yaml b/tests/restrict-github-runner-group/other-repository/resources.yaml deleted file mode 100644 index a89b31f760..0000000000 --- a/tests/restrict-github-runner-group/other-repository/resources.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config" - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 1 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/other-repository/user-info.yaml b/tests/restrict-github-runner-group/other-repository/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/other-repository/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/other-repository/values.yaml b/tests/restrict-github-runner-group/other-repository/values.yaml deleted file mode 100644 index 3b02a285e6..0000000000 --- a/tests/restrict-github-runner-group/other-repository/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/provider-records-created-group/kyverno-test.yaml b/tests/restrict-github-runner-group/provider-records-created-group/kyverno-test.yaml deleted file mode 100644 index ab4d26ca55..0000000000 --- a/tests/restrict-github-runner-group/provider-records-created-group/kyverno-test.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: {name: ksail-group-provider-records-created-group} -policies: [../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml] -resources: [resources.yaml] -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: skip diff --git a/tests/restrict-github-runner-group/provider-records-created-group/resources.yaml b/tests/restrict-github-runner-group/provider-records-created-group/resources.yaml deleted file mode 100644 index 2e956bf0d1..0000000000 --- a/tests/restrict-github-runner-group/provider-records-created-group/resources.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config", - "annotations": { - "crossplane.io/external-name": "27" - } - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/provider-records-created-group/user-info.yaml b/tests/restrict-github-runner-group/provider-records-created-group/user-info.yaml deleted file mode 100644 index cabb2b4346..0000000000 --- a/tests/restrict-github-runner-group/provider-records-created-group/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: caller} -userInfo: - username: system:serviceaccount:crossplane-system:provider-upjet-github-abcdef - groups: [system:serviceaccounts, system:authenticated] diff --git a/tests/restrict-github-runner-group/provider-records-created-group/values.yaml b/tests/restrict-github-runner-group/provider-records-created-group/values.yaml deleted file mode 100644 index ab04cbf8be..0000000000 --- a/tests/restrict-github-runner-group/provider-records-created-group/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: UPDATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/second-repository/kyverno-test.yaml b/tests/restrict-github-runner-group/second-repository/kyverno-test.yaml deleted file mode 100644 index b8051d1f93..0000000000 --- a/tests/restrict-github-runner-group/second-repository/kyverno-test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: - name: ksail-runner-group-second-repository -policies: - - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml -resources: - - resources.yaml -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass diff --git a/tests/restrict-github-runner-group/second-repository/resources.yaml b/tests/restrict-github-runner-group/second-repository/resources.yaml deleted file mode 100644 index b2fbc6a496..0000000000 --- a/tests/restrict-github-runner-group/second-repository/resources.yaml +++ /dev/null @@ -1,25 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config" - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922, - 1 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/second-repository/user-info.yaml b/tests/restrict-github-runner-group/second-repository/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/second-repository/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/second-repository/values.yaml b/tests/restrict-github-runner-group/second-repository/values.yaml deleted file mode 100644 index 3b02a285e6..0000000000 --- a/tests/restrict-github-runner-group/second-repository/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/unrestricted-public-default/kyverno-test.yaml b/tests/restrict-github-runner-group/unrestricted-public-default/kyverno-test.yaml deleted file mode 100644 index 3662ae2050..0000000000 --- a/tests/restrict-github-runner-group/unrestricted-public-default/kyverno-test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: - name: ksail-runner-group-unrestricted-public-default -policies: - - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml -resources: - - resources.yaml -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass diff --git a/tests/restrict-github-runner-group/unrestricted-public-default/resources.yaml b/tests/restrict-github-runner-group/unrestricted-public-default/resources.yaml deleted file mode 100644 index b0141489fc..0000000000 --- a/tests/restrict-github-runner-group/unrestricted-public-default/resources.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config" - }, - "spec": { - "forProvider": { - "name": "Default", - "visibility": "all", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/unrestricted-public-default/user-info.yaml b/tests/restrict-github-runner-group/unrestricted-public-default/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/unrestricted-public-default/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/unrestricted-public-default/values.yaml b/tests/restrict-github-runner-group/unrestricted-public-default/values.yaml deleted file mode 100644 index 3b02a285e6..0000000000 --- a/tests/restrict-github-runner-group/unrestricted-public-default/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/update-rebound/kyverno-test.yaml b/tests/restrict-github-runner-group/update-rebound/kyverno-test.yaml deleted file mode 100644 index 7021e6e92e..0000000000 --- a/tests/restrict-github-runner-group/update-rebound/kyverno-test.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: {name: ksail-group-update-rebound} -policies: [../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml] -resources: [resources.yaml] -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail diff --git a/tests/restrict-github-runner-group/update-rebound/resources.yaml b/tests/restrict-github-runner-group/update-rebound/resources.yaml deleted file mode 100644 index 2e956bf0d1..0000000000 --- a/tests/restrict-github-runner-group/update-rebound/resources.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config", - "annotations": { - "crossplane.io/external-name": "27" - } - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/update-rebound/user-info.yaml b/tests/restrict-github-runner-group/update-rebound/user-info.yaml deleted file mode 100644 index 52c0c6a8e8..0000000000 --- a/tests/restrict-github-runner-group/update-rebound/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: caller} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:authenticated] diff --git a/tests/restrict-github-runner-group/update-rebound/values.yaml b/tests/restrict-github-runner-group/update-rebound/values.yaml deleted file mode 100644 index 915744a3ad..0000000000 --- a/tests/restrict-github-runner-group/update-rebound/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: UPDATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "1" diff --git a/tests/restrict-github-runner-group/update-unchanged/kyverno-test.yaml b/tests/restrict-github-runner-group/update-unchanged/kyverno-test.yaml deleted file mode 100644 index bb5d12f74c..0000000000 --- a/tests/restrict-github-runner-group/update-unchanged/kyverno-test.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: {name: ksail-group-update-unchanged} -policies: [../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml] -resources: [resources.yaml] -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass diff --git a/tests/restrict-github-runner-group/update-unchanged/resources.yaml b/tests/restrict-github-runner-group/update-unchanged/resources.yaml deleted file mode 100644 index 2e956bf0d1..0000000000 --- a/tests/restrict-github-runner-group/update-unchanged/resources.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config", - "annotations": { - "crossplane.io/external-name": "27" - } - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/update-unchanged/user-info.yaml b/tests/restrict-github-runner-group/update-unchanged/user-info.yaml deleted file mode 100644 index 52c0c6a8e8..0000000000 --- a/tests/restrict-github-runner-group/update-unchanged/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: caller} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:authenticated] diff --git a/tests/restrict-github-runner-group/update-unchanged/values.yaml b/tests/restrict-github-runner-group/update-unchanged/values.yaml deleted file mode 100644 index 190c83925f..0000000000 --- a/tests/restrict-github-runner-group/update-unchanged/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: UPDATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "27" diff --git a/tests/restrict-github-runner-group/wildcard-external-name/kyverno-test.yaml b/tests/restrict-github-runner-group/wildcard-external-name/kyverno-test.yaml deleted file mode 100644 index 998d604f01..0000000000 --- a/tests/restrict-github-runner-group/wildcard-external-name/kyverno-test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: - name: ksail-runner-group-wildcard-external-name -policies: - - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml -resources: - - resources.yaml -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail diff --git a/tests/restrict-github-runner-group/wildcard-external-name/resources.yaml b/tests/restrict-github-runner-group/wildcard-external-name/resources.yaml deleted file mode 100644 index c5db6acb0a..0000000000 --- a/tests/restrict-github-runner-group/wildcard-external-name/resources.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config", - "annotations": { - "crossplane.io/external-name": "*" - } - }, - "spec": { - "forProvider": { - "name": "ksail-code-quality", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/wildcard-external-name/user-info.yaml b/tests/restrict-github-runner-group/wildcard-external-name/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/wildcard-external-name/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/wildcard-external-name/values.yaml b/tests/restrict-github-runner-group/wildcard-external-name/values.yaml deleted file mode 100644 index 3b02a285e6..0000000000 --- a/tests/restrict-github-runner-group/wildcard-external-name/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" diff --git a/tests/restrict-github-runner-group/wildcard-name/kyverno-test.yaml b/tests/restrict-github-runner-group/wildcard-name/kyverno-test.yaml deleted file mode 100644 index a2b3851e59..0000000000 --- a/tests/restrict-github-runner-group/wildcard-name/kyverno-test.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Test -metadata: - name: ksail-runner-group-wildcard-name -policies: - - ../../../k8s/bases/infrastructure/cluster-policies/best-practices/restrict-github-runner-group.yaml -resources: - - resources.yaml -variables: values.yaml -userinfo: user-info.yaml -results: - - policy: restrict-github-runner-group - rule: ksail-only-runner-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: fail - - policy: restrict-github-runner-group - rule: tenant-cannot-rebind-group - resources: [github-config/ksail-code-quality] - kind: RunnerGroup - result: pass diff --git a/tests/restrict-github-runner-group/wildcard-name/resources.yaml b/tests/restrict-github-runner-group/wildcard-name/resources.yaml deleted file mode 100644 index 1b24ff0b5b..0000000000 --- a/tests/restrict-github-runner-group/wildcard-name/resources.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{ - "apiVersion": "actions.github.m.upbound.io/v1alpha1", - "kind": "RunnerGroup", - "metadata": { - "name": "ksail-code-quality", - "namespace": "github-config" - }, - "spec": { - "forProvider": { - "name": "*", - "visibility": "selected", - "allowsPublicRepositories": true, - "restrictedToWorkflows": false, - "selectedRepositoryIds": [ - 737584922 - ], - "selectedWorkflows": [] - }, - "providerConfigRef": { - "name": "default", - "kind": "ProviderConfig" - } - } -} diff --git a/tests/restrict-github-runner-group/wildcard-name/user-info.yaml b/tests/restrict-github-runner-group/wildcard-name/user-info.yaml deleted file mode 100644 index bd3387aeeb..0000000000 --- a/tests/restrict-github-runner-group/wildcard-name/user-info.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: UserInfo -metadata: {name: tenant} -userInfo: - username: system:serviceaccount:github-config:github-config - groups: [system:serviceaccounts, system:serviceaccounts:github-config, system:authenticated] diff --git a/tests/restrict-github-runner-group/wildcard-name/values.yaml b/tests/restrict-github-runner-group/wildcard-name/values.yaml deleted file mode 100644 index 3b02a285e6..0000000000 --- a/tests/restrict-github-runner-group/wildcard-name/values.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: cli.kyverno.io/v1alpha1 -kind: Values -metadata: {name: values} -globalValues: - request.operation: CREATE -policies: - - name: restrict-github-runner-group - resources: - - name: ksail-code-quality - values: - request.oldObject: - metadata: - annotations: - crossplane.io/external-name: "" From 2638277caab6f1b6aeabd3ec9e0de51de12af1c5 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 11:53:47 +0200 Subject: [PATCH 26/62] fix: watch retained ARC release reconciliation failures --- .../hetzner/infrastructure/flux-notifications/alert.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml b/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml index dc20899363..a2e16226fe 100644 --- a/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml +++ b/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml @@ -87,6 +87,12 @@ spec: - kind: HelmRelease name: "*" namespace: actual-budget + - kind: HelmRelease + name: "*" + namespace: arc-ksail-analysis + - kind: HelmRelease + name: "*" + namespace: arc-systems - kind: HelmRelease name: "*" namespace: backstage From 783a532d6e17af7c952309ca47d02e590a6b371d Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 12:00:23 +0200 Subject: [PATCH 27/62] fix: isolate runtime App runner-group ownership and preflight access --- docs/operations/arc-runners.md | 67 +++++++----- .../actions-runners/external-secret.yaml | 7 ++ .../actions-runners/kustomization.yaml | 2 + .../actions-runners/provider-config.yaml | 14 +++ .../actions-runners/runner-group.yaml | 20 ++++ .../tests/arc-staging/runner_group_test.go | 102 ++++++++++++++++++ .../tests/test-verify-ksail-arc-runtime.sh | 10 +- .../test-wait-for-ksail-arc-registration.sh | 26 ++++- scripts/wait-for-ksail-arc-registration.sh | 28 ++++- 9 files changed, 244 insertions(+), 32 deletions(-) create mode 100644 k8s/bases/infrastructure/actions-runners/provider-config.yaml create mode 100644 k8s/bases/infrastructure/actions-runners/runner-group.yaml create mode 100644 scripts/tests/arc-staging/runner_group_test.go diff --git a/docs/operations/arc-runners.md b/docs/operations/arc-runners.md index 189f82f70b..7b56eaa8e1 100644 --- a/docs/operations/arc-runners.md +++ b/docs/operations/arc-runners.md @@ -1,10 +1,11 @@ # Organization Linux runners -The ARC controller and organization Linux pool are prepared but inactive. Neither -component is referenced by a deployment aggregate, and both HelmReleases are -suspended. Merging their definitions registers no runner, reads no App credential -and provisions no server. This is preparation for opt-in organization use (#4529), -not runtime acceptance of #4462 or a resolution of KSail #7131. +The production definition includes the ARC controller and bounded organization +Linux pool. The protected deployment must prove the existing runtime App, +registration and isolated canary before activation can merge. Initial access +admits only KSail's main-branch delivery preflight. Managed analysis remains on its +existing route until actual runner delivery and full-source calibration pass; +declaring the pool does not resolve KSail #7131. The controller chart and runner-set chart use the same immutable 0.15.0 artifacts. The runner image is digest-pinned. The controller manages runner sets only in the @@ -12,8 +13,8 @@ The runner image is digest-pinned. The controller manages runner sets only in th with `devantler-tech` in the dedicated `platform` runner group and exposes the `platform-linux` scale-set name. Repository access is opt-in through the group; no existing workflow or managed analysis setting is changed. Other repositories -can opt in without another App or a repository-specific pool. KSail is a proposed -first consumer, not the scope of the capability. +can opt in through reviewed access changes without another App or another pool. +KSail is the first selected consumer. The pool has zero idle runners and a maximum of one job runner **across all opted-in repositories**, not one runner per repository. There is no container mode, @@ -26,8 +27,9 @@ checksum-checked Go and Node archives provide compiler headers without job-time root access. The publisher builds and exercises the image on pull requests, then publishes, attests and signs only on a push to KSail main. Kyverno and Talos accept that workflow identity only for the exact analysis image repository. -The pool does not select that image until a verified published digest and the -consumer's runtime proof are available. +The pool selects the published KSail-owned image by immutable digest. The +protected canary verifies its signature, toolchain and resource limits before +any managed-analysis route changes. The toolchain smoke test starts the copied runner and compiles a Go program against GTK and WebKit as UID/GID 1001, with a read-only root filesystem and no @@ -37,7 +39,7 @@ and timestamps would fail on a group-writable Kubernetes volume. ## Activation gates -Activation requires a separate reviewed change and all of these proofs: +Activation and subsequent consumer admission require all of these proofs: 1. Reuse the production platform App used for GitHub sign-in, identified by `github_app_client_id` in the production bootstrap configuration. This is @@ -66,8 +68,14 @@ Activation requires a separate reviewed change and all of these proofs: Reuse does not narrow the shared App's authority: the runner group controls job access, not what the App credential can do. Never mount the private key in a job runner. -2. An authorized organization operator creates and verifies the `platform` - runner group **before** enabling the pool. Use **Selected repositories**, not +2. The namespaced `arc-runtime-platform-app` ProviderConfig manages the + `platform` group using the existing `arc-github-app` Secret in `arc-runners`. + Its additional JSON credential key preserves ARC's three original keys; + the private key never moves to the GitHub-management namespace. The provider + name is distinct because the pinned provider caches configurations by name; + the registration gate rejects a same-name configuration in another namespace + or provider kind. + Verify the observed group **before** delivering a job. Use **Selected repositories**, not all repositories, and select only explicitly approved consumers. Repository access alone does not make arbitrary code trusted. Before admitting a public repository, prove the group's selected-workflow/ref restrictions admit only @@ -77,6 +85,11 @@ Activation requires a separate reviewed change and all of these proofs: repository excluded. Do not weaken organization-wide runner restrictions or disable GitHub-hosted runners. A failed or unauthorized group read is unknown, not permission to use the unrestricted default group. + Initial selection is exactly KSail and + `devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main`. + The native dynamic Code Quality workflow is not admitted by this initial + selection. Its identity, supported restrictions and intercepted fork-job + denial require separate API and runtime proof before routing analysis. 3. Use the declared `autoscale-arc-runners` CX53 pool for isolated organization runner capacity. It has a minimum of zero and maximum of one node, sharing the unchanged cluster ceiling of nine nodes and account ceiling of ten. The @@ -123,12 +136,13 @@ Activation requires a separate reviewed change and all of these proofs: removed before declaring that bug fixed. Those bug-specific acceptance criteria do not route or onboard other repositories automatically. -For staged validation, run `go test ./scripts/tests/arc-staging` and build both -component directories directly with `kubectl kustomize`; normal local/prod trees -must continue to contain no ARC resources. The unconditional CI guard runs on -pull requests and merge groups. A deliberate activation revises that guard in -the same reviewed change, alongside its evidence; deleting the guard alone is -not activation proof. +For configuration validation, run `go test ./scripts/tests/arc-staging` and build +both component directories directly with `kubectl kustomize`. Local trees exclude +ARC. The unconditional guard permits activation only through the two named +production aggregates, with immutable images and the mandatory protected +runtime canary. It also permits retention of a suspended controller without a +pool. It runs on pull requests and merge groups; deleting it is not activation +proof. ## Repository opt-in @@ -154,14 +168,15 @@ finish. It does not require retiring the shared platform App. ## Rollout and recovery -After the gates above are approved, first reference and unsuspend the controller -in the controller layer and verify it is healthy. That component creates both -namespaces before the chart installs its namespace-scoped RBAC. The controller's -network policy also covers the listener, which ARC creates in that namespace. -Only then reference the pool in -the infrastructure layer, which creates its dedicated secret store and reader -identity. Verify SecretStore and ExternalSecret readiness before admitting jobs. Keep -the opt-in runner name out of workflows that have not completed onboarding. +The controller layer creates both namespaces before namespace-scoped RBAC +reconciles. Its network policy also covers the listener. The infrastructure +layer declares the pool, dedicated credential reader and native runner group. +Verify the stored App identity and encrypted credential boundary before +activating this layer. The protected canary joins the deployed Flux revision, +current provider/group observations, ARC registration and exact image before +it exercises admission, allowed connectivity, intercepted denials and cleanup. +Verify SecretStore and ExternalSecret readiness before admitting jobs. Keep the +opt-in runner name out of workflows that have not completed onboarding. To stop admitting jobs, restore every consumer's prior runner configuration and verify the readbacks, then use a reviewed values change to set both runner diff --git a/k8s/bases/infrastructure/actions-runners/external-secret.yaml b/k8s/bases/infrastructure/actions-runners/external-secret.yaml index c3250e3fc8..a043c16d2f 100644 --- a/k8s/bases/infrastructure/actions-runners/external-secret.yaml +++ b/k8s/bases/infrastructure/actions-runners/external-secret.yaml @@ -11,6 +11,13 @@ spec: target: name: arc-github-app creationPolicy: Owner + template: + engineVersion: v2 + mergePolicy: Merge + data: + # JSON encoding preserves the existing PEM and the original ARC keys. + provider-credentials: >- + {"owner":"devantler-tech","app_auth":[{"id":{{ .github_app_id | toJson }},"installation_id":{{ .github_app_installation_id | toJson }},"pem_file":{{ .github_app_private_key | toJson }}}]} data: - secretKey: github_app_id remoteRef: diff --git a/k8s/bases/infrastructure/actions-runners/kustomization.yaml b/k8s/bases/infrastructure/actions-runners/kustomization.yaml index 1cb050ffe3..7d2e9b027e 100644 --- a/k8s/bases/infrastructure/actions-runners/kustomization.yaml +++ b/k8s/bases/infrastructure/actions-runners/kustomization.yaml @@ -21,4 +21,6 @@ resources: - service-account.yaml - secret-store.yaml - external-secret.yaml + - provider-config.yaml + - runner-group.yaml - cilium-network-policy-runner.yaml diff --git a/k8s/bases/infrastructure/actions-runners/provider-config.yaml b/k8s/bases/infrastructure/actions-runners/provider-config.yaml new file mode 100644 index 0000000000..c096eedd0e --- /dev/null +++ b/k8s/bases/infrastructure/actions-runners/provider-config.yaml @@ -0,0 +1,14 @@ +# A distinct cache key keeps the runtime App separate from the management App. +# The namespaced provider reads the existing ARC Secret in this namespace. +apiVersion: github.m.upbound.io/v1beta1 +kind: ProviderConfig +metadata: + name: arc-runtime-platform-app + namespace: arc-runners +spec: + credentials: + source: Secret + secretRef: + namespace: arc-runners + name: arc-github-app + key: provider-credentials diff --git a/k8s/bases/infrastructure/actions-runners/runner-group.yaml b/k8s/bases/infrastructure/actions-runners/runner-group.yaml new file mode 100644 index 0000000000..14e0702ca1 --- /dev/null +++ b/k8s/bases/infrastructure/actions-runners/runner-group.yaml @@ -0,0 +1,20 @@ +# Initial access is restricted to the main-only KSail delivery preflight. +# Dynamic managed-analysis admission requires its own verified selection. +apiVersion: actions.github.m.upbound.io/v1alpha1 +kind: RunnerGroup +metadata: + name: platform + namespace: arc-runners +spec: + managementPolicies: [Observe, Create, Update] + providerConfigRef: + name: arc-runtime-platform-app + kind: ProviderConfig + forProvider: + name: platform + visibility: selected + allowsPublicRepositories: true + selectedRepositoryIds: [737584922] + restrictedToWorkflows: true + selectedWorkflows: + - devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main diff --git a/scripts/tests/arc-staging/runner_group_test.go b/scripts/tests/arc-staging/runner_group_test.go new file mode 100644 index 0000000000..b1dd16dcc3 --- /dev/null +++ b/scripts/tests/arc-staging/runner_group_test.go @@ -0,0 +1,102 @@ +package arcstaging_test + +import ( + "bytes" + "encoding/json" + "reflect" + "strings" + "testing" + "text/template" +) + +const runtimeProviderName = "arc-runtime-platform-app" +const acceptedWorkflow = "devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main" + +func TestRunnerGroupUsesOnlyTheRuntimeAppInsideItsNamespace(t *testing.T) { + const component = "k8s/bases/infrastructure/actions-runners/" + config := readYAML(t, component+"provider-config.yaml") + equal(t, field(t, config, "apiVersion"), "github.m.upbound.io/v1beta1") + equal(t, field(t, config, "kind"), "ProviderConfig") + equal(t, field(t, config, "metadata", "name"), runtimeProviderName) + equal(t, field(t, config, "metadata", "namespace"), "arc-runners") + credentials := field(t, config, "spec", "credentials") + equal(t, field(t, credentials, "source"), "Secret") + equal(t, field(t, credentials, "secretRef", "namespace"), "arc-runners") + equal(t, field(t, credentials, "secretRef", "name"), "arc-github-app") + equal(t, field(t, credentials, "secretRef", "key"), "provider-credentials") + group := readYAML(t, component+"runner-group.yaml") + equal(t, field(t, group, "apiVersion"), "actions.github.m.upbound.io/v1alpha1") + equal(t, field(t, group, "kind"), "RunnerGroup") + equal(t, field(t, group, "metadata", "name"), "platform") + equal(t, field(t, group, "metadata", "namespace"), "arc-runners") + if !reflect.DeepEqual(field(t, group, "spec", "managementPolicies"), []any{"Observe", "Create", "Update"}) { + t.Fatal("runner-group ownership must exclude deletion and late initialization") + } + if !reflect.DeepEqual(field(t, group, "spec", "providerConfigRef"), map[string]any{"name": runtimeProviderName, "kind": "ProviderConfig"}) { + t.Fatal("runner group must use its distinct runtime-App provider") + } + parameters := field(t, group, "spec", "forProvider") + if !reflect.DeepEqual(parameters, map[string]any{ + "name": "platform", "visibility": "selected", "allowsPublicRepositories": true, + "restrictedToWorkflows": true, "selectedRepositoryIds": []any{737584922}, + "selectedWorkflows": []any{acceptedWorkflow}, + }) { + t.Fatal("initial access must admit only KSail's main-only delivery preflight") + } + resources := field(t, readYAML(t, component+"kustomization.yaml"), "resources").([]any) + for _, required := range []string{"provider-config.yaml", "runner-group.yaml"} { + count := 0 + for _, resource := range resources { + if resource == required { + count++ + } + } + if count != 1 { + t.Fatalf("%s must be included exactly once", required) + } + } +} + +func TestProviderCredentialTemplatePreservesARCKeysAndEscapesTheExistingPEM(t *testing.T) { + secret := readYAML(t, "k8s/bases/infrastructure/actions-runners/external-secret.yaml") + configuration := field(t, secret, "spec", "target", "template") + equal(t, field(t, configuration, "engineVersion"), "v2") + equal(t, field(t, configuration, "mergePolicy"), "Merge") + data := field(t, configuration, "data").(map[string]any) + if len(data) != 1 { + t.Fatal("template must add only the provider's JSON credential key") + } + source := field(t, data, "provider-credentials").(string) + parsed, err := template.New("credentials").Option("missingkey=error").Funcs(template.FuncMap{ + "toJson": func(value any) (string, error) { + encoded, err := json.Marshal(value) + return string(encoded), err + }, + }).Parse(source) + if err != nil { + t.Fatal(err) + } + for _, pem := range []string{"synthetic-key\nsecond-line\n", "synthetic\"key\\with\ncontrols\t"} { + inputs := map[string]string{ + "github_app_id": "123", "github_app_installation_id": "456", "github_app_private_key": pem, + } + var output bytes.Buffer + if err := parsed.Execute(&output, inputs); err != nil { + t.Fatal(err) + } + var value map[string]any + if err := json.Unmarshal(output.Bytes(), &value); err != nil { + t.Fatalf("credential JSON cannot round-trip a PEM: %v", err) + } + want := map[string]any{"owner": "devantler-tech", "app_auth": []any{ + map[string]any{"id": "123", "installation_id": "456", "pem_file": pem}, + }} + if !reflect.DeepEqual(value, want) { + t.Fatal("provider credentials changed the existing App fields") + } + } + var output bytes.Buffer + if err := parsed.Execute(&output, map[string]string{}); err == nil || !strings.Contains(err.Error(), "github_app_") { + t.Fatal("missing existing App fields must fail the credential template") + } +} diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh index 6f1d29731b..e79122d214 100755 --- a/scripts/tests/test-verify-ksail-arc-runtime.sh +++ b/scripts/tests/test-verify-ksail-arc-runtime.sh @@ -108,6 +108,12 @@ touch "$ARC_TEST_ROOT/runtime-access" shift 3 # context pair and request-timeout args="$*" case "$args" in + *'get providerconfigs.github.m.upbound.io --all-namespaces '*) + jq -cn '{items:[{apiVersion:"github.m.upbound.io/v1beta1",kind:"ProviderConfig", + metadata:{name:"arc-runtime-platform-app",namespace:"arc-runners"}, + spec:{credentials:{source:"Secret",secretRef:{namespace:"arc-runners",name:"arc-github-app",key:"provider-credentials"}}}}]}' ;; + *'get clusterproviderconfigs.github.m.upbound.io '*|*'get providerconfigs.github.upbound.io '*) + printf '{"items":[]}' ;; *'get deployment cluster-autoscaler-hetzner-cluster-autoscaler '*) printf '%s\n' --cloud-provider=hetzner --max-nodes-total=9 --nodes=0:1:cx53:fsn1:autoscale-arc-runners [[ "$ARC_TEST_CASE" != wider-ceiling ]] || printf '%s\n' --max-nodes-total=10 ;; @@ -125,8 +131,8 @@ case "$args" in *'get runnergroups.actions.github.m.upbound.io platform '*) jq -cn '{apiVersion:"actions.github.m.upbound.io/v1alpha1",kind:"RunnerGroup", metadata:{name:"platform",namespace:"arc-runners",uid:"group-uid",generation:2,annotations:{"crossplane.io/external-name":"27"}}, - spec:{providerConfigRef:{name:"runtime-app",kind:"ProviderConfig"},forProvider:{name:"platform",visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:false,selectedRepositoryIds:[737584922],selectedWorkflows:[]}}, - status:{atProvider:{id:"27",name:"platform",default:false,inherited:false,visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:false,selectedRepositoryIds:[737584922],selectedWorkflows:[], + spec:{providerConfigRef:{name:"arc-runtime-platform-app",kind:"ProviderConfig"},forProvider:{name:"platform",visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:true,selectedRepositoryIds:[737584922],selectedWorkflows:["devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main"]}}, + status:{atProvider:{id:"27",name:"platform",default:false,inherited:false,visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:true,selectedRepositoryIds:[737584922],selectedWorkflows:["devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main"], runnersUrl:"https://api.github.com/orgs/devantler-tech/actions/runner-groups/27/runners", selectedRepositoriesUrl:"https://api.github.com/orgs/devantler-tech/actions/runner-groups/27/repositories"}, conditions:[{type:"Ready",status:"True",observedGeneration:2},{type:"Synced",status:"True",observedGeneration:2}]}}' ;; diff --git a/scripts/tests/test-wait-for-ksail-arc-registration.sh b/scripts/tests/test-wait-for-ksail-arc-registration.sh index 5d25d9c1ac..a8f24b1473 100755 --- a/scripts/tests/test-wait-for-ksail-arc-registration.sh +++ b/scripts/tests/test-wait-for-ksail-arc-registration.sh @@ -18,6 +18,20 @@ cat >"$scratch/bin/kubectl" <<'SH' set -euo pipefail [[ "$*" == *'--context admin@prod'* ]] || exit 92 case "$*" in + *'get providerconfigs.github.m.upbound.io --all-namespaces '*) + [[ "$GROUP_CASE" != unreadable-provider ]] || exit 1 + jq -cn --arg scenario "$GROUP_CASE" '{items:[{ + apiVersion:"github.m.upbound.io/v1beta1",kind:"ProviderConfig", + metadata:{name:"arc-runtime-platform-app",namespace:"arc-runners"}, + spec:{credentials:{source:"Secret",secretRef:{namespace:"arc-runners",name:"arc-github-app",key:"provider-credentials"}}} + }]} | + if $scenario=="duplicate-provider" then .items+=[.items[0]|.metadata.namespace="other"] + elif $scenario=="wrong-provider-secret" then .items[0].spec.credentials.secretRef.name="management-app" + else . end' ;; + *'get clusterproviderconfigs.github.m.upbound.io '*) + jq -cn --arg scenario "$GROUP_CASE" '{items:(if $scenario=="cluster-provider-collision" then [{metadata:{name:"arc-runtime-platform-app"}}] else [] end)}' ;; + *'get providerconfigs.github.upbound.io '*) + jq -cn --arg scenario "$GROUP_CASE" '{items:(if $scenario=="legacy-provider-collision" then [{metadata:{name:"arc-runtime-platform-app"}}] else [] end)}' ;; *'get kustomization '*) jq -cn --arg revision "latest@$GROUP_DIGEST" '{metadata:{generation:1},status:{observedGeneration:1,lastAppliedRevision:$revision,conditions:[{type:"Ready",status:"True"}]}}' ;; *'get runnergroups.actions.github.m.upbound.io platform '*) @@ -25,8 +39,8 @@ case "$*" in jq -cn --arg scenario "$GROUP_CASE" '{ apiVersion:"actions.github.m.upbound.io/v1alpha1",kind:"RunnerGroup", metadata:{name:"platform",namespace:"arc-runners",uid:"00000000-0000-0000-0000-000000000002",generation:2,annotations:{"crossplane.io/external-name":"27"}}, - spec:{providerConfigRef:{name:"runtime-app",kind:"ProviderConfig"},forProvider:{name:"platform",visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:false,selectedRepositoryIds:[737584922],selectedWorkflows:[]}}, - status:{atProvider:{id:"27",name:"platform",default:false,inherited:false,visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:false,selectedRepositoryIds:[737584922],selectedWorkflows:[], + spec:{providerConfigRef:{name:"arc-runtime-platform-app",kind:"ProviderConfig"},forProvider:{name:"platform",visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:true,selectedRepositoryIds:[737584922],selectedWorkflows:["devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main"]}}, + status:{atProvider:{id:"27",name:"platform",default:false,inherited:false,visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:true,selectedRepositoryIds:[737584922],selectedWorkflows:["devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main"], runnersUrl:"https://api.github.com/orgs/devantler-tech/actions/runner-groups/27/runners", selectedRepositoriesUrl:"https://api.github.com/orgs/devantler-tech/actions/runner-groups/27/repositories"}, conditions:[{type:"Ready",status:"True",observedGeneration:2},{type:"Synced",status:"True",observedGeneration:2}]} @@ -39,6 +53,10 @@ case "$*" in elif $scenario=="stale-observation" then .status.conditions[1].observedGeneration=1 elif $scenario=="missing-observation" then del(.status.conditions[1].observedGeneration) elif $scenario=="unbound-id" then .metadata.annotations["crossplane.io/external-name"]="1" + elif $scenario=="unrestricted-workflows" then .status.atProvider.restrictedToWorkflows=false + elif $scenario=="extra-workflow" then .status.atProvider.selectedWorkflows+=["devantler-tech/ksail/.github/workflows/ci.yaml@refs/heads/main"] + elif $scenario=="wrong-workflow-ref" then .status.atProvider.selectedWorkflows=["devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/pull/1/head"] + elif $scenario=="unrestricted-desired" then .spec.forProvider.restrictedToWorkflows=false else . end' ;; *'get autoscalingrunnerset.actions.github.com platform-linux '*) jq -cn --arg scenario "$GROUP_CASE" '{metadata:{generation:1,annotations:{"runner-scale-set-id":"12","actions.github.com/runner-group-name":"platform","actions.github.com/runner-scale-set-name":"platform-linux"}}, @@ -50,7 +68,9 @@ esac SH chmod 700 "$scratch/bin/"* for scenario in normal all-repositories extra-repository default-group inherited-group wrong-remote-org \ - stale-observation missing-observation unbound-id unreadable-group wrong-registered-group; do + stale-observation missing-observation unbound-id unreadable-group wrong-registered-group \ + unrestricted-workflows extra-workflow wrong-workflow-ref unrestricted-desired \ + duplicate-provider wrong-provider-secret cluster-provider-collision legacy-provider-collision unreadable-provider; do if PATH="$scratch/bin:$PATH" GROUP_DIGEST="sha256:$(printf 'a%.0s' {1..64})" GROUP_CASE="$scenario" \ GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform \ bash "$root/scripts/wait-for-ksail-arc-registration.sh" "sha256:$(printf 'a%.0s' {1..64})" \ diff --git a/scripts/wait-for-ksail-arc-registration.sh b/scripts/wait-for-ksail-arc-registration.sh index 76734b873b..511651c82b 100755 --- a/scripts/wait-for-ksail-arc-registration.sh +++ b/scripts/wait-for-ksail-arc-registration.sh @@ -19,6 +19,28 @@ while ((SECONDS < deadline)); do ready=false fi done + # The pinned provider caches by reference name, across namespaces and kinds. + # Bind this distinct name to exactly one native configuration and Secret. + if ! state=$(kc get providerconfigs.github.m.upbound.io --all-namespaces -o json) || + ! jq -e ' + [.items[] | select(.metadata.name == "arc-runtime-platform-app")] as $configs | + ($configs | length) == 1 and + $configs[0].apiVersion == "github.m.upbound.io/v1beta1" and + $configs[0].kind == "ProviderConfig" and + $configs[0].metadata.namespace == "arc-runners" and + $configs[0].metadata.deletionTimestamp == null and + $configs[0].spec.credentials == {"source":"Secret","secretRef":{ + "namespace":"arc-runners","name":"arc-github-app","key":"provider-credentials"}} + ' <<<"$state" >/dev/null 2>&1; then + ready=false + fi + for resource in clusterproviderconfigs.github.m.upbound.io providerconfigs.github.upbound.io; do + if ! state=$(kc get "$resource" -o json) || + ! jq -e '[.items[] | select(.metadata.name == "arc-runtime-platform-app")] | length == 0' \ + <<<"$state" >/dev/null 2>&1; then + ready=false + fi + done # The provider observation comes from the existing organization's App. # Desired selection alone cannot establish the remote access boundary. if ! state=$(kc -n arc-runners get runnergroups.actions.github.m.upbound.io platform -o json) || @@ -31,13 +53,17 @@ while ((SECONDS < deadline)); do ($generation | type == "number" and . > 0) and ($id | type == "string" and test("^[1-9][0-9]{0,18}$")) and .metadata.annotations["crossplane.io/external-name"] == $id and - .spec.providerConfigRef == {"name":"runtime-app","kind":"ProviderConfig"} and + .spec.providerConfigRef == {"name":"arc-runtime-platform-app","kind":"ProviderConfig"} and .spec.forProvider.name == "platform" and .spec.forProvider.visibility == "selected" and .spec.forProvider.selectedRepositoryIds == [737584922] and .spec.forProvider.allowsPublicRepositories == true and + .spec.forProvider.restrictedToWorkflows == true and + .spec.forProvider.selectedWorkflows == ["devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main"] and .status.atProvider.name == "platform" and .status.atProvider.visibility == "selected" and .status.atProvider.selectedRepositoryIds == [737584922] and .status.atProvider.allowsPublicRepositories == true and + .status.atProvider.restrictedToWorkflows == true and + .status.atProvider.selectedWorkflows == ["devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main"] and .status.atProvider.default == false and .status.atProvider.inherited == false and .status.atProvider.runnersUrl == ("https://api.github.com/orgs/devantler-tech/actions/runner-groups/" + $id + "/runners") and .status.atProvider.selectedRepositoriesUrl == ("https://api.github.com/orgs/devantler-tech/actions/runner-groups/" + $id + "/repositories") and From 8c9a62a8b54fc00eb2757c93f6f1b2b3b196f462 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 12:46:50 +0200 Subject: [PATCH 28/62] fix(ci): protect existing ARC credential transport with native TLS --- .../actions-runners/secret-store.yaml | 6 +- .../controllers/kustomization.yaml | 2 + .../openbao/transport/ca-certificate.yaml | 22 +++ .../openbao/transport/ca-issuer.yaml | 7 + .../openbao/transport/certificate-policy.yaml | 35 +++++ .../openbao/transport/certificate.yaml | 21 +++ ...ilium-network-policy-external-secrets.yaml | 20 +++ .../transport/cilium-network-policy.yaml | 20 +++ .../openbao/transport/cluster-issuer.yaml | 7 + .../openbao/transport/helm-release-patch.yaml | 131 ++++++++++++++++++ .../openbao/transport/kustomization.yaml | 13 ++ .../transport/listener-config-map.yaml | 15 ++ .../openbao/transport/service.yaml | 17 +++ .../openbao/transport/trust-bundle.yaml | 15 ++ scripts/tests/arc-staging/transport_test.go | 94 +++++++++++++ scripts/tests/test-openbao-standby-canary.sh | 27 +++- 16 files changed, 448 insertions(+), 4 deletions(-) create mode 100644 k8s/providers/hetzner/infrastructure/controllers/openbao/transport/ca-certificate.yaml create mode 100644 k8s/providers/hetzner/infrastructure/controllers/openbao/transport/ca-issuer.yaml create mode 100644 k8s/providers/hetzner/infrastructure/controllers/openbao/transport/certificate-policy.yaml create mode 100644 k8s/providers/hetzner/infrastructure/controllers/openbao/transport/certificate.yaml create mode 100644 k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy-external-secrets.yaml create mode 100644 k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy.yaml create mode 100644 k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cluster-issuer.yaml create mode 100644 k8s/providers/hetzner/infrastructure/controllers/openbao/transport/helm-release-patch.yaml create mode 100644 k8s/providers/hetzner/infrastructure/controllers/openbao/transport/kustomization.yaml create mode 100644 k8s/providers/hetzner/infrastructure/controllers/openbao/transport/listener-config-map.yaml create mode 100644 k8s/providers/hetzner/infrastructure/controllers/openbao/transport/service.yaml create mode 100644 k8s/providers/hetzner/infrastructure/controllers/openbao/transport/trust-bundle.yaml create mode 100644 scripts/tests/arc-staging/transport_test.go diff --git a/k8s/bases/infrastructure/actions-runners/secret-store.yaml b/k8s/bases/infrastructure/actions-runners/secret-store.yaml index e791537783..4ce79932b6 100644 --- a/k8s/bases/infrastructure/actions-runners/secret-store.yaml +++ b/k8s/bases/infrastructure/actions-runners/secret-store.yaml @@ -8,7 +8,11 @@ metadata: spec: provider: vault: - server: "http://openbao-active.openbao.svc.cluster.local:8200" + server: "https://openbao-arc.openbao.svc.cluster.local:8204" + caProvider: + type: ConfigMap + name: arc-openbao-ca + key: ca.crt path: "secret" version: "v2" auth: diff --git a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml index f7c4269891..509eed4c54 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml @@ -61,6 +61,7 @@ resources: # HelmRelease is included via ../../../../bases/...; this dir only adds the # prod-only ConfigMap, and the patch below rewires the HelmRelease. - velero/ + - openbao/transport/ # Opt-in game-day chaos engineering — uncomment to enable, re-comment when the # game day ends (see docs/chaos-engineering.md). Left off by default: the # chaos-daemon is a permanently-privileged DaemonSet, so it ships only while @@ -128,6 +129,7 @@ patches: - path: openbao/patches/keep-off-autoscale-nodes.yaml # Stage only one OpenBao replica; lower ordinals wait for healthy readback. - path: openbao/patches/standby-oidc-canary.yaml + - path: openbao/transport/helm-release-patch.yaml # Umami's Namespace moved into the preceding controllers layer with its # Kyverno Role; keep the prod-only user-namespace label with its new owner. - path: kyverno/patches/enable-user-namespaces.yaml diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/ca-certificate.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/ca-certificate.yaml new file mode 100644 index 0000000000..1c587c99d4 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/ca-certificate.yaml @@ -0,0 +1,22 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: arc-openbao-ca + namespace: cert-manager +spec: + isCA: true + commonName: arc-openbao-ca + secretName: arc-openbao-ca + duration: 87600h + renewBefore: 8760h + privateKey: + algorithm: ECDSA + size: 256 + rotationPolicy: Never + usages: + - cert sign + - crl sign + issuerRef: + name: arc-openbao-selfsigned + kind: Issuer + group: cert-manager.io diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/ca-issuer.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/ca-issuer.yaml new file mode 100644 index 0000000000..8741b81586 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/ca-issuer.yaml @@ -0,0 +1,7 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: arc-openbao-selfsigned + namespace: cert-manager +spec: + selfSigned: {} diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/certificate-policy.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/certificate-policy.yaml new file mode 100644 index 0000000000..4e4ce8e72f --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/certificate-policy.yaml @@ -0,0 +1,35 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: restrict-arc-openbao-certificate + annotations: + policies.kyverno.io/title: Restrict ARC OpenBao Certificate Issuance + policies.kyverno.io/category: Security + policies.kyverno.io/severity: high +spec: + background: true + rules: + - name: only-openbao-may-use-credential-ca + match: + any: + - resources: + kinds: + - cert-manager.io/v1/Certificate + - cert-manager.io/v1/CertificateRequest + preconditions: + all: + - key: "{{ request.object.spec.issuerRef.name || '' }}" + operator: Equals + value: arc-openbao-ca + - key: "{{ request.object.spec.issuerRef.kind || 'Issuer' }}" + operator: Equals + value: ClusterIssuer + validate: + failureAction: Enforce + message: The dedicated credential-service CA may issue only in OpenBao. + deny: + conditions: + any: + - key: "{{ request.object.metadata.namespace || '' }}" + operator: NotEquals + value: openbao diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/certificate.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/certificate.yaml new file mode 100644 index 0000000000..b7c41bda25 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/certificate.yaml @@ -0,0 +1,21 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: arc-openbao-server-tls + namespace: openbao +spec: + secretName: arc-openbao-server-tls + duration: 2160h + renewBefore: 720h + privateKey: + algorithm: ECDSA + size: 256 + rotationPolicy: Always + usages: + - server auth + dnsNames: + - openbao-arc.openbao.svc.cluster.local + issuerRef: + name: arc-openbao-ca + kind: ClusterIssuer + group: cert-manager.io diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy-external-secrets.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy-external-secrets.yaml new file mode 100644 index 0000000000..cb30736be2 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy-external-secrets.yaml @@ -0,0 +1,20 @@ +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: allow-arc-openbao-tls + namespace: external-secrets +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: external-secrets + egress: + - toEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: openbao + app.kubernetes.io/name: openbao + app.kubernetes.io/instance: openbao + statefulset.kubernetes.io/pod-name: openbao-2 + toPorts: + - ports: + - port: "8204" + protocol: TCP diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy.yaml new file mode 100644 index 0000000000..81bccebd2d --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy.yaml @@ -0,0 +1,20 @@ +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: allow-arc-openbao-tls + namespace: openbao +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: openbao + app.kubernetes.io/instance: openbao + statefulset.kubernetes.io/pod-name: openbao-2 + ingress: + - fromEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: external-secrets + app.kubernetes.io/name: external-secrets + toPorts: + - ports: + - port: "8204" + protocol: TCP diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cluster-issuer.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cluster-issuer.yaml new file mode 100644 index 0000000000..b4f9f89467 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cluster-issuer.yaml @@ -0,0 +1,7 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: arc-openbao-ca +spec: + ca: + secretName: arc-openbao-ca diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/helm-release-patch.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/helm-release-patch.yaml new file mode 100644 index 0000000000..b08ada28b5 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/helm-release-patch.yaml @@ -0,0 +1,131 @@ +# Additional -config listeners append; Raft, audit, unseal and the legacy +# listener stay in the base. The existing partition still replaces only #2. +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: openbao + namespace: openbao +spec: + dependsOn: + - name: cert-manager + namespace: cert-manager + postRenderers: + - kustomize: + patches: + - target: + group: apps + version: v1 + kind: StatefulSet + name: openbao + namespace: openbao + patch: | + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: openbao + namespace: openbao + spec: + updateStrategy: + type: RollingUpdate + rollingUpdate: + partition: 2 + template: + spec: + automountServiceAccountToken: false + values: + server: + extraArgs: -config=/openbao/arc-transport/listener.hcl + extraPorts: + - name: arc-tls + containerPort: 8204 + protocol: TCP + # Helm values lists replace rather than merge; retain both base mounts. + volumes: + - name: unseal-keys + secret: + secretName: openbao-unseal + optional: true + - name: tmp + emptyDir: {} + - name: arc-transport + configMap: + name: arc-openbao-listener + - name: arc-tls + secret: + secretName: arc-openbao-server-tls + defaultMode: 0440 + - name: kube-api-access + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: + path: token + expirationSeconds: 3600 + - configMap: + name: kube-root-ca.crt + items: + - key: ca.crt + path: ca.crt + - downwardAPI: + items: + - path: namespace + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: unseal-keys + mountPath: /openbao/unseal + readOnly: true + - name: tmp + mountPath: /tmp + - name: arc-transport + mountPath: /openbao/arc-transport + readOnly: true + - name: arc-tls + mountPath: /openbao/arc-tls + readOnly: true + - name: kube-api-access + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true + # OpenBao 2.6 reloads TLS certificates on SIGHUP. A same-identity + # sidecar signals only the one server process; no API credentials, + # private-key mount, extra Linux capability or host namespace is used. + shareProcessNamespace: true + extraContainers: + - name: arc-tls-reload + image: quay.io/openbao/openbao:2.6.3 + command: ["/bin/sh", "-ec"] + args: + - | + while sleep 300; do + server_pid="" + for proc in /proc/[0-9]*/comm; do + [ "$(cat "$proc" 2>/dev/null)" = "bao" ] || continue + candidate="$(basename "$(dirname "$proc")")" + command="$(tr '\000' ' ' < "/proc/$candidate/cmdline")" + case "$command" in + "bao server "*|"/usr/local/bin/bao server "*) + [ -z "$server_pid" ] || exit 1 + server_pid="$candidate" + ;; + esac + done + [ -n "$server_pid" ] || continue + wget -q -O /dev/null 'http://127.0.0.1:8200/v1/sys/health?standbyok=true' || continue + kill -HUP "$server_pid" + done + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 100 + runAsGroup: 1000 + capabilities: + drop: [ALL] + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 2m + memory: 8Mi + limits: + memory: 32Mi diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/kustomization.yaml new file mode 100644 index 0000000000..a478feb504 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ca-issuer.yaml + - ca-certificate.yaml + - cluster-issuer.yaml + - certificate.yaml + - trust-bundle.yaml + - service.yaml + - listener-config-map.yaml + - cilium-network-policy.yaml + - cilium-network-policy-external-secrets.yaml + - certificate-policy.yaml diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/listener-config-map.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/listener-config-map.yaml new file mode 100644 index 0000000000..821eadc648 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/listener-config-map.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: arc-openbao-listener + namespace: openbao +data: + listener.hcl: | + listener "tcp" { + address = "0.0.0.0:8204" + cluster_address = "127.0.0.1:8205" + tls_cert_file = "/openbao/arc-tls/tls.crt" + tls_key_file = "/openbao/arc-tls/tls.key" + tls_min_version = "tls12" + tls_disable_client_certs = true + } diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/service.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/service.yaml new file mode 100644 index 0000000000..044d0cb3af --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/service.yaml @@ -0,0 +1,17 @@ +# Keep the held lower Raft ordinals untouched. OpenBao forwards standby +# requests through its native authenticated cluster TLS connection. +apiVersion: v1 +kind: Service +metadata: + name: openbao-arc + namespace: openbao +spec: + selector: + app.kubernetes.io/name: openbao + app.kubernetes.io/instance: openbao + statefulset.kubernetes.io/pod-name: openbao-2 + ports: + - name: arc-tls + port: 8204 + targetPort: 8204 + protocol: TCP diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/trust-bundle.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/trust-bundle.yaml new file mode 100644 index 0000000000..b1ce229cb8 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/trust-bundle.yaml @@ -0,0 +1,15 @@ +apiVersion: trust.cert-manager.io/v1alpha1 +kind: Bundle +metadata: + name: arc-openbao-ca +spec: + sources: + - secret: + name: arc-openbao-ca + key: tls.crt + target: + configMap: + key: ca.crt + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: arc-runners diff --git a/scripts/tests/arc-staging/transport_test.go b/scripts/tests/arc-staging/transport_test.go new file mode 100644 index 0000000000..305ebc8a7a --- /dev/null +++ b/scripts/tests/arc-staging/transport_test.go @@ -0,0 +1,94 @@ +package arcstaging_test + +import ( + "strings" + "testing" +) + +const transportPath = "k8s/providers/hetzner/infrastructure/controllers/openbao/transport/" + +func TestCredentialStoreRequiresDedicatedVerifiedTLS(t *testing.T) { + store := readYAML(t, "k8s/bases/infrastructure/actions-runners/secret-store.yaml") + vault := field(t, store, "spec", "provider", "vault") + equal(t, field(t, vault, "server"), "https://openbao-arc.openbao.svc.cluster.local:8204") + equal(t, field(t, vault, "caProvider", "type"), "ConfigMap") + equal(t, field(t, vault, "caProvider", "name"), "arc-openbao-ca") + equal(t, field(t, vault, "caProvider", "key"), "ca.crt") +} + +func TestCredentialTLSDoesNotAdvanceTheHeldRaftRollout(t *testing.T) { + canary := readYAML(t, "k8s/providers/hetzner/infrastructure/controllers/openbao/patches/standby-oidc-canary.yaml") + patches := field(t, canary, "spec", "postRenderers").([]any) + patch := field(t, patches[0], "kustomize", "patches").([]any)[0] + text := field(t, patch, "patch").(string) + if !strings.Contains(text, "partition: 2") { + t.Fatal("TLS must not advance the held lower Raft ordinals") + } + service := readYAML(t, transportPath+"service.yaml") + equal(t, field(t, service, "spec", "selector", "statefulset.kubernetes.io/pod-name"), "openbao-2") + ports := field(t, service, "spec", "ports").([]any) + if len(ports) != 1 { + t.Fatal("credential service must expose only TLS") + } + equal(t, field(t, ports[0], "port"), 8204) + equal(t, field(t, ports[0], "targetPort"), 8204) +} + +func TestCredentialTLSHasNarrowCertificateAndTrust(t *testing.T) { + certificate := readYAML(t, transportPath+"certificate.yaml") + equal(t, field(t, certificate, "metadata", "namespace"), "openbao") + equal(t, field(t, certificate, "spec", "issuerRef", "name"), "arc-openbao-ca") + equal(t, field(t, certificate, "spec", "secretName"), "arc-openbao-server-tls") + names := field(t, certificate, "spec", "dnsNames").([]any) + if len(names) != 1 || names[0] != "openbao-arc.openbao.svc.cluster.local" { + t.Fatal("certificate must bind the one credential service identity") + } + bundle := readYAML(t, transportPath+"trust-bundle.yaml") + equal(t, field(t, bundle, "metadata", "name"), "arc-openbao-ca") + equal(t, field(t, bundle, "spec", "target", "configMap", "key"), "ca.crt") + equal(t, field(t, bundle, "spec", "target", "namespaceSelector", "matchLabels", "kubernetes.io/metadata.name"), "arc-runners") + sources := field(t, bundle, "spec", "sources").([]any) + if len(sources) != 1 { + t.Fatal("credential trust must not include unrelated authorities") + } + equal(t, field(t, sources[0], "secret", "name"), "arc-openbao-ca") + equal(t, field(t, sources[0], "secret", "key"), "tls.crt") +} + +func TestCredentialTLSAppendsListenerWithoutReplacingStorage(t *testing.T) { + config := readYAML(t, transportPath+"listener-config-map.yaml") + text := field(t, config, "data", "listener.hcl").(string) + for _, fragment := range []string{"listener \"tcp\"", "0.0.0.0:8204", "127.0.0.1:8205", "tls_min_version = \"tls12\"", "tls_cert_file", "tls_key_file"} { + if !strings.Contains(text, fragment) { + t.Fatalf("missing native TLS setting %s", fragment) + } + } + for _, forbidden := range []string{"tls_disable =", "storage ", "audit ", "api_addr", "cluster_addr ="} { + if strings.Contains(text, forbidden) { + t.Fatalf("supplemental listener changes existing server setting %s", forbidden) + } + } + patch := readYAML(t, transportPath+"helm-release-patch.yaml") + equal(t, field(t, patch, "spec", "values", "server", "extraArgs"), "-config=/openbao/arc-transport/listener.hcl") +} + +func TestCertificateReloadDoesNotReceiveAPIOrKeyMounts(t *testing.T) { + patch := readYAML(t, transportPath+"helm-release-patch.yaml") + server := field(t, patch, "spec", "values", "server") + renderers := field(t, patch, "spec", "postRenderers").([]any) + patches := field(t, renderers[0], "kustomize", "patches").([]any) + if !strings.Contains(field(t, patches[0], "patch").(string), "automountServiceAccountToken: false") { + t.Fatal("the post-rendered Pod must disable API token injection") + } + containers := field(t, server, "extraContainers").([]any) + if len(containers) != 1 { + t.Fatal("expected only the native TLS reload helper") + } + reloader := containers[0].(map[string]any) + if _, ok := reloader["volumeMounts"]; ok { + t.Fatal("reload helper must not receive credential mounts") + } + equal(t, field(t, reloader, "securityContext", "runAsUser"), 100) + equal(t, field(t, reloader, "securityContext", "allowPrivilegeEscalation"), false) + equal(t, field(t, reloader, "securityContext", "readOnlyRootFilesystem"), true) +} diff --git a/scripts/tests/test-openbao-standby-canary.sh b/scripts/tests/test-openbao-standby-canary.sh index cc4f56b6c6..215eddedcc 100755 --- a/scripts/tests/test-openbao-standby-canary.sh +++ b/scripts/tests/test-openbao-standby-canary.sh @@ -75,11 +75,32 @@ yq -e '.spec.template.spec.containers[] | select(.name == "openbao") | .image == "quay.io/openbao/openbao:2.6.3"' "${statefulset}" >/dev/null || fail 'the canary must use the released standby OIDC repair' +yq -e '.spec.template.spec.automountServiceAccountToken == false' "${statefulset}" >/dev/null || + fail 'the certificate reload helper must not receive an injected API token' + before="$(yq ea -o=json -I=0 'select(.kind == "StatefulSet" and .metadata.name == "openbao") | - .spec | del(.updateStrategy)' "${scratch}/rendered.yaml" | jq -cS .)" -after="$(yq -o=json -I=0 '.spec | del(.updateStrategy)' "${statefulset}" | jq -cS .)" + .spec | del(.updateStrategy, .template.spec.automountServiceAccountToken)' "${scratch}/rendered.yaml" | jq -cS .)" +after="$(yq -o=json -I=0 '.spec | del(.updateStrategy, .template.spec.automountServiceAccountToken)' "${statefulset}" | jq -cS .)" [[ "${before}" == "${after}" ]] || - fail 'the canary post-renderer must not alter storage, probes, unseal, identity or placement' + fail 'the canary post-renderer must change only partition and the checked token-injection setting' + +# Validate the actual pinned chart and post-rendered security boundary. An +# unsupported Helm value must not look like it removed the sidecar's token. +yq -o=json "${statefulset}" | jq -e '.spec.template.spec.shareProcessNamespace == true and + ((.spec.template.spec.hostPID // false) == false) and + ((.spec.template.spec.hostNetwork // false) == false)' >/dev/null || + fail 'native certificate reload must stay inside the trusted server Pod' +yq -o=json "${statefulset}" | jq -e '.spec.template.spec.containers[] | select(.name == "arc-tls-reload") | + .image == "quay.io/openbao/openbao:2.6.3" and + ((.volumeMounts // []) | length == 0) and + .securityContext.runAsUser == 100 and + .securityContext.allowPrivilegeEscalation == false and + .securityContext.readOnlyRootFilesystem == true' >/dev/null || + fail 'certificate reload must receive neither credential mounts nor additional privilege' +yq -o=json "${statefulset}" | jq -e '.spec.template.spec.containers[] | select(.name == "openbao") | + any(.volumeMounts[]; .name == "kube-api-access" and + .mountPath == "/var/run/secrets/kubernetes.io/serviceaccount" and .readOnly == true)' >/dev/null || + fail 'the native server must retain its own explicit Kubernetes API identity' # Local/default installations keep the chart's deliberate OnDelete behavior. yq '.spec.values' "${root_dir}/k8s/bases/infrastructure/controllers/openbao/helm-release.yaml" >"${scratch}/base-values.yaml" From 2db4313c6ab87951336cbe4ef567269e739c78aa Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 13:06:30 +0200 Subject: [PATCH 29/62] fix(ci): stage verified ARC credential transport without runners --- .github/workflows/ci.yaml | 1 + docs/operations/arc-runners.md | 28 ++- .../credentials/kustomization.yaml | 5 + .../{ => credentials}/secret-store.yaml | 0 .../{ => credentials}/service-account.yaml | 0 .../actions-runners/kustomization.yaml | 3 +- .../kustomization.yaml | 6 + .../hetzner/infrastructure/kustomization.yaml | 1 + scripts/tests/arc-staging/auth_test.go | 6 +- scripts/tests/arc-staging/staging_test.go | 15 +- scripts/tests/arc-staging/transport_test.go | 18 +- .../openbao-transport-runtime/tls_test.go | 193 ++++++++++++++++++ scripts/tests/test-openbao-standby-canary.sh | 27 +++ .../approved-surface-history.md | 20 ++ .../approved-surface.txt | 2 +- tests/policy-failure-actions.json | 1 + .../kyverno-test.yaml | 39 ++++ .../resources.yaml | 65 ++++++ 18 files changed, 419 insertions(+), 11 deletions(-) create mode 100644 k8s/bases/infrastructure/actions-runners/credentials/kustomization.yaml rename k8s/bases/infrastructure/actions-runners/{ => credentials}/secret-store.yaml (100%) rename k8s/bases/infrastructure/actions-runners/{ => credentials}/service-account.yaml (100%) create mode 100644 k8s/providers/hetzner/infrastructure/arc-credential-transport/kustomization.yaml create mode 100644 scripts/tests/openbao-transport-runtime/tls_test.go create mode 100644 tests/restrict-arc-openbao-certificate/kyverno-test.yaml create mode 100644 tests/restrict-arc-openbao-certificate/resources.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2bb21d9409..a89b495d94 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -488,6 +488,7 @@ jobs: - '.github/workflows/validate-image-verifier-liveness.yaml' - 'scripts/tests/test-openbao-oidc-role.sh' - 'scripts/tests/test-openbao-standby-canary.sh' + - 'scripts/tests/openbao-transport-runtime/**' - 'scripts/tests/test-umami-rendered-placement.sh' - 'scripts/tests/test-human-api-access-boundary.sh' - 'scripts/tests/test-world-zone-signatures.sh' diff --git a/docs/operations/arc-runners.md b/docs/operations/arc-runners.md index 5dc061cab4..35a95c0412 100644 --- a/docs/operations/arc-runners.md +++ b/docs/operations/arc-runners.md @@ -38,6 +38,30 @@ and timestamps would fail on a group-writable Kubernetes volume. ## Activation gates +Production prepares credential transport separately from runner activation. +The dedicated native TLS listener is served by the already repaired highest +Raft ordinal; the held partition is unchanged. Standby requests use OpenBao's +authenticated cluster connection to reach the leader. The existing API listener, +Raft storage, audit configuration and unseal hook remain in the base configuration. +An additional listener file is appended through the pinned chart. + +The dedicated certificate authority is trusted only in the runner namespace. +Its issuance policy denies use outside OpenBao. The credential store requires +HTTPS and that authority, with hostname verification and no plaintext fallback. +Authentication staging includes only the dedicated reader ServiceAccount and +SecretStore; it does not synchronize an App key or install a runner pool. +OpenBao's same-identity reload helper has no credential volume or API token. +It signals the server after an unsealed health response so certificate renewal +does not depend on replacing the held Raft replicas. + +The pinned-server regression exercises TLS, wrong-name and wrong-authority +rejection, obsolete-protocol rejection and actual certificate reload against +an empty synthetic local store. These checks are not production transport, +stored-key identity, secret synchronization or same-node runner-isolation proof. +Those observations remain mandatory before credential materialization and ARC +activation. The production deployment must confirm the listener and dedicated +store; the protected identity verifier then establishes the actual stored App. + Activation requires a separate reviewed change and all of these proofs: 1. Reuse the production platform App used for GitHub sign-in, identified by @@ -63,7 +87,9 @@ Activation requires a separate reviewed change and all of these proofs: verification outcome, never the key or tokens. Prove authenticated, encrypted credential transport and protection against observation by untrusted workloads, including workloads on the same node. - Do not activate the store until this transport boundary is verified. + Do not materialize the App key or activate runners until this transport + boundary is verified. Authentication-only staging must use the declared TLS + endpoint and its dedicated trust bundle. Reuse does not narrow the shared App's authority: the runner group controls job access, not what the App credential can do. Never mount the private key in a job runner. diff --git a/k8s/bases/infrastructure/actions-runners/credentials/kustomization.yaml b/k8s/bases/infrastructure/actions-runners/credentials/kustomization.yaml new file mode 100644 index 0000000000..6874b197ff --- /dev/null +++ b/k8s/bases/infrastructure/actions-runners/credentials/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - service-account.yaml + - secret-store.yaml diff --git a/k8s/bases/infrastructure/actions-runners/secret-store.yaml b/k8s/bases/infrastructure/actions-runners/credentials/secret-store.yaml similarity index 100% rename from k8s/bases/infrastructure/actions-runners/secret-store.yaml rename to k8s/bases/infrastructure/actions-runners/credentials/secret-store.yaml diff --git a/k8s/bases/infrastructure/actions-runners/service-account.yaml b/k8s/bases/infrastructure/actions-runners/credentials/service-account.yaml similarity index 100% rename from k8s/bases/infrastructure/actions-runners/service-account.yaml rename to k8s/bases/infrastructure/actions-runners/credentials/service-account.yaml diff --git a/k8s/bases/infrastructure/actions-runners/kustomization.yaml b/k8s/bases/infrastructure/actions-runners/kustomization.yaml index 7ac51fbf47..a9ae04bfdc 100644 --- a/k8s/bases/infrastructure/actions-runners/kustomization.yaml +++ b/k8s/bases/infrastructure/actions-runners/kustomization.yaml @@ -4,7 +4,6 @@ kind: Kustomization resources: - oci-repository.yaml - helm-release.yaml - - service-account.yaml - - secret-store.yaml + - credentials/ - external-secret.yaml - cilium-network-policy-runner.yaml diff --git a/k8s/providers/hetzner/infrastructure/arc-credential-transport/kustomization.yaml b/k8s/providers/hetzner/infrastructure/arc-credential-transport/kustomization.yaml new file mode 100644 index 0000000000..36eef1ddfc --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/arc-credential-transport/kustomization.yaml @@ -0,0 +1,6 @@ +# Authentication and CA validation can be exercised before reading the App key +# or activating ARC. The full pool replaces this staging reference on activation. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../../../../bases/infrastructure/actions-runners/credentials/ diff --git a/k8s/providers/hetzner/infrastructure/kustomization.yaml b/k8s/providers/hetzner/infrastructure/kustomization.yaml index 3fe12bc31b..cd12975f9d 100644 --- a/k8s/providers/hetzner/infrastructure/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/kustomization.yaml @@ -6,6 +6,7 @@ resources: # Re-adopt the retained release without activating a runner. Suspension does # not drain an installed scale set; runtime acceptance remains a separate gate. - retained-ksail-analysis/ + - arc-credential-transport/ # Prod-only audit-log forwarder. It consumes the `coroot-api-key` generated # from the Hetzner Coroot CR patch, so it belongs in this infrastructure # layer alongside that CR rather than in the earlier wait-gated controllers diff --git a/scripts/tests/arc-staging/auth_test.go b/scripts/tests/arc-staging/auth_test.go index 35a1d6b046..8643f70f9b 100644 --- a/scripts/tests/arc-staging/auth_test.go +++ b/scripts/tests/arc-staging/auth_test.go @@ -11,7 +11,7 @@ import ( func TestCredentialReaderIsSeparateFromJobRunners(t *testing.T) { component := "k8s/bases/infrastructure/actions-runners/" - store := readYAML(t, component+"secret-store.yaml") + store := readYAML(t, component+"credentials/secret-store.yaml") equal(t, field(t, store, "kind"), "SecretStore") equal(t, field(t, store, "metadata", "name"), "openbao") equal(t, field(t, store, "metadata", "namespace"), "arc-runners") @@ -22,11 +22,11 @@ func TestCredentialReaderIsSeparateFromJobRunners(t *testing.T) { equal(t, field(t, auth, "mountPath"), "kubernetes") equal(t, field(t, auth, "role"), "arc-secret-reader") equal(t, field(t, auth, "serviceAccountRef", "name"), "arc-secret-reader") - account := readYAML(t, component+"service-account.yaml") + account := readYAML(t, component+"credentials/service-account.yaml") equal(t, field(t, account, "metadata", "name"), "arc-secret-reader") equal(t, field(t, account, "metadata", "namespace"), "arc-runners") equal(t, field(t, account, "automountServiceAccountToken"), false) - resources := field(t, readYAML(t, component+"kustomization.yaml"), "resources").([]any) + resources := field(t, readYAML(t, component+"credentials/kustomization.yaml"), "resources").([]any) for _, required := range []string{"secret-store.yaml", "service-account.yaml"} { found := false for _, resource := range resources { diff --git a/scripts/tests/arc-staging/staging_test.go b/scripts/tests/arc-staging/staging_test.go index 9c140f4ed1..4df221c4b6 100644 --- a/scripts/tests/arc-staging/staging_test.go +++ b/scripts/tests/arc-staging/staging_test.go @@ -192,7 +192,7 @@ func TestAppLookupHasDedicatedReadOnlyAuthentication(t *testing.T) { component := "k8s/bases/infrastructure/actions-runners/" secret := readYAML(t, component+"external-secret.yaml") equal(t, field(t, secret, "spec", "secretStoreRef", "kind"), "SecretStore") - store := readYAML(t, component+"secret-store.yaml") + store := readYAML(t, component+"credentials/secret-store.yaml") equal(t, field(t, store, "metadata", "name"), field(t, secret, "spec", "secretStoreRef", "name")) equal(t, field(t, store, "metadata", "namespace"), "arc-runners") vault := field(t, store, "spec", "provider", "vault") @@ -201,11 +201,11 @@ func TestAppLookupHasDedicatedReadOnlyAuthentication(t *testing.T) { identity := field(t, vault, "auth", "kubernetes") equal(t, field(t, identity, "mountPath"), "kubernetes") role := field(t, identity, "role").(string) - account := readYAML(t, component+"service-account.yaml") + account := readYAML(t, component+"credentials/service-account.yaml") equal(t, field(t, account, "metadata", "name"), field(t, identity, "serviceAccountRef", "name")) equal(t, field(t, account, "metadata", "namespace"), "arc-runners") equal(t, field(t, account, "automountServiceAccountToken"), false) - resources := field(t, readYAML(t, component+"kustomization.yaml"), "resources").([]any) + resources := field(t, readYAML(t, component+"credentials/kustomization.yaml"), "resources").([]any) for _, needed := range []string{"secret-store.yaml", "service-account.yaml"} { found := false for _, resource := range resources { @@ -322,6 +322,7 @@ func TestNoDeploymentAggregateActivatesARC(t *testing.T) { const retainedControllerAggregate = "k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" const retainedControllerReference = "../../../../bases/infrastructure/controllers/actions-runner-controller/" controllerFound := false + credentialStageFound := false err := filepath.WalkDir(filepath.Join(repoRoot, "k8s"), func(path string, entry os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr @@ -350,6 +351,11 @@ func TestNoDeploymentAggregateActivatesARC(t *testing.T) { controllerFound = true continue } + if relative == "k8s/providers/hetzner/infrastructure/arc-credential-transport/kustomization.yaml" && + reference == "../../../../bases/infrastructure/actions-runners/credentials/" && !credentialStageFound { + credentialStageFound = true + continue + } t.Errorf("%s activates ARC through %q", path, reference) } } @@ -361,6 +367,9 @@ func TestNoDeploymentAggregateActivatesARC(t *testing.T) { if !controllerFound { t.Fatal("production must retain the suspended controller in its inventory") } + if !credentialStageFound { + t.Fatal("production must stage only the dedicated credential authentication resources") + } } func TestRetainedAnalysisResourcesCannotActivateOrLosePruneProtection(t *testing.T) { diff --git a/scripts/tests/arc-staging/transport_test.go b/scripts/tests/arc-staging/transport_test.go index 305ebc8a7a..53381208d4 100644 --- a/scripts/tests/arc-staging/transport_test.go +++ b/scripts/tests/arc-staging/transport_test.go @@ -8,7 +8,7 @@ import ( const transportPath = "k8s/providers/hetzner/infrastructure/controllers/openbao/transport/" func TestCredentialStoreRequiresDedicatedVerifiedTLS(t *testing.T) { - store := readYAML(t, "k8s/bases/infrastructure/actions-runners/secret-store.yaml") + store := readYAML(t, "k8s/bases/infrastructure/actions-runners/credentials/secret-store.yaml") vault := field(t, store, "spec", "provider", "vault") equal(t, field(t, vault, "server"), "https://openbao-arc.openbao.svc.cluster.local:8204") equal(t, field(t, vault, "caProvider", "type"), "ConfigMap") @@ -16,6 +16,22 @@ func TestCredentialStoreRequiresDedicatedVerifiedTLS(t *testing.T) { equal(t, field(t, vault, "caProvider", "key"), "ca.crt") } +func TestCredentialAuthenticationStagingDoesNotReadAppKeys(t *testing.T) { + stage := readYAML(t, "k8s/providers/hetzner/infrastructure/arc-credential-transport/kustomization.yaml") + resources := field(t, stage, "resources").([]any) + if len(resources) != 1 { + t.Fatal("credential staging must contain only its authentication component") + } + equal(t, resources[0], "../../../../bases/infrastructure/actions-runners/credentials/") + credentials := readYAML(t, "k8s/bases/infrastructure/actions-runners/credentials/kustomization.yaml") + resources = field(t, credentials, "resources").([]any) + if len(resources) != 2 { + t.Fatal("authentication staging must not include any key reader or runner") + } + equal(t, resources[0], "service-account.yaml") + equal(t, resources[1], "secret-store.yaml") +} + func TestCredentialTLSDoesNotAdvanceTheHeldRaftRollout(t *testing.T) { canary := readYAML(t, "k8s/providers/hetzner/infrastructure/controllers/openbao/patches/standby-oidc-canary.yaml") patches := field(t, canary, "spec", "postRenderers").([]any) diff --git a/scripts/tests/openbao-transport-runtime/tls_test.go b/scripts/tests/openbao-transport-runtime/tls_test.go new file mode 100644 index 0000000000..196de3f4e8 --- /dev/null +++ b/scripts/tests/openbao-transport-runtime/tls_test.go @@ -0,0 +1,193 @@ +package transport_test + +import ( + "bytes" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "io" + "math/big" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "gopkg.in/yaml.v3" +) + +// TestNativeTLSAndCertificateReload exercises the declared supplemental listener +// with the actual pinned server, synthetic certificates and an empty local store. +// It never initializes the store or accesses a production credential. +func TestNativeTLSAndCertificateReload(t *testing.T) { + binary := os.Getenv("OPENBAO_TLS_TEST_BINARY") + if binary == "" { + t.Skip("native binary is supplied by the mandatory chart regression") + } + version, err := exec.Command(binary, "version").Output() + if err != nil || !strings.HasPrefix(string(version), "OpenBao v2.6.3 ") { + t.Fatal("expected the verified OpenBao 2.6.3 binary") + } + root := t.TempDir() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + ca := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "synthetic-transport-ca"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + ca, err = x509.ParseCertificate(caDER) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + pool.AddCert(ca) + const hostname = "openbao-arc.openbao.svc.cluster.local" + writeLeaf := func(serial int64) { + t.Helper() + leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + leaf := &x509.Certificate{SerialNumber: big.NewInt(serial), DNSNames: []string{hostname}, + NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}} + der, err := x509.CreateCertificate(rand.Reader, leaf, ca, &leafKey.PublicKey, key) + if err != nil { + t.Fatal(err) + } + private, err := x509.MarshalPKCS8PrivateKey(leafKey) + if err != nil { + t.Fatal(err) + } + for name, block := range map[string]*pem.Block{"tls.crt": {Type: "CERTIFICATE", Bytes: der}, "tls.key": {Type: "PRIVATE KEY", Bytes: private}} { + if err := os.WriteFile(filepath.Join(root, name), pem.EncodeToMemory(block), 0600); err != nil { + t.Fatal(err) + } + } + } + writeLeaf(2) + freePort := func() int { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := listener.Addr().(*net.TCPAddr).Port + if err := listener.Close(); err != nil { + t.Fatal(err) + } + return port + } + plainPort, tlsPort, clusterPort, tlsClusterPort := freePort(), freePort(), freePort(), freePort() + base := fmt.Sprintf("disable_mlock = true\napi_addr = \"http://127.0.0.1:%d\"\ncluster_addr = \"https://127.0.0.1:%d\"\nlistener \"tcp\" {\n address = \"127.0.0.1:%d\"\n cluster_address = \"127.0.0.1:%d\"\n tls_disable = true\n}\nstorage \"file\" {\n path = %q\n}\n", plainPort, clusterPort, plainPort, clusterPort, filepath.Join(root, "data")) + if err := os.WriteFile(filepath.Join(root, "base.hcl"), []byte(base), 0600); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile("../../../k8s/providers/hetzner/infrastructure/controllers/openbao/transport/listener-config-map.yaml") + if err != nil { + t.Fatal(err) + } + var declaration struct{ Data map[string]string } + if err := yaml.Unmarshal(data, &declaration); err != nil { + t.Fatal(err) + } + listenerConfig := declaration.Data["listener.hcl"] + for old, next := range map[string]string{ + "0.0.0.0:8204": fmt.Sprintf("127.0.0.1:%d", tlsPort), + "127.0.0.1:8205": fmt.Sprintf("127.0.0.1:%d", tlsClusterPort), + "/openbao/arc-tls/tls.crt": filepath.Join(root, "tls.crt"), + "/openbao/arc-tls/tls.key": filepath.Join(root, "tls.key"), + } { + listenerConfig = strings.ReplaceAll(listenerConfig, old, next) + } + if err := os.WriteFile(filepath.Join(root, "listener.hcl"), []byte(listenerConfig), 0600); err != nil { + t.Fatal(err) + } + command := exec.Command(binary, "server", "-config="+filepath.Join(root, "base.hcl"), "-config="+filepath.Join(root, "listener.hcl")) + var privateLog bytes.Buffer + command.Stdout, command.Stderr = &privateLog, &privateLog + if err := command.Start(); err != nil { + t.Fatal("native server did not start") + } + done := make(chan error, 1) + go func() { done <- command.Wait() }() + t.Cleanup(func() { + _ = command.Process.Signal(syscall.SIGTERM) + select { + case <-done: + case <-time.After(3 * time.Second): + _ = command.Process.Kill() + <-done + } + }) + endpoint := fmt.Sprintf("https://127.0.0.1:%d/v1/sys/health", tlsPort) + client := func(servername string, roots *x509.CertPool, min, max uint16) *http.Client { + return &http.Client{Timeout: time.Second, Transport: &http.Transport{TLSClientConfig: &tls.Config{ + RootCAs: roots, ServerName: servername, MinVersion: min, MaxVersion: max, + }, DisableKeepAlives: true}, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + } + readHealth := func(c *http.Client) (int64, error) { + response, err := c.Get(endpoint) + if err != nil { + return 0, err + } + defer response.Body.Close() + _, _ = io.Copy(io.Discard, response.Body) + if response.StatusCode != 501 || response.TLS == nil || len(response.TLS.PeerCertificates) == 0 { + return 0, fmt.Errorf("unexpected native empty-store health response") + } + return response.TLS.PeerCertificates[0].SerialNumber.Int64(), nil + } + valid := client(hostname, pool, tls.VersionTLS12, tls.VersionTLS13) + deadline := time.Now().Add(15 * time.Second) + for { + serial, err := readHealth(valid) + if err == nil && serial == 2 { + break + } + if time.Now().After(deadline) { + t.Fatal("native declared TLS listener did not become available") + } + time.Sleep(50 * time.Millisecond) + } + for name, rejected := range map[string]*http.Client{ + "wrong hostname": client("wrong.invalid", pool, tls.VersionTLS12, tls.VersionTLS13), + "untrusted CA": client(hostname, x509.NewCertPool(), tls.VersionTLS12, tls.VersionTLS13), + "obsolete TLS": client(hostname, pool, tls.VersionTLS10, tls.VersionTLS11), + } { + t.Run(name, func(t *testing.T) { + if _, err := readHealth(rejected); err == nil { + t.Fatal("native listener accepted invalid TLS") + } + }) + } + writeLeaf(3) + if err := command.Process.Signal(syscall.SIGHUP); err != nil { + t.Fatal("native certificate reload signal failed") + } + deadline = time.Now().Add(5 * time.Second) + for { + serial, err := readHealth(valid) + if err == nil && serial == 3 { + break + } + if time.Now().After(deadline) { + t.Fatal("native listener did not reload its renewed certificate") + } + time.Sleep(50 * time.Millisecond) + } +} diff --git a/scripts/tests/test-openbao-standby-canary.sh b/scripts/tests/test-openbao-standby-canary.sh index 215eddedcc..561f07bfea 100755 --- a/scripts/tests/test-openbao-standby-canary.sh +++ b/scripts/tests/test-openbao-standby-canary.sh @@ -38,6 +38,33 @@ else fi [[ "${actual_sha}" == "${expected_sha}" ]] || fail 'the pinned OpenBao chart checksum changed' +# Exercise this exact server's listener parsing and SIGHUP reload behavior, +# without starting Kubernetes, initializing a vault or accessing credentials. +case "$(uname -s)/$(uname -m)" in + Linux/x86_64) + native_asset=openbao_2.6.3_linux_amd64.tar.gz + native_sha=c6463ddd4fdc4214b62a7ffdeaa0fc6df170f7e6b75c6dea2c5e525bdc932ed3 + ;; + Darwin/arm64) + native_asset=openbao_2.6.3_darwin_arm64.tar.gz + native_sha=ed17491ebc6415d075b7b2b2c0ef6b5908b84b161b8e5a50c7da303cd8d15df4 + ;; + *) fail 'the native TLS regression needs a reviewed server archive for this host' ;; +esac +curl --proto '=https' --proto-redir '=https' --location --tlsv1.2 --fail --silent --show-error --retry 2 --max-time 120 \ + "https://github.com/openbao/openbao/releases/download/v2.6.3/${native_asset}" \ + --output "${scratch}/native.tar.gz" +if command -v sha256sum >/dev/null 2>&1; then + actual_native_sha="$(sha256sum "${scratch}/native.tar.gz" | cut -d ' ' -f 1)" +else + actual_native_sha="$(shasum -a 256 "${scratch}/native.tar.gz" | cut -d ' ' -f 1)" +fi +[[ "${actual_native_sha}" == "${native_sha}" ]] || fail 'the pinned native OpenBao archive checksum changed' +mkdir "${scratch}/native" +tar -xzf "${scratch}/native.tar.gz" -C "${scratch}/native" +OPENBAO_TLS_TEST_BINARY="${scratch}/native/bao" go test \ + "${root_dir}/scripts/tests/openbao-transport-runtime" -count=1 + replicas="$(yq -er '.data.openbao_replicas' "${root_dir}/k8s/clusters/prod/bootstrap/config-map.yaml")" readonly replicas [[ "${replicas}" == '3' ]] || fail 'the OpenBao canary requires revalidation when replica count changes' diff --git a/scripts/validate-eks-ci-role-policy/approved-surface-history.md b/scripts/validate-eks-ci-role-policy/approved-surface-history.md index 1113717962..19e778f56c 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface-history.md +++ b/scripts/validate-eks-ci-role-policy/approved-surface-history.md @@ -3256,3 +3256,23 @@ No entry was added or removed. Its approved fingerprint moves from The source delta is the callback alone; no RBAC, IAM, service account, or permission grant changes. The authorization command passes with this one entry re-approved; its negative controls remain intact. + +## 2026-10-06 — Dedicated encrypted ARC credential transport + +The checksum-verified kubectl v1.36.2 / Kustomize v5.8.1 renderer measured +exactly one changed authorization entry for #4570, against the retained ARC +recovery at `2638277c`: HelmRelease openbao/openbao. Nothing was added or +removed from the selected authorization surface. Its fingerprint moves from +`068ad0ebbbc391a567ad211c7254130c0b949de8bec7a27f87fcab8ac034b452` to +`a3e829429965ed7f8c430424564624d176c1ae96be38a05db1bd38d018ebc1d4`. + +The reviewed source appends a dedicated TLS listener, certificate mounts and +same-image certificate reload helper. The existing server retains its API +identity through an explicit projection; automatic token injection is disabled +and the helper receives no credential mounts. The existing three Raft members, +partition 2, storage, audit, unseal hook and legacy listener are preserved. +The chart render and native OpenBao 2.6.3 TLS/reload regression exercise those +settings. No RBAC binding, IAM policy, reader role, App identity or App grant +changes. The required authorization gate and its negative controls remain +enabled. Production TLS, actual credential identity and untrusted workload +isolation remain separate protected runtime checks before ARC activation. diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index e3ec6d9d1e..0a6c9e5d4b 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -88,7 +88,7 @@ helm.toolkit.fluxcd.io/v2|HelmRelease|observability|coroot-operator a6c5d6dfdec7 helm.toolkit.fluxcd.io/v2|HelmRelease|open-feature-operator-system|open-feature-operator da38c30c19aea0c3e099bb86cda46d89ab6301ff5898b4fad8f020812f8ad800 -helm.toolkit.fluxcd.io/v2|HelmRelease|openbao|openbao 068ad0ebbbc391a567ad211c7254130c0b949de8bec7a27f87fcab8ac034b452 +helm.toolkit.fluxcd.io/v2|HelmRelease|openbao|openbao a3e829429965ed7f8c430424564624d176c1ae96be38a05db1bd38d018ebc1d4 helm.toolkit.fluxcd.io/v2|HelmRelease|policy-reporter|policy-reporter f92d67d39b5638fa931d19f4015ee490d6c98043e00a01319f4ef1a5fc64bc29 diff --git a/tests/policy-failure-actions.json b/tests/policy-failure-actions.json index bc65a4bbbf..890a87b041 100644 --- a/tests/policy-failure-actions.json +++ b/tests/policy-failure-actions.json @@ -1,4 +1,5 @@ { + "restrict-arc-openbao-certificate/only-openbao-may-use-credential-ca": "Enforce", "require-baseline-workers-for-hcloud-volumes/require-baseline-worker-affinity": "Enforce", "require-baseline-workers-for-hcloud-volumes/report-claim-lookup-failure": "Audit", "audit-privileged-rbac/privileged-rules": "Enforce", diff --git a/tests/restrict-arc-openbao-certificate/kyverno-test.yaml b/tests/restrict-arc-openbao-certificate/kyverno-test.yaml new file mode 100644 index 0000000000..e0426fd1c2 --- /dev/null +++ b/tests/restrict-arc-openbao-certificate/kyverno-test.yaml @@ -0,0 +1,39 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: restrict-arc-openbao-certificate +policies: + - ../../k8s/providers/hetzner/infrastructure/controllers/openbao/transport/certificate-policy.yaml +resources: + - resources.yaml +results: + - policy: restrict-arc-openbao-certificate + rule: only-openbao-may-use-credential-ca + resources: + - openbao/server-certificate + kind: Certificate + result: pass + - policy: restrict-arc-openbao-certificate + rule: only-openbao-may-use-credential-ca + resources: + - openbao/server-request + kind: CertificateRequest + result: pass + - policy: restrict-arc-openbao-certificate + rule: only-openbao-may-use-credential-ca + resources: + - arc-runners/impersonated-server + kind: Certificate + result: fail + - policy: restrict-arc-openbao-certificate + rule: only-openbao-may-use-credential-ca + resources: + - arc-runners/impersonated-request + kind: CertificateRequest + result: fail + - policy: restrict-arc-openbao-certificate + rule: only-openbao-may-use-credential-ca + resources: + - cert-manager/unrelated-ca + kind: Certificate + result: skip diff --git a/tests/restrict-arc-openbao-certificate/resources.yaml b/tests/restrict-arc-openbao-certificate/resources.yaml new file mode 100644 index 0000000000..507e6e8b77 --- /dev/null +++ b/tests/restrict-arc-openbao-certificate/resources.yaml @@ -0,0 +1,65 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: server-certificate + namespace: openbao +spec: + secretName: synthetic-server-tls + dnsNames: + - openbao-arc.openbao.svc.cluster.local + issuerRef: + name: arc-openbao-ca + kind: ClusterIssuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: CertificateRequest +metadata: + name: server-request + namespace: openbao +spec: + request: dGVzdA== + issuerRef: + name: arc-openbao-ca + kind: ClusterIssuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: impersonated-server + namespace: arc-runners +spec: + secretName: synthetic-impersonation-tls + dnsNames: + - openbao-arc.openbao.svc.cluster.local + issuerRef: + name: arc-openbao-ca + kind: ClusterIssuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: CertificateRequest +metadata: + name: impersonated-request + namespace: arc-runners +spec: + request: dGVzdA== + issuerRef: + name: arc-openbao-ca + kind: ClusterIssuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: unrelated-ca + namespace: cert-manager +spec: + secretName: synthetic-unrelated-ca + isCA: true + commonName: synthetic-unrelated-ca + issuerRef: + name: unrelated-selfsigned + kind: Issuer + group: cert-manager.io From 664809493b81fd5301ede50a1aff5f13db14c476 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 13:14:10 +0200 Subject: [PATCH 30/62] fix(arc): deploy certificate policy after admission controllers --- .../restrict-arc-openbao-certificate.yaml} | 0 .../controllers/openbao/transport/kustomization.yaml | 1 - k8s/providers/hetzner/infrastructure/kustomization.yaml | 3 +++ tests/restrict-arc-openbao-certificate/kyverno-test.yaml | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) rename k8s/providers/hetzner/infrastructure/{controllers/openbao/transport/certificate-policy.yaml => cluster-policies/restrict-arc-openbao-certificate.yaml} (100%) diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/certificate-policy.yaml b/k8s/providers/hetzner/infrastructure/cluster-policies/restrict-arc-openbao-certificate.yaml similarity index 100% rename from k8s/providers/hetzner/infrastructure/controllers/openbao/transport/certificate-policy.yaml rename to k8s/providers/hetzner/infrastructure/cluster-policies/restrict-arc-openbao-certificate.yaml diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/kustomization.yaml index a478feb504..0dd3c3bad7 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/kustomization.yaml @@ -10,4 +10,3 @@ resources: - listener-config-map.yaml - cilium-network-policy.yaml - cilium-network-policy-external-secrets.yaml - - certificate-policy.yaml diff --git a/k8s/providers/hetzner/infrastructure/kustomization.yaml b/k8s/providers/hetzner/infrastructure/kustomization.yaml index cd12975f9d..9fb24f71ab 100644 --- a/k8s/providers/hetzner/infrastructure/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/kustomization.yaml @@ -7,6 +7,9 @@ resources: # not drain an installed scale set; runtime acceptance remains a separate gate. - retained-ksail-analysis/ - arc-credential-transport/ + # Issuance restrictions must be active before credentials or runners are used. + # This layer follows installation of the Kyverno admission controller. + - cluster-policies/restrict-arc-openbao-certificate.yaml # Prod-only audit-log forwarder. It consumes the `coroot-api-key` generated # from the Hetzner Coroot CR patch, so it belongs in this infrastructure # layer alongside that CR rather than in the earlier wait-gated controllers diff --git a/tests/restrict-arc-openbao-certificate/kyverno-test.yaml b/tests/restrict-arc-openbao-certificate/kyverno-test.yaml index e0426fd1c2..ac431ea03a 100644 --- a/tests/restrict-arc-openbao-certificate/kyverno-test.yaml +++ b/tests/restrict-arc-openbao-certificate/kyverno-test.yaml @@ -3,7 +3,7 @@ kind: Test metadata: name: restrict-arc-openbao-certificate policies: - - ../../k8s/providers/hetzner/infrastructure/controllers/openbao/transport/certificate-policy.yaml + - ../../k8s/providers/hetzner/infrastructure/cluster-policies/restrict-arc-openbao-certificate.yaml resources: - resources.yaml results: From 08831aaf7410dc2f2330477bbf07d949efb6d1d1 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 13:27:18 +0200 Subject: [PATCH 31/62] fix(arc): follow manifest naming and patch placement rules --- .../hetzner/infrastructure/controllers/kustomization.yaml | 2 +- .../openbao/transport/{trust-bundle.yaml => bundle.yaml} | 0 .../{ca-certificate.yaml => certificate-ca.yaml} | 0 ...{listener-config-map.yaml => config-map-listener.yaml} | 0 .../transport/{ca-issuer.yaml => issuer-selfsigned.yaml} | 0 .../controllers/openbao/transport/kustomization.yaml | 8 ++++---- .../enable-arc-transport.yaml} | 0 scripts/tests/arc-staging/transport_test.go | 8 ++++---- scripts/tests/openbao-transport-runtime/tls_test.go | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) rename k8s/providers/hetzner/infrastructure/controllers/openbao/transport/{trust-bundle.yaml => bundle.yaml} (100%) rename k8s/providers/hetzner/infrastructure/controllers/openbao/transport/{ca-certificate.yaml => certificate-ca.yaml} (100%) rename k8s/providers/hetzner/infrastructure/controllers/openbao/transport/{listener-config-map.yaml => config-map-listener.yaml} (100%) rename k8s/providers/hetzner/infrastructure/controllers/openbao/transport/{ca-issuer.yaml => issuer-selfsigned.yaml} (100%) rename k8s/providers/hetzner/infrastructure/controllers/openbao/transport/{helm-release-patch.yaml => patches/enable-arc-transport.yaml} (100%) diff --git a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml index be4e07a9fb..d599e9eb3d 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml @@ -132,7 +132,7 @@ patches: - path: openbao/patches/keep-off-autoscale-nodes.yaml # Stage only one OpenBao replica; lower ordinals wait for healthy readback. - path: openbao/patches/standby-oidc-canary.yaml - - path: openbao/transport/helm-release-patch.yaml + - path: openbao/transport/patches/enable-arc-transport.yaml # Umami's Namespace moved into the preceding controllers layer with its # Kyverno Role; keep the prod-only user-namespace label with its new owner. - path: kyverno/patches/enable-user-namespaces.yaml diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/trust-bundle.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/bundle.yaml similarity index 100% rename from k8s/providers/hetzner/infrastructure/controllers/openbao/transport/trust-bundle.yaml rename to k8s/providers/hetzner/infrastructure/controllers/openbao/transport/bundle.yaml diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/ca-certificate.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/certificate-ca.yaml similarity index 100% rename from k8s/providers/hetzner/infrastructure/controllers/openbao/transport/ca-certificate.yaml rename to k8s/providers/hetzner/infrastructure/controllers/openbao/transport/certificate-ca.yaml diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/listener-config-map.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/config-map-listener.yaml similarity index 100% rename from k8s/providers/hetzner/infrastructure/controllers/openbao/transport/listener-config-map.yaml rename to k8s/providers/hetzner/infrastructure/controllers/openbao/transport/config-map-listener.yaml diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/ca-issuer.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/issuer-selfsigned.yaml similarity index 100% rename from k8s/providers/hetzner/infrastructure/controllers/openbao/transport/ca-issuer.yaml rename to k8s/providers/hetzner/infrastructure/controllers/openbao/transport/issuer-selfsigned.yaml diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/kustomization.yaml index 0dd3c3bad7..473187cfc7 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/kustomization.yaml @@ -1,12 +1,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ca-issuer.yaml - - ca-certificate.yaml + - issuer-selfsigned.yaml + - certificate-ca.yaml - cluster-issuer.yaml - certificate.yaml - - trust-bundle.yaml + - bundle.yaml - service.yaml - - listener-config-map.yaml + - config-map-listener.yaml - cilium-network-policy.yaml - cilium-network-policy-external-secrets.yaml diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/helm-release-patch.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/patches/enable-arc-transport.yaml similarity index 100% rename from k8s/providers/hetzner/infrastructure/controllers/openbao/transport/helm-release-patch.yaml rename to k8s/providers/hetzner/infrastructure/controllers/openbao/transport/patches/enable-arc-transport.yaml diff --git a/scripts/tests/arc-staging/transport_test.go b/scripts/tests/arc-staging/transport_test.go index 53381208d4..e7e66de8d4 100644 --- a/scripts/tests/arc-staging/transport_test.go +++ b/scripts/tests/arc-staging/transport_test.go @@ -59,7 +59,7 @@ func TestCredentialTLSHasNarrowCertificateAndTrust(t *testing.T) { if len(names) != 1 || names[0] != "openbao-arc.openbao.svc.cluster.local" { t.Fatal("certificate must bind the one credential service identity") } - bundle := readYAML(t, transportPath+"trust-bundle.yaml") + bundle := readYAML(t, transportPath+"bundle.yaml") equal(t, field(t, bundle, "metadata", "name"), "arc-openbao-ca") equal(t, field(t, bundle, "spec", "target", "configMap", "key"), "ca.crt") equal(t, field(t, bundle, "spec", "target", "namespaceSelector", "matchLabels", "kubernetes.io/metadata.name"), "arc-runners") @@ -72,7 +72,7 @@ func TestCredentialTLSHasNarrowCertificateAndTrust(t *testing.T) { } func TestCredentialTLSAppendsListenerWithoutReplacingStorage(t *testing.T) { - config := readYAML(t, transportPath+"listener-config-map.yaml") + config := readYAML(t, transportPath+"config-map-listener.yaml") text := field(t, config, "data", "listener.hcl").(string) for _, fragment := range []string{"listener \"tcp\"", "0.0.0.0:8204", "127.0.0.1:8205", "tls_min_version = \"tls12\"", "tls_cert_file", "tls_key_file"} { if !strings.Contains(text, fragment) { @@ -84,12 +84,12 @@ func TestCredentialTLSAppendsListenerWithoutReplacingStorage(t *testing.T) { t.Fatalf("supplemental listener changes existing server setting %s", forbidden) } } - patch := readYAML(t, transportPath+"helm-release-patch.yaml") + patch := readYAML(t, transportPath+"patches/enable-arc-transport.yaml") equal(t, field(t, patch, "spec", "values", "server", "extraArgs"), "-config=/openbao/arc-transport/listener.hcl") } func TestCertificateReloadDoesNotReceiveAPIOrKeyMounts(t *testing.T) { - patch := readYAML(t, transportPath+"helm-release-patch.yaml") + patch := readYAML(t, transportPath+"patches/enable-arc-transport.yaml") server := field(t, patch, "spec", "values", "server") renderers := field(t, patch, "spec", "postRenderers").([]any) patches := field(t, renderers[0], "kustomize", "patches").([]any) diff --git a/scripts/tests/openbao-transport-runtime/tls_test.go b/scripts/tests/openbao-transport-runtime/tls_test.go index 196de3f4e8..eef58c2e8d 100644 --- a/scripts/tests/openbao-transport-runtime/tls_test.go +++ b/scripts/tests/openbao-transport-runtime/tls_test.go @@ -97,7 +97,7 @@ func TestNativeTLSAndCertificateReload(t *testing.T) { if err := os.WriteFile(filepath.Join(root, "base.hcl"), []byte(base), 0600); err != nil { t.Fatal(err) } - data, err := os.ReadFile("../../../k8s/providers/hetzner/infrastructure/controllers/openbao/transport/listener-config-map.yaml") + data, err := os.ReadFile("../../../k8s/providers/hetzner/infrastructure/controllers/openbao/transport/config-map-listener.yaml") if err != nil { t.Fatal(err) } From be86d4c298856d12dcd6b2e62ebf2a291bea9917 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 13:49:07 +0200 Subject: [PATCH 32/62] test(arc): prove credential transport against same-node workloads --- .github/workflows/ci.yaml | 6 + .github/workflows/verify-arc-transport.yaml | 68 ++++++ docs/operations/arc-runners.md | 15 ++ scripts/tests/test-verify-arc-transport.sh | 118 ++++++++++ scripts/verify-arc-transport.sh | 226 ++++++++++++++++++++ scripts/verify-arc-transport/main.go | 166 ++++++++++++++ scripts/verify-arc-transport/proof_test.go | 88 ++++++++ 7 files changed, 687 insertions(+) create mode 100644 .github/workflows/verify-arc-transport.yaml create mode 100755 scripts/tests/test-verify-arc-transport.sh create mode 100755 scripts/verify-arc-transport.sh create mode 100644 scripts/verify-arc-transport/main.go create mode 100644 scripts/verify-arc-transport/proof_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 49e2299ced..c4299dbb18 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -149,6 +149,12 @@ jobs: - name: 🛑 Validate inactive ARC staging boundaries run: go test ./scripts/tests/arc-staging + - name: 🔐 Validate ARC transport proof boundaries + run: | + go test -race ./scripts/verify-arc-transport + shellcheck scripts/verify-arc-transport.sh scripts/tests/test-verify-arc-transport.sh + bash scripts/tests/test-verify-arc-transport.sh + # Shared Wedding backup access is retired on this proof's word, so it must # pass only on an AccessDenied for every access, after a positive control # and against an object the shared credential showed exists. These cases diff --git a/.github/workflows/verify-arc-transport.yaml b/.github/workflows/verify-arc-transport.yaml new file mode 100644 index 0000000000..6216ba0310 --- /dev/null +++ b/.github/workflows/verify-arc-transport.yaml @@ -0,0 +1,68 @@ +name: Verify ARC Credential Transport + +on: + workflow_dispatch: + +permissions: {} + +concurrency: + group: prod-deploy + cancel-in-progress: false + queue: max + +jobs: + verify: + name: Verify same-node credential transport isolation + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: prod + permissions: + contents: read + steps: + - name: Require the current reviewed main commit before credentials + shell: bash + env: + RUN_REF: ${{ github.ref }} + RUN_SHA: ${{ github.sha }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + [[ "$RUN_REF" == refs/heads/main ]] + [[ "$RUN_SHA" =~ ^[0-9a-f]{40}$ ]] + [[ "$(gh api repos/devantler-tech/platform/branches/main --jq .commit.sha)" == "$RUN_SHA" ]] + + - name: Checkout the reviewed dispatch commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: Restore the established production kubeconfig + shell: bash + env: + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + run: | + set -euo pipefail + umask 077 + mkdir -p ~/.kube + printf '%s' "$KUBE_CONFIG" > ~/.kube/config + + - name: Select the stable production API endpoint + shell: bash + env: + HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + run: bash scripts/use-prod-stable-api-endpoint.sh >/dev/null + + # Creates only bounded, unprivileged probe Pods, never an App credential, + # runner registration, policy, node or service-account TokenRequest. + - name: Prove native TLS health and intercepted same-node attempts + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: bash scripts/verify-arc-transport.sh --same-node diff --git a/docs/operations/arc-runners.md b/docs/operations/arc-runners.md index 35a95c0412..902edc5e6c 100644 --- a/docs/operations/arc-runners.md +++ b/docs/operations/arc-runners.md @@ -62,6 +62,21 @@ Those observations remain mandatory before credential materialization and ARC activation. The production deployment must confirm the listener and dedicated store; the protected identity verifier then establishes the actual stored App. +Dispatch `Verify ARC Credential Transport` from the current reviewed main commit +before that identity verification. It uses the established protected production +path and serializes with deployments. The challenge verifies the dedicated +authority, hostname and actual initialized, unsealed canary, then places an +unprivileged, token-free probe on each current OpenBao-canary and secret-controller +node. Both legacy HTTP and native TLS connection attempts must time out and have +a corresponding Hubble `DROPPED / POLICY_DENIED` flow for the exact Pod, addresses, +node, port and attempt window. A certificate error, unreachable healthy control, +observer loss or changed workload identity fails the proof. Server admission +must reject privileged, host-volume, host-network, host-process and `NET_RAW` +variants. Only the owned probe Pods are created; UID-preconditioned deletion and +absence readback are mandatory. This workflow reads no App key and requests no +reader token. Its counts-only result is transport evidence, not stored-key, +runner-registration or managed-analysis evidence. + Activation requires a separate reviewed change and all of these proofs: 1. Reuse the production platform App used for GitHub sign-in, identified by diff --git a/scripts/tests/test-verify-arc-transport.sh b/scripts/tests/test-verify-arc-transport.sh new file mode 100755 index 0000000000..4cdb3db9b4 --- /dev/null +++ b/scripts/tests/test-verify-arc-transport.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# No credentials or cluster: execute the real orchestration and evidence parser +# against bounded API/exec fixtures, including failure and replacement races. +set -euo pipefail +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/bin" "$scratch/state" +go build -o "$scratch/evidence" ./scripts/verify-arc-transport +export PROOF_TEST_ROOT="$scratch" +readonly head=1111111111111111111111111111111111111111 +export GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform GITHUB_REF=refs/heads/main GITHUB_SHA="$head" GITHUB_RUN_ID=12345 GITHUB_RUN_ATTEMPT=1 +yq -o=json '.spec.rules[0]' k8s/providers/hetzner/infrastructure/cluster-policies/restrict-arc-openbao-certificate.yaml \ + | jq '{spec:{rules:[.]},status:{conditions:[{type:"Ready",status:"True"}]}}' >"$scratch/policy.json" +cat >"$scratch/bin/gh" <<'MOCK' +#!/usr/bin/env bash +printf '%s\n' "${GITHUB_SHA}" +MOCK +cat >"$scratch/bin/go" <<'MOCK' +#!/usr/bin/env bash +while [[ "$#" -gt 0 ]]; do + if [[ "$1" == -o ]]; then cp "$PROOF_TEST_ROOT/evidence" "$2"; exit; fi + shift +done +exit 1 +MOCK +cat >"$scratch/bin/curl" <<'MOCK' +#!/usr/bin/env bash +if [[ "${PROOF_CASE:-}" == tls_error ]]; then exit 60; fi +if [[ "${PROOF_CASE:-}" == sealed ]]; then printf '{"initialized":true,"sealed":true,"version":"2.6.3","cluster_id":"fixture"}'; exit; fi +printf '{"initialized":true,"sealed":false,"version":"2.6.3","cluster_id":"fixture"}' +MOCK +cat >"$scratch/bin/kubectl" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail +state="$PROOF_TEST_ROOT/state" +printf '%s\n' "$*" >>"$state/calls" +while [[ "$1" == --context* || "$1" == --request-timeout* ]]; do + if [[ "$1" == --context ]]; then shift 2; else shift; fi +done +ns='' +if [[ "$1" == -n ]]; then ns=$2; shift 2; fi +args="$*" +case "$args" in + 'port-forward '*) printf 'Forwarding from 127.0.0.1:38204 -> 8204\n'; exec sleep 300 ;; + 'get externalsecrets '*) printf '{"items":[]}' ;; + 'get helmreleases '*) printf '{"items":[]}' ;; + 'get secretstore '*) jq -n '{spec:{provider:{vault:{server:"https://openbao-arc.openbao.svc.cluster.local:8204",caProvider:{type:"ConfigMap",name:"arc-openbao-ca",key:"ca.crt"},path:"secret",version:"v2",auth:{kubernetes:{mountPath:"kubernetes",role:"arc-secret-reader",serviceAccountRef:{name:"arc-secret-reader"}}}}}}}' ;; + 'get configmap arc-openbao-ca '*) printf '{"data":{"ca.crt":"-----BEGIN CERTIFICATE-----fixture"}}' ;; + 'get configmap cilium-config '*) printf '{"data":{"cluster-name":"prod"}}' ;; + 'get clusterpolicy '*) cat "$PROOF_TEST_ROOT/policy.json" ;; + 'get kustomization '*) printf '{"metadata":{"generation":1},"status":{"observedGeneration":1,"lastAppliedRevision":"latest@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","conditions":[{"type":"Ready","status":"True"}]}}' ;; + 'get service '*) printf '{"spec":{"selector":{"app.kubernetes.io/name":"openbao","app.kubernetes.io/instance":"openbao","statefulset.kubernetes.io/pod-name":"openbao-2"},"ports":[{"name":"arc-tls","port":8204,"targetPort":8204,"protocol":"TCP"}]}}' ;; + 'get node '*) printf '{"metadata":{"uid":"node-uid","labels":{}},"status":{"conditions":[{"type":"Ready","status":"True"}]}}' ;; + 'get --raw '*) printf '{"initialized":true,"sealed":false,"version":"2.6.3","cluster_id":"fixture"}' ;; + 'get pods '*arc-role=runner*) : ;; + 'get pods '*arc-transport-probe*) + if [[ "${PROOF_CASE:-}" == stale_probe && ! -e "$state/created" ]]; then printf '{"items":[{}]}'; else printf '{"items":[]}'; fi ;; + 'get pods '*external-secrets*) printf '{"items":[{"metadata":{"uid":"eso-uid"},"spec":{"nodeName":"node-a"},"status":{"podIP":"10.1.1.3","conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"eso","ready":true,"restartCount":0,"containerID":"containerd://eso"}]}}]}' ;; + 'get pods '*k8s-app=cilium*) printf '{"items":[{"metadata":{"name":"cilium-a","uid":"agent-uid"},"spec":{"nodeName":"node-a"},"status":{"podIP":"10.1.1.4","conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"cilium-agent","ready":true,"restartCount":0,"containerID":"containerd://agent"}]}}]}' ;; + 'get pod openbao-2 '*) printf '{"metadata":{"uid":"bao-uid"},"spec":{"nodeName":"node-a","containers":[{"name":"openbao"},{"name":"arc-tls-reload"}]},"status":{"podIP":"10.1.1.2","conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"openbao","ready":true,"restartCount":0,"containerID":"containerd://bao"},{"name":"arc-tls-reload","ready":true,"restartCount":0,"containerID":"containerd://reload"}]}}' ;; + 'get pod cilium-a '*) printf '{"metadata":{"uid":"agent-uid"},"spec":{"nodeName":"node-a"},"status":{"podIP":"10.1.1.4","conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"cilium-agent","ready":true,"restartCount":0,"containerID":"containerd://agent"}]}}' ;; + 'get pod arc-transport-'*) + [[ -e "$state/created" ]] || exit 0 + if [[ "${PROOF_CASE:-}" == replacement && -e "$state/attempt" ]]; then jq '.metadata.uid="replacement-uid"' "$state/pod.json"; else cat "$state/pod.json"; fi ;; + 'create --dry-run=server '*) + file=${args##* -f } + if jq -e '.spec.containers[0].securityContext.privileged == true' "$file" >/dev/null; then problem=privileged + elif jq -e 'any(.spec.volumes[]; .hostPath != null)' "$file" >/dev/null; then problem=hostPath + elif jq -e '.spec.hostNetwork == true' "$file" >/dev/null; then problem=hostNetwork + elif jq -e '.spec.hostPID == true' "$file" >/dev/null; then problem=hostPID + elif jq -e '.spec.containers[0].securityContext.capabilities.add == ["NET_RAW"]' "$file" >/dev/null; then problem=NET_RAW + else exit 0; fi + [[ "${PROOF_CASE:-}" != admit_privilege ]] || exit 0 + if [[ "${PROOF_CASE:-}" == api_failure ]]; then printf 'connection refused\n' >&2; else printf 'violates PodSecurity restricted: %s\n' "$problem" >&2; fi + exit 1 ;; + 'create -f '*) + file=$3 + jq '.metadata.uid="probe-uid" | .status={podIP:"10.1.1.1",containerStatuses:[{name:"probe",ready:true,restartCount:0,containerID:"containerd://probe"}]}' "$file" >"$state/pod.json" + if [[ "${PROOF_CASE:-}" == wrong_node ]]; then jq '.spec.nodeName="other"' "$state/pod.json" >"$state/changed"; mv "$state/changed" "$state/pod.json"; fi + touch "$state/created"; cat "$state/pod.json" ;; + 'wait '*) : ;; + 'exec '*hubble\ status*) : ;; + 'exec '*hubble\ observe*) + while [[ "$#" -gt 0 ]]; do + case "$1" in --since) when=$2;; --to-port) port=$2;; esac; shift + done + reason=POLICY_DENIED; [[ "${PROOF_CASE:-}" != wrong_reason ]] || reason=CT_TRUNCATED_OR_INVALID_HEADER + jq -n --arg reason "$reason" --arg when "$when" --argjson port "$port" '{flow:{verdict:"DROPPED",drop_reason_desc:$reason,traffic_direction:"EGRESS",time:$when,node_name:"prod/node-a",source:{namespace:"arc-runners",pod_name:"arc-transport-12345-1-1"},destination:{namespace:"openbao",pod_name:"openbao-2"},IP:{source:"10.1.1.1",destination:"10.1.1.2"},l4:{TCP:{destination_port:$port,flags:{SYN:true}}}}}' + [[ "${PROOF_CASE:-}" != lost_events ]] || printf '\n{"lost_events":{}}\n' ;; + 'exec '*curl*) touch "$state/attempt"; [[ "${PROOF_CASE:-}" != reachable ]] || exit 0; [[ "${PROOF_CASE:-}" != cert_error ]] || exit 60; exit 28 ;; + 'exec '*test*) : ;; + 'delete --raw='*) + [[ "${PROOF_CASE:-}" != cleanup_failure ]] || exit 1 + jq -e '.preconditions.uid == "probe-uid"' "${args##* -f }" >/dev/null + rm -f "$state/created" ;; + *) printf 'unexpected fixture command\n' >&2; exit 99 ;; +esac +MOCK +chmod +x "$scratch/bin/gh" "$scratch/bin/go" "$scratch/bin/curl" "$scratch/bin/kubectl" +export PATH="$scratch/bin:$PATH" +cases=0 +run_case() { + local name=$1 expected=$2 result=0 + rm -f "$scratch/state/"* + PROOF_CASE=$name bash scripts/verify-arc-transport.sh --same-node >"$scratch/output" 2>"$scratch/error" || result=$? + [[ "$result" == "$expected" ]] || { printf 'FAIL %s: exit %s expected %s\n' "$name" "$result" "$expected"; cat "$scratch/error"; exit 1; } + # No secret, TokenRequest, JIT, host exec, policy change or node deletion is permitted. + if [[ -f "$scratch/state/calls" ]] && grep -Eqi 'get secret |tokenrequest|create token|ephemeralrunner|delete node|patch |apply ' "$scratch/state/calls"; then exit 1; fi + if [[ "$name" == replacement ]] && grep -q 'delete --raw=' "$scratch/state/calls"; then printf 'FAIL replacement delete\n'; exit 1; fi + cases=$((cases+1)) +} +GITHUB_REF=refs/heads/feature run_case unreviewed_ref 1 +run_case healthy_denied 0 +run_case tls_error 60 +for name in sealed stale_probe admit_privilege api_failure wrong_node reachable cert_error wrong_reason lost_events; do run_case "$name" 1; done +run_case replacement 4 +run_case cleanup_failure 4 +printf 'PASS: %s transport orchestration cases\n' "$cases" diff --git a/scripts/verify-arc-transport.sh b/scripts/verify-arc-transport.sh new file mode 100755 index 0000000000..55b7d83fd5 --- /dev/null +++ b/scripts/verify-arc-transport.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash +# Protected-main-only challenge; never reads an App key, Secret or runner JIT data. +set -euo pipefail +umask 077 +fail() { printf 'ARC transport: FAIL at %s\n' "$1" >&2; exit 1; } +[[ "$#" == 1 && "$1" == --same-node ]] || fail arguments +[[ "${GITHUB_ACTIONS:-}" == true && "${GITHUB_REPOSITORY:-}" == devantler-tech/platform && + "${GITHUB_REF:-}" == refs/heads/main && "${GITHUB_SHA:-}" =~ ^[0-9a-f]{40}$ && + "${GITHUB_RUN_ID:-}" =~ ^[1-9][0-9]{0,19}$ && "${GITHUB_RUN_ATTEMPT:-}" =~ ^[1-9][0-9]{0,2}$ ]] || fail identity +current_main() { [[ "$(timeout 20s gh api repos/devantler-tech/platform/branches/main --jq .commit.sha 2>/dev/null)" == "$GITHUB_SHA" ]] || fail moved-main; } +current_main +readonly context=admin@prod namespace=arc-runners host=openbao-arc.openbao.svc.cluster.local +readonly label=platform.devantler.tech/arc-transport-probe +readonly image=ghcr.io/devantler-tech/ksail-analysis-runner@sha256:1ab01644fd6f67e0b1ab0b456e10b78f58be92ac67e5e06d103cd8c12bbb119b +scratch=$(mktemp -d) +readonly scratch +probe='' uid='' created=false forward_pid='' +stage=initialization +kc() { timeout 35s kubectl --context "$context" --request-timeout=30s "$@" 2>"$scratch/kubectl-error"; } +quiet() { "$@" >"$scratch/command-output" 2>"$scratch/command-error"; } +identity() { jq -ce '{uid:.metadata.uid,ip:.status.podIP,node:.spec.nodeName,containers:[.status.containerStatuses[]|{name,containerID,restartCount,ready}]}'; } +cleanup_probe() { + local now + "$created" || return 0 + [[ -n "$uid" ]] || return 1 + now=$(kc -n "$namespace" get pod "$probe" --ignore-not-found -o json) || return 1 + if [[ -n "$now" ]]; then + jq -e --arg uid "$uid" --arg key "$label" --arg owner "$probe" \ + '.metadata.uid == $uid and .metadata.labels[$key] == $owner' <<<"$now" >/dev/null || return 1 + jq -n --arg uid "$uid" '{apiVersion:"v1",kind:"DeleteOptions",preconditions:{uid:$uid},gracePeriodSeconds:5}' >"$scratch/delete.json" + quiet kc delete --raw="/api/v1/namespaces/$namespace/pods/$probe" -f "$scratch/delete.json" || return 1 + quiet kc -n "$namespace" wait "pod/$probe" --for=delete --timeout=25s || return 1 + fi + now=$(kc -n "$namespace" get pod "$probe" --ignore-not-found -o json) || return 1 + [[ -z "$now" ]] || return 1 + kc -n "$namespace" get pods -l "$label=$probe" -o json | jq -e '.items | length == 0' >/dev/null || return 1 + created=false +} +cleanup() { + local result=$? failed=0 + trap - EXIT ERR INT TERM + if [[ -n "$forward_pid" ]]; then kill "$forward_pid" 2>/dev/null || true; wait "$forward_pid" 2>/dev/null || true; fi + cleanup_probe || failed=1 + rm -rf "$scratch" + if [[ "$failed" != 0 ]]; then printf 'ARC transport: FAIL cleanup\n' >&2; exit 4; fi + printf 'ARC transport: owned probe cleanup verified\n' + exit "$result" +} +trap cleanup EXIT +trap 'printf "ARC transport: FAIL at %s\n" "$stage" >&2' ERR +trap 'exit 143' INT TERM +go build -trimpath -o "$scratch/evidence" ./scripts/verify-arc-transport >"$scratch/build-output" 2>"$scratch/build-error" + +stage=inactive-credential-boundary +# Inspect only metadata counts: neither existing App credentials nor JIT data is read. +kc -n "$namespace" get externalsecrets -o json | jq -e '.items | length == 0' >/dev/null +kc -n "$namespace" get helmreleases -o json | jq -e 'all(.items[]; .spec.suspend == true)' >/dev/null +kc -n "$namespace" get pods -l platform.devantler.tech/arc-role=runner -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}' >"$scratch/runners" +[[ ! -s "$scratch/runners" ]] || fail active-runner +kc -n "$namespace" get pods -l "$label" -o json | jq -e '.items | length == 0' >/dev/null +kc -n "$namespace" get secretstore openbao -o json >"$scratch/store.json" +jq -e --arg server "https://$host:8204" '.spec.provider.vault.server == $server and + .spec.provider.vault.caProvider == {type:"ConfigMap",name:"arc-openbao-ca",key:"ca.crt"} and + .spec.provider.vault.path == "secret" and .spec.provider.vault.version == "v2" and + .spec.provider.vault.auth.kubernetes == {mountPath:"kubernetes",role:"arc-secret-reader",serviceAccountRef:{name:"arc-secret-reader"}}' "$scratch/store.json" >/dev/null +kc -n "$namespace" get configmap arc-openbao-ca -o json >"$scratch/ca.json" +jq -er '.data["ca.crt"] | select(startswith("-----BEGIN CERTIFICATE-----"))' "$scratch/ca.json" >"$scratch/ca.pem" +kc get clusterpolicy restrict-arc-openbao-certificate -o json >"$scratch/policy.json" +yq -o=json '.spec.rules[0]' k8s/providers/hetzner/infrastructure/cluster-policies/restrict-arc-openbao-certificate.yaml >"$scratch/rule.json" +jq -e --slurpfile rule "$scratch/rule.json" '(.spec.rules | length) == 1 and + .spec.rules[0].match == $rule[0].match and .spec.rules[0].preconditions == $rule[0].preconditions and + .spec.rules[0].validate.deny == $rule[0].validate.deny and .spec.rules[0].validate.failureAction == "Enforce" and + any(.status.conditions[]; .type == "Ready" and .status == "True")' "$scratch/policy.json" >/dev/null +for layer in infrastructure infrastructure-controllers; do + kc -n flux-system get kustomization "$layer" -o json | jq -ce 'select(.status.observedGeneration == .metadata.generation and + any(.status.conditions[]; .type == "Ready" and .status == "True")) | .status.lastAppliedRevision | select(test("^latest@sha256:[0-9a-f]{64}$"))' >"$scratch/$layer-revision" +done +cmp -s "$scratch/infrastructure-revision" "$scratch/infrastructure-controllers-revision" || fail split-revision + +stage=healthy-targets +kc -n openbao get pod openbao-2 -o json >"$scratch/bao.json" +jq -e '.metadata.uid != null and .status.podIP != null and .spec.nodeName != null and + any(.status.conditions[]; .type == "Ready" and .status == "True") and + (.status.containerStatuses | length) == 2 and all(.status.containerStatuses[]; .ready == true and .containerID != "") and + any(.spec.containers[]; .name == "arc-tls-reload")' "$scratch/bao.json" >/dev/null +bao_ip=$(jq -er '.status.podIP' "$scratch/bao.json") +bao_node=$(jq -er '.spec.nodeName' "$scratch/bao.json") +kc -n openbao get service openbao-arc -o json | jq -e '.spec.selector == {"app.kubernetes.io/name":"openbao","app.kubernetes.io/instance":"openbao","statefulset.kubernetes.io/pod-name":"openbao-2"} and + .spec.ports == [{name:"arc-tls",port:8204,protocol:"TCP",targetPort:8204}]' >/dev/null +kc -n external-secrets get pods -l app.kubernetes.io/name=external-secrets -o json >"$scratch/eso.json" +jq -e '(.items | length) > 0 and all(.items[]; .metadata.uid != null and .spec.nodeName != null and + any(.status.conditions[]; .type == "Ready" and .status == "True") and + all(.status.containerStatuses[]; .ready == true and .containerID != ""))' "$scratch/eso.json" >/dev/null +{ printf '%s\n' "$bao_node"; jq -r '.items[].spec.nodeName' "$scratch/eso.json"; } | sort -u >"$scratch/nodes" +[[ "$(wc -l <"$scratch/nodes")" -le 4 ]] || fail exposure-bound +kubectl --context "$context" --request-timeout=30s -n openbao port-forward --address=127.0.0.1 pod/openbao-2 :8204 >"$scratch/forward" 2>"$scratch/forward-error" & +forward_pid=$! +for _ in {1..30}; do + kill -0 "$forward_pid" 2>/dev/null || fail port-forward + port=$(sed -nE 's/^Forwarding from 127\.0\.0\.1:([0-9]+) -> 8204$/\1/p' "$scratch/forward") + [[ "$port" =~ ^[0-9]+$ ]] && break + sleep 1 +done +[[ "$port" =~ ^[0-9]+$ ]] || fail port-forward +healthy() { + curl --noproxy '*' --proto '=https' --tlsv1.2 --cacert "$scratch/ca.pem" --resolve "$host:$port:127.0.0.1" \ + --fail --silent --show-error --max-time 15 "https://$host:$port/v1/sys/health?standbyok=true" >"$scratch/tls-health" 2>"$scratch/tls-error" + "$scratch/evidence" health <"$scratch/tls-health" + kc get --raw '/api/v1/namespaces/openbao/pods/openbao-2:8200/proxy/v1/sys/health?standbyok=true' >"$scratch/http-health" + "$scratch/evidence" health <"$scratch/http-health" + [[ "$(jq -r .cluster_id "$scratch/tls-health")" == "$(jq -r .cluster_id "$scratch/http-health")" ]] || fail split-health +} +healthy +cluster=$(kc -n kube-system get configmap cilium-config -o json | jq -er '.data["cluster-name"] | select(length > 0)') +count=0 +while IFS= read -r node; do + stage=same-node-challenge + count=$((count+1)) + probe="arc-transport-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$count" + kc get node "$node" -o json >"$scratch/node.json" + jq -e '.metadata.uid != null and .metadata.labels["platform.devantler.tech/ci-runner"] != "enabled" and + any(.status.conditions[]; .type == "Ready" and .status == "True")' "$scratch/node.json" >/dev/null + kc -n kube-system get pods -l k8s-app=cilium --field-selector "spec.nodeName=$node" -o json \ + | jq -e 'if (.items | length) == 1 then .items[0] else error("observer count") end | + select(any(.status.conditions[]; .type == "Ready" and .status == "True"))' >"$scratch/agent.json" + agent=$(jq -er '.metadata.name' "$scratch/agent.json") + quiet kc -n kube-system exec "$agent" -c cilium-agent -- hubble status --server unix:///var/run/cilium/hubble.sock --timeout 5s --request-timeout 5s --output json + jq -n --arg probe "$probe" --arg label "$label" --arg node "$node" --arg image "$image" \ + '{apiVersion:"v1",kind:"Pod",metadata:{name:$probe,namespace:"arc-runners",labels:{($label):$probe}},spec:{ + nodeName:$node,automountServiceAccountToken:false,restartPolicy:"Never",activeDeadlineSeconds:300, + securityContext:{runAsNonRoot:true,runAsUser:1001,runAsGroup:1001,seccompProfile:{type:"RuntimeDefault"}}, + containers:[{name:"probe",image:$image,command:["/bin/sh","-ec","test ! -e /var/run/secrets/kubernetes.io/serviceaccount/token; grep -Eq \u0027^CapEff:[[:space:]]+0+$\u0027 /proc/self/status; printf ready > /tmp/ready; sleep 260"], + readinessProbe:{exec:{command:["/bin/sh","-ec","test -f /tmp/ready"]},periodSeconds:2}, + securityContext:{privileged:false,allowPrivilegeEscalation:false,readOnlyRootFilesystem:true,capabilities:{drop:["ALL"]}}, + resources:{requests:{cpu:"10m",memory:"32Mi"},limits:{cpu:"100m",memory:"128Mi"}},volumeMounts:[{name:"tmp",mountPath:"/tmp"}]}], + volumes:[{name:"tmp",emptyDir:{sizeLimit:"16Mi"}}]}}' >"$scratch/pod.json" + quiet kc create --dry-run=server -f "$scratch/pod.json" + for negative in privileged hostpath hostnetwork hostpid netraw; do + case "$negative" in + privileged) jq '.spec.containers[0].securityContext.privileged=true' "$scratch/pod.json" >"$scratch/negative.json" ;; + hostpath) jq '.spec.volumes += [{name:"host",hostPath:{path:"/"}}]' "$scratch/pod.json" >"$scratch/negative.json" ;; + hostnetwork) jq '.spec.hostNetwork=true' "$scratch/pod.json" >"$scratch/negative.json" ;; + hostpid) jq '.spec.hostPID=true' "$scratch/pod.json" >"$scratch/negative.json" ;; + netraw) jq '.spec.containers[0].securityContext.capabilities.add=["NET_RAW"]' "$scratch/pod.json" >"$scratch/negative.json" ;; + esac + if quiet kc create --dry-run=server -f "$scratch/negative.json"; then fail negative-admission; fi + grep -Eq 'violates PodSecurity.*restricted|admission webhook.*denied the request' "$scratch/kubectl-error" || fail admission-transport-failure + case "$negative" in + privileged) grep -qi privileged "$scratch/kubectl-error" ;; hostpath) grep -Eqi 'hostPath|host.path' "$scratch/kubectl-error" ;; + hostnetwork) grep -qi hostNetwork "$scratch/kubectl-error" ;; netraw) grep -qi NET_RAW "$scratch/kubectl-error" ;; + hostpid) grep -qi hostPID "$scratch/kubectl-error" ;; + esac + done + created=true + kc create -f "$scratch/pod.json" -o json | jq -er --arg name "$probe" --arg label "$label" \ + 'select(.metadata.name == $name and .metadata.namespace == "arc-runners" and .metadata.labels[$label] == $name) | + .metadata.uid | select(type == "string" and length > 0)' >"$scratch/uid" + uid=$(cat "$scratch/uid") + quiet timeout 130s kubectl --context "$context" --request-timeout=120s -n "$namespace" wait "pod/$probe" --for=condition=Ready --timeout=90s + kc -n "$namespace" get pod "$probe" -o json >"$scratch/probe.json" + jq -e --arg uid "$uid" --arg node "$node" --arg image "$image" \ + '.metadata.uid == $uid and .spec.nodeName == $node and .spec.automountServiceAccountToken == false and + (.spec.hostNetwork // false) == false and (.spec.hostPID // false) == false and (.spec.hostIPC // false) == false and + (.spec.containers | length) == 1 and .spec.containers[0].image == $image and + .spec.containers[0].securityContext.capabilities.drop == ["ALL"] and (.spec.containers[0].securityContext.capabilities.add // [] | length) == 0 and + .spec.containers[0].securityContext.privileged == false and .spec.containers[0].securityContext.allowPrivilegeEscalation == false and + .spec.containers[0].securityContext.readOnlyRootFilesystem == true and + .spec.volumes == [{name:"tmp",emptyDir:{sizeLimit:"16Mi"}}] and (.spec.initContainers // [] | length) == 0 and + (.spec.ephemeralContainers // [] | length) == 0 and (.spec.shareProcessNamespace // false) == false and + .spec.securityContext.runAsNonRoot == true and .spec.securityContext.runAsUser == 1001 and + .spec.securityContext.runAsGroup == 1001 and .spec.securityContext.seccompProfile.type == "RuntimeDefault" and + (.spec.containers[0].env // [] | length) == 0 and (.spec.containers[0].envFrom // [] | length) == 0 and + .spec.containers[0].volumeMounts == [{name:"tmp",mountPath:"/tmp"}] and + (.status.containerStatuses | length) == 1 and .status.containerStatuses[0].ready == true and + .status.containerStatuses[0].restartCount == 0 and .status.containerStatuses[0].containerID != ""' "$scratch/probe.json" >/dev/null + source_ip=$(jq -er '.status.podIP' "$scratch/probe.json") + for destination_port in 8200 8204; do + healthy + start=$(date -u +%Y-%m-%dT%H:%M:%S.%NZ) + scheme=http; [[ "$destination_port" == 8204 ]] && scheme=https + if quiet kc -n "$namespace" exec "$probe" -c probe -- curl --noproxy '*' --connect-timeout 3 --max-time 5 \ + --silent --show-error --output /dev/null "$scheme://$bao_ip:$destination_port/v1/sys/health?standbyok=true"; then fail reachable-openbao; else result=$?; fi + [[ "$result" == 28 ]] || fail unproven-network-timeout + quiet kc -n "$namespace" exec "$probe" -c probe -- test -f /tmp/ready + end=$(date -u +%Y-%m-%dT%H:%M:%S.%NZ) + kc -n kube-system exec "$agent" -c cilium-agent -- hubble observe --server unix:///var/run/cilium/hubble.sock --timeout 5s \ + --since "$start" --until "$end" --from-ip "$source_ip" --from-pod "$namespace/$probe" --to-ip "$bao_ip" \ + --to-port "$destination_port" --protocol tcp --verdict DROPPED --drop-reason-desc POLICY_DENIED --output jsonpb >"$scratch/flows" + [[ ! -s "$scratch/kubectl-error" ]] || fail observer-diagnostics + "$scratch/evidence" denial --pod "$probe" --source "$source_ip" --destination "$bao_ip" --port "$destination_port" \ + --node "$cluster/$node" --since "$start" --until "$end" <"$scratch/flows" + healthy + done + for binding in 'arc-runners probe' 'kube-system agent' 'openbao bao'; do + read -r ns object <<<"$binding" + name=$probe; [[ "$object" == agent ]] && name=$agent; [[ "$object" == bao ]] && name=openbao-2 + identity <"$scratch/$object.json" >"$scratch/before" + kc -n "$ns" get pod "$name" -o json | identity >"$scratch/after" + cmp -s "$scratch/before" "$scratch/after" || fail workload-churn + done + kc get node "$node" -o json | jq -ce '{uid:.metadata.uid,ready:[.status.conditions[]|select(.type=="Ready")|.status]}' >"$scratch/after" + jq -ce '{uid:.metadata.uid,ready:[.status.conditions[]|select(.type=="Ready")|.status]}' "$scratch/node.json" >"$scratch/before" + cmp -s "$scratch/before" "$scratch/after" || fail node-churn + cleanup_probe || { printf 'ARC transport: FAIL cleanup\n' >&2; exit 4; } +done <"$scratch/nodes" +stage=final-bindings +for object in 'secretstore openbao' 'configmap arc-openbao-ca'; do + read -r kind name <<<"$object" + kc -n "$namespace" get "$kind" "$name" -o json | jq -Sc '{uid:.metadata.uid,spec:.spec,data:.data}' >"$scratch/after" + original=store; [[ "$kind" == configmap ]] && original=ca + jq -Sc '{uid:.metadata.uid,spec:.spec,data:.data}' "$scratch/$original.json" >"$scratch/before" + cmp -s "$scratch/before" "$scratch/after" || fail transport-churn +done +kc get clusterpolicy restrict-arc-openbao-certificate -o json | jq -Sc '{uid:.metadata.uid,spec:.spec}' >"$scratch/after" +jq -Sc '{uid:.metadata.uid,spec:.spec}' "$scratch/policy.json" >"$scratch/before" +cmp -s "$scratch/before" "$scratch/after" || fail issuance-policy-churn +kc -n external-secrets get pods -l app.kubernetes.io/name=external-secrets -o json | jq -Sc '[.items[]|{uid:.metadata.uid,ip:.status.podIP,node:.spec.nodeName,containers:.status.containerStatuses}] | sort_by(.uid)' >"$scratch/after" +jq -Sc '[.items[]|{uid:.metadata.uid,ip:.status.podIP,node:.spec.nodeName,containers:.status.containerStatuses}] | sort_by(.uid)' "$scratch/eso.json" >"$scratch/before" +cmp -s "$scratch/before" "$scratch/after" || fail eso-churn +for layer in infrastructure infrastructure-controllers; do + kc -n flux-system get kustomization "$layer" -o json | jq -ce 'select(.status.observedGeneration == .metadata.generation and + any(.status.conditions[]; .type == "Ready" and .status == "True")) | .status.lastAppliedRevision' >"$scratch/after" + cmp -s "$scratch/$layer-revision" "$scratch/after" || fail revision-churn +done +current_main +printf 'PASS: verified native TLS health and %s same-node HTTP/TLS policy-denial pairs; no App key read\n' "$count" diff --git a/scripts/verify-arc-transport/main.go b/scripts/verify-arc-transport/main.go new file mode 100644 index 0000000000..70494f100e --- /dev/null +++ b/scripts/verify-arc-transport/main.go @@ -0,0 +1,166 @@ +// Protected CI's bounded evidence parser. It prints no health body, address or flow. +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/netip" + "os" + "time" +) + +type target struct { + pod, source, destination, node string + port uint16 + since, until time.Time +} + +type endpoint struct { + Namespace string `json:"namespace"` + Pod string `json:"pod_name"` +} +type flow struct { + Verdict string `json:"verdict"` + Reason string `json:"drop_reason_desc"` + Direction string `json:"traffic_direction"` + Source endpoint `json:"source"` + Destination endpoint `json:"destination"` + IP struct { + Source string `json:"source"` + Destination string `json:"destination"` + } `json:"IP"` + L4 struct { + TCP struct { + Port uint16 `json:"destination_port"` + Flags struct { + SYN bool `json:"SYN"` + } `json:"flags"` + } `json:"TCP"` + } `json:"l4"` + Node string `json:"node_name"` + Time string `json:"time"` +} + +func verifyHealth(input io.Reader) error { + bounded := &io.LimitedReader{R: input, N: 1 << 20} + decoder := json.NewDecoder(bounded) + var health struct { + Initialized *bool `json:"initialized"` + Sealed *bool `json:"sealed"` + Version string `json:"version"` + Cluster string `json:"cluster_id"` + } + if err := decoder.Decode(&health); err != nil { + return errors.New("invalid health response") + } + var extra any + if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) || bounded.N <= 0 { + return errors.New("incomplete health response") + } + if health.Initialized == nil || !*health.Initialized || health.Sealed == nil || *health.Sealed || health.Version != "2.6.3" || health.Cluster == "" { + return errors.New("canary is not initialized and unsealed at the pinned version") + } + return nil +} + +func verifyDenial(input io.Reader, expected target) error { + bounded := &io.LimitedReader{R: input, N: 4 << 20} + decoder := json.NewDecoder(bounded) + matched := false + for { + var response map[string]json.RawMessage + if err := decoder.Decode(&response); err != nil { + if errors.Is(err, io.EOF) { + break + } + return errors.New("malformed observer response") + } + for _, field := range []string{"lost_events", "lostEvents", "node_status", "nodeStatus"} { + if _, ok := response[field]; ok { + return errors.New("incomplete observer response") + } + } + raw, ok := response["flow"] + if !ok { + return errors.New("unknown observer response") + } + var observed flow + if err := json.Unmarshal(raw, &observed); err != nil { + return errors.New("malformed flow") + } + when, err := time.Parse(time.RFC3339Nano, observed.Time) + if err != nil || when.Before(expected.since) || when.After(expected.until) { + continue + } + // Source default-deny or destination ingress denial both prove interception. + // UNKNOWN, missing and other directions never qualify. + if observed.Verdict == "DROPPED" && observed.Reason == "POLICY_DENIED" && + (observed.Direction == "EGRESS" || observed.Direction == "INGRESS") && + observed.Source.Namespace == "arc-runners" && observed.Source.Pod == expected.pod && + observed.Destination.Namespace == "openbao" && observed.Destination.Pod == "openbao-2" && + observed.IP.Source == expected.source && observed.IP.Destination == expected.destination && + observed.L4.TCP.Port == expected.port && observed.L4.TCP.Flags.SYN && observed.Node == expected.node { + matched = true + } + } + if bounded.N <= 0 { + return errors.New("observer response exceeds bound") + } + if !matched { + return errors.New("no correlated policy denial") + } + return nil +} + +func run(args []string) error { + if len(args) == 1 && args[0] == "health" { + return verifyHealth(os.Stdin) + } + if len(args) == 0 || args[0] != "denial" { + return errors.New("invalid command") + } + flags := flag.NewFlagSet("denial", flag.ContinueOnError) + flags.SetOutput(io.Discard) + var expected target + var port uint + var since, until string + flags.StringVar(&expected.pod, "pod", "", "") + flags.StringVar(&expected.source, "source", "", "") + flags.StringVar(&expected.destination, "destination", "", "") + flags.StringVar(&expected.node, "node", "", "") + flags.UintVar(&port, "port", 0, "") + flags.StringVar(&since, "since", "", "") + flags.StringVar(&until, "until", "", "") + if err := flags.Parse(args[1:]); err != nil || flags.NArg() != 0 { + return errors.New("invalid arguments") + } + if expected.pod == "" || expected.node == "" || (port != 8200 && port != 8204) { + return errors.New("invalid target") + } + for _, address := range []string{expected.source, expected.destination} { + if _, err := netip.ParseAddr(address); err != nil { + return errors.New("invalid address") + } + } + var err error + expected.since, err = time.Parse(time.RFC3339Nano, since) + if err != nil { + return errors.New("invalid start") + } + expected.until, err = time.Parse(time.RFC3339Nano, until) + if err != nil || expected.until.Before(expected.since) || expected.until.Sub(expected.since) > 45*time.Second { + return errors.New("invalid window") + } + expected.port = uint16(port) + return verifyDenial(os.Stdin, expected) +} + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, "ARC transport evidence: FAIL:", err) + os.Exit(1) + } +} diff --git a/scripts/verify-arc-transport/proof_test.go b/scripts/verify-arc-transport/proof_test.go new file mode 100644 index 0000000000..2392006ff0 --- /dev/null +++ b/scripts/verify-arc-transport/proof_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +func TestHealthRequiresActualUnsealedCanary(t *testing.T) { + for _, test := range []struct { + body string + ok bool + }{ + {`{"initialized":true,"sealed":false,"version":"2.6.3","cluster_id":"canary"}`, true}, + {`{"initialized":false,"sealed":false,"version":"2.6.3","cluster_id":"canary"}`, false}, + {`{"initialized":true,"sealed":true,"version":"2.6.3","cluster_id":"canary"}`, false}, + {`{"initialized":true,"sealed":false,"version":"2.5.4","cluster_id":"canary"}`, false}, + {`{"initialized":true,"version":"2.6.3","cluster_id":"canary"}`, false}, + {`{"initialized":true,"sealed":false,"version":"2.6.3","cluster_id":""}`, false}, + {`{"initialized":true,"sealed":false,"version":"2.6.3","cluster_id":"canary"} {}`, false}, + } { + if err := verifyHealth(strings.NewReader(test.body)); (err == nil) != test.ok { + t.Errorf("health accepted=%v want=%v", err == nil, test.ok) + } + } +} + +func TestDenialMustJoinTheActualAttempt(t *testing.T) { + start, _ := time.Parse(time.RFC3339Nano, "2026-10-06T12:00:00Z") + want := target{pod: "probe", source: "10.1.1.1", destination: "10.1.1.2", node: "prod/node", port: 8204, since: start, until: start.Add(5 * time.Second)} + valid := map[string]any{"verdict": "DROPPED", "drop_reason_desc": "POLICY_DENIED", "traffic_direction": "EGRESS", "time": "2026-10-06T12:00:01Z", "node_name": "prod/node", + "source": map[string]any{"namespace": "arc-runners", "pod_name": "probe"}, "destination": map[string]any{"namespace": "openbao", "pod_name": "openbao-2"}, + "IP": map[string]any{"source": "10.1.1.1", "destination": "10.1.1.2"}, "l4": map[string]any{"TCP": map[string]any{"destination_port": 8204, "flags": map[string]any{"SYN": true}}}} + encode := func(flow map[string]any) string { + data, _ := json.Marshal(map[string]any{"flow": flow}) + return string(data) + } + for _, direction := range []string{"EGRESS", "INGRESS"} { + valid["traffic_direction"] = direction + if err := verifyDenial(strings.NewReader(encode(valid)), want); err != nil { + t.Fatal(err) + } + } + for _, mutation := range []func(map[string]any){ + func(v map[string]any) { v["verdict"] = "FORWARDED" }, + func(v map[string]any) { v["drop_reason_desc"] = "CT_TRUNCATED_OR_INVALID_HEADER" }, + func(v map[string]any) { v["traffic_direction"] = "UNKNOWN" }, + func(v map[string]any) { v["node_name"] = "prod/other" }, + func(v map[string]any) { v["time"] = "2026-10-06T11:59:59Z" }, + func(v map[string]any) { v["time"] = "2026-10-06T12:00:06Z" }, + func(v map[string]any) { + v["source"] = map[string]any{"namespace": "external-secrets", "pod_name": "probe"} + }, + func(v map[string]any) { v["source"] = map[string]any{"namespace": "arc-runners", "pod_name": "other"} }, + func(v map[string]any) { + v["destination"] = map[string]any{"namespace": "openbao", "pod_name": "openbao-0"} + }, + func(v map[string]any) { v["IP"] = map[string]any{"source": "10.1.1.3", "destination": "10.1.1.2"} }, + func(v map[string]any) { v["IP"] = map[string]any{"source": "10.1.1.1", "destination": "10.1.1.4"} }, + func(v map[string]any) { + v["l4"] = map[string]any{"TCP": map[string]any{"destination_port": 8200, "flags": map[string]any{"SYN": true}}} + }, + func(v map[string]any) { + v["l4"] = map[string]any{"TCP": map[string]any{"destination_port": 8204, "flags": map[string]any{"SYN": false}}} + }, + } { + var changed map[string]any + _ = json.Unmarshal([]byte(mustJSON(valid)), &changed) + mutation(changed) + if err := verifyDenial(strings.NewReader(encode(changed)), want); err == nil { + t.Fatal("uncorrelated flow accepted") + } + } + for _, extra := range []string{`{"lost_events":{}}`, `{"lostEvents":{}}`, `{"node_status":{}}`, `{"nodeStatus":{}}`, `{}`, `garbage`, strings.Repeat(" ", 4<<20)} { + if err := verifyDenial(strings.NewReader(encode(valid)+"\n"+extra), want); err == nil { + t.Fatal("incomplete evidence accepted") + } + } +} + +func mustJSON(value any) string { + data, err := json.Marshal(value) + if err != nil { + panic(err) + } + return string(data) +} From c9136514e68a7eb7b8d6d17f43e7faea407c9205 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 14:49:15 +0200 Subject: [PATCH 33/62] fix(arc): reconcile retained releases in verified drain mode --- .github/actions/deploy-prod/action.yml | 10 + .github/workflows/ci.yaml | 3 + docs/operations/arc-runners.md | 22 +- .../helm-release.yaml | 5 +- .../retained-ksail-analysis/helm-release.yaml | 9 +- scripts/guard-arc-recovery/main.go | 331 ++++++++++++++++++ scripts/guard-arc-recovery/main_test.go | 217 ++++++++++++ scripts/tests/arc-staging/staging_test.go | 38 +- .../approved-surface.txt | 4 +- 9 files changed, 620 insertions(+), 19 deletions(-) create mode 100644 scripts/guard-arc-recovery/main.go create mode 100644 scripts/guard-arc-recovery/main_test.go diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index 17969c9f29..5d9965cce7 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -209,6 +209,12 @@ runs: shellcheck scripts/tests/test-pvc-prune-safety.sh bash scripts/tests/test-pvc-prune-safety.sh + - name: 🛑 Verify ARC drain recovery has no active pool + # Read complete metadata lists before the mutable artifact can move. + # No App key, JIT configuration or full Pod response is requested. + shell: bash + run: go run ./scripts/guard-arc-recovery before-publish + - name: 🛡️ Suspend autoscaling for gated Cilium rollout id: cilium_rollout_gate # The widened device component deliberately uses OnDelete so the Cilium @@ -338,6 +344,10 @@ runs: PLATFORM_MANIFEST_DIGEST: ${{ steps.publish_platform_manifest.outputs.digest }} run: ./scripts/wait-for-platform-flux-revision.sh "${PLATFORM_MANIFEST_DIGEST}" + - name: 🛑 Verify ARC stayed inactive across recovery + shell: bash + run: go run ./scripts/guard-arc-recovery after-reconcile + - name: Verify dedicated Wedding backup credential and active archive id: verify_wedding_backup_staging shell: bash diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index be9036e1ae..bdc7b02247 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -149,6 +149,9 @@ jobs: - name: 🛑 Validate inactive ARC staging boundaries run: go test ./scripts/tests/arc-staging + - name: 🛑 Validate ARC recovery metadata proof + run: go test -race ./scripts/guard-arc-recovery + # Shared Wedding backup access is retired on this proof's word, so it must # pass only on an AccessDenied for every access, after a positive control # and against an object the shared credential showed exists. These cases diff --git a/docs/operations/arc-runners.md b/docs/operations/arc-runners.md index 5dc061cab4..9cc4fed599 100644 --- a/docs/operations/arc-runners.md +++ b/docs/operations/arc-runners.md @@ -1,10 +1,16 @@ # Organization Linux runners The organization Linux pool is prepared but inactive and remains outside the -deployment aggregate. Both HelmReleases are suspended. Production retains the -controller and the former analysis release in Flux inventory to preserve ownership -of resources left by failed activation. Suspension stops Helm reconciliation; it -does not drain an installed scale set. Registration, execution and cleanup still +deployment aggregate and its HelmRelease is suspended. Production reconciles the +scoped controller and the former analysis release in Flux inventory to preserve +ownership of resources left by failed activation. The former release has explicit +zero minimum and maximum runner bounds. Suspension stops Helm reconciliation, +does not drain an installed scale set, and prevents a failed release from recovering +its readiness status. Native Flux and Helm health checks stay enabled. The protected +deploy proves complete, empty metadata lists for organization runner sets, releases, +credential-sync resources and both runner namespaces' pods before publication and +after reconciliation. It requests no App credentials, JIT configuration or full +Pod responses. Registration, execution and cleanup still need the separate proofs below before KSail #7131 can close. The controller chart and runner-set chart use the same immutable 0.15.0 artifacts. @@ -126,7 +132,7 @@ Activation requires a separate reviewed change and all of these proofs: For staged validation, run `go test ./scripts/tests/arc-staging` and build both component directories directly with `kubectl kustomize`. Local trees exclude ARC; -production retains the suspended controller and protected legacy analysis resources +production retains the reconciled controller and drained protected legacy analysis resources without including the organization pool. The unconditional CI guard runs on pull requests and merge groups. A deliberate activation revises that guard in the same reviewed change, alongside its evidence; deleting the guard alone is @@ -156,8 +162,10 @@ finish. It does not require retiring the shared platform App. ## Rollout and recovery -After the gates above are approved, first reference and unsuspend the controller -in the controller layer and verify it is healthy. That component creates both +After the gates above are approved, verify the scoped controller is healthy and +retire its `platform.devantler.tech/arc-recovery: drain-only` marker in the reviewed +activation change. The recovery guard intentionally refuses a credentialed pool +while that marker remains. That component creates both namespaces before the chart installs its namespace-scoped RBAC. The controller's network policy also covers the listener, which ARC creates in that namespace. Only then reference the pool in diff --git a/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml b/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml index d687140872..061fe2609a 100644 --- a/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml +++ b/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml @@ -1,4 +1,4 @@ -# Production retains this suspended release in Flux inventory. The organization +# Production reconciles this scoped release in Flux inventory. The organization # pool stays unreferenced; activation is separate. See docs/operations/arc-runners.md. apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease @@ -7,10 +7,11 @@ metadata: namespace: arc-systems annotations: kustomize.toolkit.fluxcd.io/prune: disabled + platform.devantler.tech/arc-recovery: drain-only labels: helm.toolkit.fluxcd.io/crds: enabled spec: - suspend: true + suspend: false interval: 10m timeout: 10m chartRef: diff --git a/k8s/providers/hetzner/infrastructure/retained-ksail-analysis/helm-release.yaml b/k8s/providers/hetzner/infrastructure/retained-ksail-analysis/helm-release.yaml index 710ec99d71..8d798f91b9 100644 --- a/k8s/providers/hetzner/infrastructure/retained-ksail-analysis/helm-release.yaml +++ b/k8s/providers/hetzner/infrastructure/retained-ksail-analysis/helm-release.yaml @@ -1,5 +1,6 @@ -# Inactive, uncalibrated analysis-only pool. Do not route a workflow here until -# the registration, isolation, toolchain and capacity gates in the runbook pass. +# Retained analysis release reconciles in explicit drain mode. Suspending a +# failed HelmRelease prevents recovery of its Ready status and blocks Flux. +# The controller watches only arc-runners, so this legacy namespace is ignored. apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -8,7 +9,7 @@ metadata: annotations: kustomize.toolkit.fluxcd.io/prune: disabled spec: - suspend: true + suspend: false interval: 10m timeout: 10m chartRef: @@ -23,7 +24,7 @@ spec: githubConfigSecret: arc-ksail-app runnerScaleSetName: ksail-code-quality minRunners: 0 - maxRunners: 1 + maxRunners: 0 controllerServiceAccount: namespace: arc-systems name: arc-controller diff --git a/scripts/guard-arc-recovery/main.go b/scripts/guard-arc-recovery/main.go new file mode 100644 index 0000000000..c0aa9ea400 --- /dev/null +++ b/scripts/guard-arc-recovery/main.go @@ -0,0 +1,331 @@ +// guard-arc-recovery proves the retained ARC charts cannot admit work while +// their Helm reconciliation recovers. It never requests credentials or JIT data. +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +const metadataAccept = "application/json;as=PartialObjectMetadataList;g=meta.k8s.io;v=v1" +const controllerDeploymentPath = "/apis/apps/v1/namespaces/arc-systems/deployments/arc-controller" + +// RE2 has no negative lookahead; these alternatives match every nonempty +// method except the exact GET spelling, including nonstandard methods. +const rejectNonGET = "^(G|GE|GET.+|[^G].*|G[^E].*|GE[^T].*)$" + +var recoveryEndpoints = []string{ + "/apis/actions.github.com/v1alpha1/namespaces/arc-runners/autoscalingrunnersets", + "/apis/actions.github.com/v1alpha1/namespaces/arc-runners/autoscalinglisteners", + "/apis/actions.github.com/v1alpha1/namespaces/arc-runners/ephemeralrunnersets", + "/apis/actions.github.com/v1alpha1/namespaces/arc-runners/ephemeralrunners", + "/apis/actions.github.com/v1alpha1/namespaces/arc-systems/autoscalingrunnersets", + "/apis/actions.github.com/v1alpha1/namespaces/arc-systems/autoscalinglisteners", + "/apis/actions.github.com/v1alpha1/namespaces/arc-systems/ephemeralrunnersets", + "/apis/actions.github.com/v1alpha1/namespaces/arc-systems/ephemeralrunners", + "/apis/helm.toolkit.fluxcd.io/v2/namespaces/arc-runners/helmreleases", + "/apis/external-secrets.io/v1/namespaces/arc-runners/externalsecrets", + "/api/v1/namespaces/arc-runners/pods", + "/api/v1/namespaces/arc-ksail-analysis/pods", +} + +type release struct { + Metadata struct { + Annotations map[string]string `yaml:"annotations"` + } `yaml:"metadata"` + Spec struct { + Suspend *bool `yaml:"suspend"` + Values struct { + Flags struct { + Namespace string `yaml:"watchSingleNamespace"` + } `yaml:"flags"` + } `yaml:"values"` + } `yaml:"spec"` +} + +func recoveryArmed(root string) (bool, error) { + controller, err := os.ReadFile(filepath.Join(root, "k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml")) + if err != nil { + return false, errors.New("controller declaration unreadable") + } + var c release + if yaml.Unmarshal(controller, &c) != nil { + return false, errors.New("controller declaration invalid") + } + mode := c.Metadata.Annotations["platform.devantler.tech/arc-recovery"] + if mode == "" { + return false, nil + } + if mode != "drain-only" || c.Spec.Suspend == nil || *c.Spec.Suspend || c.Spec.Values.Flags.Namespace != "arc-runners" { + return false, errors.New("recovery controller scope invalid") + } + data, err := os.ReadFile(filepath.Join(root, "k8s/providers/hetzner/infrastructure/retained-ksail-analysis/helm-release.yaml")) + if err != nil { + return false, errors.New("legacy drain declaration unreadable") + } + // Use explicit pointer fields: omitted zero bounds mean unbounded ARC scaling. + var legacy struct { + Spec struct { + Suspend *bool `yaml:"suspend"` + Values struct { + Min *int `yaml:"minRunners"` + Max *int `yaml:"maxRunners"` + } `yaml:"values"` + } `yaml:"spec"` + } + if yaml.Unmarshal(data, &legacy) != nil || legacy.Spec.Suspend == nil || *legacy.Spec.Suspend || legacy.Spec.Values.Min == nil || legacy.Spec.Values.Max == nil || *legacy.Spec.Values.Min != 0 || *legacy.Spec.Values.Max != 0 { + return false, errors.New("legacy pool is not explicitly drained and reconciling") + } + return true, nil +} + +func metadataClient() *http.Client { + return &http.Client{ + Timeout: 10 * time.Second, + Transport: &http.Transport{Proxy: nil, DisableKeepAlives: true}, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + } +} + +func requireEmptyMetadata(ctx context.Context, client *http.Client, endpoint string) error { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return errors.New("metadata request invalid") + } + request.Header.Set("Accept", metadataAccept) + response, err := client.Do(request) + if err != nil { + return errors.New("metadata read failed") + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return errors.New("metadata read refused or redirected") + } + data, err := io.ReadAll(io.LimitReader(response.Body, (1<<20)+1)) + if err != nil || len(data) > 1<<20 { + return errors.New("metadata response incomplete or oversized") + } + var list struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Metadata struct { + ResourceVersion string `json:"resourceVersion"` + Continue string `json:"continue"` + Remaining *int64 `json:"remainingItemCount"` + } `json:"metadata"` + Items []struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Metadata map[string]json.RawMessage `json:"metadata"` + } `json:"items"` + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if decoder.Decode(&list) != nil || list.APIVersion != "meta.k8s.io/v1" || list.Kind != "PartialObjectMetadataList" || list.Metadata.ResourceVersion == "" || list.Metadata.Continue != "" || list.Items == nil || (list.Metadata.Remaining != nil && *list.Metadata.Remaining != 0) { + return errors.New("metadata response is not a complete metadata list") + } + if decoder.Decode(new(any)) != io.EOF { + return errors.New("metadata response has trailing data") + } + if len(list.Items) != 0 { + return errors.New("ARC recovery found a live pool, credential-sync resource or pod") + } + return nil +} + +func requireControllerScope(ctx context.Context, client *http.Client, endpoint string) error { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return errors.New("controller scope request invalid") + } + request.Header.Set("Accept", "application/json") + response, err := client.Do(request) + if err != nil { + return errors.New("controller scope read failed") + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return errors.New("controller scope read refused or redirected") + } + data, err := io.ReadAll(io.LimitReader(response.Body, (1<<20)+1)) + if err != nil || len(data) > 1<<20 { + return errors.New("controller scope response incomplete or oversized") + } + var deployment struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Metadata struct { + Name, Namespace, UID string + Generation int64 + DeletionTimestamp *string + } `json:"metadata"` + Spec struct { + Replicas int + Template struct { + Spec struct { + Containers []struct { + Name string + Command, Args []string + } `json:"containers"` + } `json:"spec"` + } `json:"template"` + } `json:"spec"` + Status struct { + ObservedGeneration int64 + Replicas, UpdatedReplicas, ReadyReplicas, AvailableReplicas, UnavailableReplicas int + } `json:"status"` + } + decoder := json.NewDecoder(bytes.NewReader(data)) + if decoder.Decode(&deployment) != nil || decoder.Decode(new(any)) != io.EOF || deployment.APIVersion != "apps/v1" || deployment.Kind != "Deployment" || deployment.Metadata.Name != "arc-controller" || deployment.Metadata.Namespace != "arc-systems" || deployment.Metadata.UID == "" || deployment.Metadata.Generation < 1 || deployment.Metadata.DeletionTimestamp != nil { + return errors.New("controller scope response invalid") + } + if deployment.Status.ObservedGeneration != deployment.Metadata.Generation || deployment.Spec.Replicas != 1 || deployment.Status.Replicas != 1 || deployment.Status.UpdatedReplicas != 1 || deployment.Status.ReadyReplicas != 1 || deployment.Status.AvailableReplicas != 1 || deployment.Status.UnavailableReplicas != 0 || len(deployment.Spec.Template.Spec.Containers) != 1 { + return errors.New("controller scope has not fully rolled out") + } + container := deployment.Spec.Template.Spec.Containers[0] + if container.Name != "manager" || len(container.Command) != 1 || container.Command[0] != "/manager" { + return errors.New("controller executable differs from the pinned chart") + } + scope, mode := 0, 0 + for _, arg := range container.Args { + switch { + case strings.HasPrefix(arg, "--watch-single-namespace"): + if arg != "--watch-single-namespace=arc-runners" { + return errors.New("controller still watches an unapproved namespace") + } + scope++ + case strings.HasPrefix(arg, "--watch-namespace"): + return errors.New("controller carries an additional namespace scope") + case strings.HasPrefix(arg, "--auto-scaling-runner-set-only"): + if arg != "--auto-scaling-runner-set-only" { + return errors.New("controller mode differs from the pinned chart") + } + mode++ + } + } + if scope != 1 || mode != 1 { + return errors.New("controller watch scope or mode is missing or duplicated") + } + return nil +} + +// kubectl retains its existing protected CI kubeconfig. The short-lived proxy +// accepts only these GET paths on loopback; no token is copied into this command. +func startMetadataProxy(ctx context.Context, command func(context.Context, ...string) *exec.Cmd) (string, func(), error) { + childCtx, cancel := context.WithCancel(ctx) + paths := make([]string, len(recoveryEndpoints)+1) + for i, path := range recoveryEndpoints { + paths[i] = regexp.QuoteMeta(path) + } + paths[len(recoveryEndpoints)] = regexp.QuoteMeta(controllerDeploymentPath) + cmd := command(childCtx, "--context", "admin@prod", "proxy", "--address=127.0.0.1", "--port=0", "--accept-hosts=^127\\.0\\.0\\.1$", "--accept-paths=^("+strings.Join(paths, "|")+")$", "--reject-methods="+rejectNonGET) + cmd.Stderr = io.Discard + cmd.WaitDelay = 2 * time.Second + stdout, err := cmd.StdoutPipe() + if err != nil { + cancel() + return "", nil, errors.New("metadata proxy output unavailable") + } + if cmd.Start() != nil { + cancel() + return "", nil, errors.New("metadata proxy failed to start") + } + done := make(chan struct{}) + go func() { _ = cmd.Wait(); close(done) }() + stop := func() { cancel(); <-done } + address := make(chan string, 1) + go func() { + scanner := bufio.NewScanner(stdout) + scanner.Buffer(make([]byte, 1024), 4096) + if scanner.Scan() { + address <- scanner.Text() + } else { + address <- "" + } + _, _ = io.Copy(io.Discard, stdout) + }() + select { + case line := <-address: + match := regexp.MustCompile(`^Starting to serve on 127\.0\.0\.1:([0-9]+)$`).FindStringSubmatch(line) + if len(match) != 2 { + stop() + return "", nil, errors.New("metadata proxy address invalid") + } + port, err := strconv.Atoi(match[1]) + if err != nil || port < 1 || port > 65535 { + stop() + return "", nil, errors.New("metadata proxy port invalid") + } + return "http://127.0.0.1:" + match[1], stop, nil + case <-ctx.Done(): + stop() + return "", nil, errors.New("metadata proxy readiness timed out") + } +} + +func run(stage string) error { + if stage != "before-publish" && stage != "after-reconcile" { + return errors.New("invalid recovery observation stage") + } + armed, err := recoveryArmed(".") + if err != nil { + return err + } + if !armed { + fmt.Println("ARC drain recovery guard is not armed by this declaration") + return nil + } + ref := os.Getenv("GITHUB_REF") + if os.Getenv("GITHUB_ACTIONS") != "true" || os.Getenv("GITHUB_REPOSITORY") != "devantler-tech/platform" || !regexp.MustCompile(`^[0-9a-f]{40}$`).MatchString(os.Getenv("GITHUB_SHA")) || (ref != "refs/heads/main" && !strings.HasPrefix(ref, "refs/heads/gh-readonly-queue/main/")) { + return errors.New("ARC recovery metadata proof requires the protected production workflow") + } + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + base, stop, err := startMetadataProxy(ctx, func(ctx context.Context, args ...string) *exec.Cmd { + return exec.CommandContext(ctx, "kubectl", args...) + }) + if err != nil { + return err + } + defer stop() + client := metadataClient() + defer client.CloseIdleConnections() + if stage == "after-reconcile" { + if err := requireControllerScope(ctx, client, base+controllerDeploymentPath); err != nil { + return err + } + } + for _, endpoint := range recoveryEndpoints { + if err := requireEmptyMetadata(ctx, client, base+endpoint); err != nil { + return err + } + } + fmt.Printf("ARC drain recovery %s: %d complete metadata lists prove zero active organization resources or ARC pods\n", stage, len(recoveryEndpoints)) + return nil +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: guard-arc-recovery ") + os.Exit(2) + } + if err := run(os.Args[1]); err != nil { + fmt.Fprintln(os.Stderr, "ARC recovery refused:", err) + os.Exit(1) + } +} diff --git a/scripts/guard-arc-recovery/main_test.go b/scripts/guard-arc-recovery/main_test.go new file mode 100644 index 0000000000..fda9de707b --- /dev/null +++ b/scripts/guard-arc-recovery/main_test.go @@ -0,0 +1,217 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/pem" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestRecoveryMetadataReadsRequireCompleteEmptyMetadataLists(t *testing.T) { + const empty = `{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{"resourceVersion":"123"},"items":[]}` + for _, tc := range []struct { + name, body string + code int + ok bool + }{ + {"empty", empty, 200, true}, + {"live resource", strings.Replace(empty, `"items":[]`, `"items":[{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadata","metadata":{"name":"active"}}]`, 1), 200, false}, + {"full pod list", strings.Replace(empty, "PartialObjectMetadataList", "PodList", 1), 200, false}, + {"full data masquerading as metadata", strings.Replace(empty, `"items":[]`, `"spec":{"token":"synthetic"},"items":[]`, 1), 200, false}, + {"missing items", strings.Replace(empty, `,"items":[]`, "", 1), 200, false}, + {"null items", strings.Replace(empty, `"items":[]`, `"items":null`, 1), 200, false}, + {"pagination", strings.Replace(empty, `"resourceVersion":"123"`, `"resourceVersion":"123","continue":"next"`, 1), 200, false}, + {"missing revision", strings.Replace(empty, `"resourceVersion":"123"`, `"resourceVersion":""`, 1), 200, false}, + {"wrong api", strings.Replace(empty, "meta.k8s.io/v1", "v1", 1), 200, false}, + {"trailing document", empty + empty, 200, false}, + {"malformed", `{`, 200, false}, + {"too large", empty + strings.Repeat(" ", 1<<20), 200, false}, + {"forbidden", empty, 403, false}, + {"redirect", empty, 302, false}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" || r.Header.Get("Accept") != metadataAccept { + t.Error("metadata request widened its method or response type") + } + w.WriteHeader(tc.code) + _, _ = w.Write([]byte(tc.body)) + })) + defer server.Close() + err := requireEmptyMetadata(context.Background(), metadataClient(), server.URL+"/api/v1/namespaces/arc-runners/pods") + if (err == nil) != tc.ok { + t.Fatalf("accepted=%v, want %v: %v", err == nil, tc.ok, err) + } + if err != nil && strings.Contains(err.Error(), "synthetic") { + t.Fatal("response contents escaped into diagnostics") + } + }) + } +} + +func TestRecoveryDeclarationRefusesSuspensionAndImplicitRunnerBounds(t *testing.T) { + for _, tc := range []struct { + name, controller, legacy string + ok bool + }{ + {"drained", "false", "suspend: false\n values: {minRunners: 0, maxRunners: 0}", true}, + {"suspended controller", "true", "suspend: false\n values: {minRunners: 0, maxRunners: 0}", false}, + {"suspended legacy", "false", "suspend: true\n values: {minRunners: 0, maxRunners: 0}", false}, + {"implicit maximum", "false", "suspend: false\n values: {minRunners: 0}", false}, + {"implicit minimum", "false", "suspend: false\n values: {maxRunners: 0}", false}, + {"admits one runner", "false", "suspend: false\n values: {minRunners: 0, maxRunners: 1}", false}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + files := map[string]string{ + "k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml": "metadata:\n annotations: {platform.devantler.tech/arc-recovery: drain-only}\nspec:\n suspend: " + tc.controller + "\n values:\n flags: {watchSingleNamespace: arc-runners}\n", + "k8s/providers/hetzner/infrastructure/retained-ksail-analysis/helm-release.yaml": "spec:\n " + tc.legacy + "\n", + } + for path, data := range files { + full := filepath.Join(root, path) + if err := os.MkdirAll(filepath.Dir(full), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(data), 0600); err != nil { + t.Fatal(err) + } + } + armed, err := recoveryArmed(root) + if (err == nil && armed) != tc.ok { + t.Fatalf("armed=%v err=%v", armed, err) + } + }) + } +} + +// Exercise the real kubectl proxy with a temporary synthetic kubeconfig and +// TLS API server. This never opens the operator's kubeconfig or a live cluster. +func TestNativeMetadataProxyPreservesAcceptAndRejectsWrites(t *testing.T) { + kubectl, err := exec.LookPath("kubectl") + if err != nil { + t.Fatal("native metadata-proxy regression requires kubectl") + } + requests := make(chan string, 16) + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests <- r.Method + " " + r.URL.Path + if r.Method != "GET" || r.Header.Get("Accept") != metadataAccept { + t.Error("native proxy widened the read") + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{"resourceVersion":"123"},"items":[]}`)) + })) + defer server.Close() + cert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) + config := fmt.Sprintf("apiVersion: v1\nkind: Config\nclusters:\n- name: synthetic\n cluster:\n server: %s\n certificate-authority-data: %s\ncontexts:\n- name: admin@prod\n context: {cluster: synthetic, user: synthetic}\nusers:\n- name: synthetic\n user: {token: synthetic}\n", server.URL, base64.StdEncoding.EncodeToString(cert)) + configPath := filepath.Join(t.TempDir(), "kubeconfig") + if err := os.WriteFile(configPath, []byte(config), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var child *exec.Cmd + base, stop, err := startMetadataProxy(ctx, func(ctx context.Context, args ...string) *exec.Cmd { + child = exec.CommandContext(ctx, kubectl, args...) + child.Env = append(os.Environ(), "KUBECONFIG="+configPath) + return child + }) + if err != nil { + t.Fatal(err) + } + defer stop() + client := metadataClient() + defer client.CloseIdleConnections() + for _, path := range recoveryEndpoints { + if err := requireEmptyMetadata(ctx, client, base+path); err != nil { + t.Fatal(err) + } + } + for _, tc := range []struct{ method, path string }{ + {"DELETE", recoveryEndpoints[0]}, + {"POST", recoveryEndpoints[0]}, + {"PUT", recoveryEndpoints[0]}, + {"PATCH", recoveryEndpoints[0]}, + {"HEAD", recoveryEndpoints[0]}, + {"OPTIONS", recoveryEndpoints[0]}, + {"PROPFIND", recoveryEndpoints[0]}, + {"get", recoveryEndpoints[0]}, + {"G", recoveryEndpoints[0]}, + {"GE", recoveryEndpoints[0]}, + {"GETTING", recoveryEndpoints[0]}, + {"GET", "/api/v1/namespaces/arc-runners/secrets"}, + {"GET", "/apis/actions.github.com/v1alpha1/namespaces/arc-runners/ephemeralrunners/active"}, + } { + request, _ := http.NewRequestWithContext(ctx, tc.method, base+tc.path, nil) + response, err := client.Do(request) + if err != nil { + t.Fatal(err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusForbidden { + t.Fatalf("unexpected denied-proxy status %d", response.StatusCode) + } + } + if len(requests) != len(recoveryEndpoints) { + t.Fatal("denied request reached the API server") + } + stop() + if child.ProcessState == nil { + t.Fatal("native proxy was not reaped") + } +} + +func TestRecoveryUsesOnlyMetadataEndpoints(t *testing.T) { + if len(recoveryEndpoints) != 12 { + t.Fatal("incomplete recovery observation set") + } + for _, path := range recoveryEndpoints { + if strings.Contains(path, "secrets") && !strings.Contains(path, "externalsecrets") { + t.Fatal("credential read") + } + if !strings.Contains(path, "/namespaces/arc-runners/") && !strings.Contains(path, "/namespaces/arc-ksail-analysis/") && !strings.Contains(path, "/namespaces/arc-systems/") { + t.Fatal("foreign namespace") + } + } +} + +const recoveredDeployment = `{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"name":"arc-controller","namespace":"arc-systems","uid":"synthetic","generation":2},"spec":{"replicas":1,"template":{"spec":{"containers":[{"name":"manager","command":["/manager"],"args":["--auto-scaling-runner-set-only","--watch-single-namespace=arc-runners"]}]}}},"status":{"observedGeneration":2,"replicas":1,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1}}` + +func TestRecoveryRequiresAppliedControllerScope(t *testing.T) { + for _, tc := range []struct { + name, body string + ok bool + }{ + {"recovered", recoveredDeployment, true}, + {"legacy scope", strings.Replace(recoveredDeployment, "--watch-single-namespace=arc-runners", "--watch-single-namespace=arc-ksail-analysis", 1), false}, + {"duplicate scope", strings.Replace(recoveredDeployment, `"--watch-single-namespace=arc-runners"`, `"--watch-single-namespace=arc-runners","--watch-single-namespace=arc-ksail-analysis"`, 1), false}, + {"broad scope", strings.Replace(recoveredDeployment, "--watch-single-namespace=arc-runners", "--watch-namespace=", 1), false}, + {"legacy mode", strings.Replace(recoveredDeployment, "--auto-scaling-runner-set-only", "--other-mode", 1), false}, + {"stale generation", strings.Replace(recoveredDeployment, `"observedGeneration":2`, `"observedGeneration":1`, 1), false}, + {"old replica", strings.Replace(recoveredDeployment, `"replicas":1,"updatedReplicas"`, `"replicas":2,"updatedReplicas"`, 1), false}, + {"unready", strings.Replace(recoveredDeployment, `"readyReplicas":1`, `"readyReplicas":0`, 1), false}, + {"wrong resource", strings.Replace(recoveredDeployment, `"kind":"Deployment"`, `"kind":"Pod"`, 1), false}, + {"trailing document", recoveredDeployment + recoveredDeployment, false}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" || r.Header.Get("Accept") != "application/json" || r.URL.Path != controllerDeploymentPath { + t.Error("controller read widened") + } + _, _ = w.Write([]byte(tc.body)) + })) + defer server.Close() + err := requireControllerScope(context.Background(), metadataClient(), server.URL+controllerDeploymentPath) + if (err == nil) != tc.ok { + t.Fatalf("accepted=%v, want %v: %v", err == nil, tc.ok, err) + } + }) + } +} diff --git a/scripts/tests/arc-staging/staging_test.go b/scripts/tests/arc-staging/staging_test.go index 9c140f4ed1..f087e10c5f 100644 --- a/scripts/tests/arc-staging/staging_test.go +++ b/scripts/tests/arc-staging/staging_test.go @@ -48,14 +48,14 @@ func equal(t *testing.T, value, want any) { } } -func TestStagedReleasesAreSuspendedAndPinned(t *testing.T) { +func TestRecoveryControllerReconcilesWhileOrganizationPoolStaysSuspended(t *testing.T) { for _, component := range []string{ "k8s/bases/infrastructure/controllers/actions-runner-controller", "k8s/bases/infrastructure/actions-runners", } { t.Run(filepath.Base(component), func(t *testing.T) { release := readYAML(t, component+"/helm-release.yaml") - equal(t, field(t, release, "spec", "suspend"), true) + equal(t, field(t, release, "spec", "suspend"), filepath.Base(component) == "actions-runners") equal(t, field(t, release, "spec", "chartRef", "kind"), "OCIRepository") source := readYAML(t, component+"/oci-repository.yaml") equal(t, field(t, source, "spec", "ref", "tag"), "0.15.0") @@ -359,7 +359,7 @@ func TestNoDeploymentAggregateActivatesARC(t *testing.T) { t.Fatal(err) } if !controllerFound { - t.Fatal("production must retain the suspended controller in its inventory") + t.Fatal("production must retain the scoped controller in its inventory") } } @@ -372,7 +372,37 @@ func TestRetainedAnalysisResourcesCannotActivateOrLosePruneProtection(t *testing equal(t, field(t, release, "metadata", "name"), "ksail-analysis-runners") equal(t, field(t, release, "metadata", "namespace"), "arc-ksail-analysis") equal(t, field(t, release, "metadata", "annotations", "kustomize.toolkit.fluxcd.io/prune"), "disabled") - equal(t, field(t, release, "spec", "suspend"), true) + equal(t, field(t, release, "spec", "suspend"), false) + equal(t, field(t, release, "spec", "values", "minRunners"), 0) + equal(t, field(t, release, "spec", "values", "maxRunners"), 0) + controller := readYAML(t, "k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml") + equal(t, field(t, controller, "spec", "values", "flags", "watchSingleNamespace"), "arc-runners") + equal(t, field(t, controller, "metadata", "annotations", "platform.devantler.tech/arc-recovery"), "drain-only") +} + +func TestRecoveryMetadataProofSurroundsPublicationAndReconciliation(t *testing.T) { + action := readYAML(t, ".github/actions/deploy-prod/action.yml") + before, publish, reconcile, after := -1, -1, -1, -1 + for index, step := range field(t, action, "runs", "steps").([]any) { + value := step.(map[string]any) + if value["uses"] == "./.github/actions/deploy-prod/publish-platform-manifests" { + publish = index + } + if value["run"] == "./scripts/reconcile-flux-workloads.sh" { + reconcile = index + } + for stage, target := range map[string]*int{"before-publish": &before, "after-reconcile": &after} { + if value["run"] == "go run ./scripts/guard-arc-recovery "+stage { + if _, conditional := value["if"]; conditional { + t.Fatal("recovery proof must not be skipped") + } + *target = index + } + } + } + if before < 0 || before >= publish || after <= reconcile { + t.Fatal("recovery metadata proof must run before publication and after reconciliation") + } } func TestDNSAccessIsLimitedToTheDeclaredExternalDependencies(t *testing.T) { diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index e3ec6d9d1e..f640d824ed 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -14,9 +14,9 @@ helm.toolkit.fluxcd.io/v2|HelmRelease|actual-budget|actual-budget adc009b64f2a9da1427f0c9b80f6ea2490607a8b55ceb6e139f73de92399d5da -helm.toolkit.fluxcd.io/v2|HelmRelease|arc-ksail-analysis|ksail-analysis-runners 6d362a8f340a13e3f71293e341a762a590c3fab61eb81e2167bdc1975c67748a +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-ksail-analysis|ksail-analysis-runners f59980a0d056bb796135803ce4cfb1a84947957e943abe9854a78eaf1249b2b0 -helm.toolkit.fluxcd.io/v2|HelmRelease|arc-systems|arc-controller 655a911346c6ea07b14e8561759f735e643090a03454688186e8f31ac898cbf6 +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-systems|arc-controller 5af946415fcf05e49fb79e9a07cd82cd1da6d3c8ee33a6ebbe0483fc6e00afa1 helm.toolkit.fluxcd.io/v2|HelmRelease|backstage|backstage 5ff19be19e3525e641b0a7a22e2a7047e1fbda16ef9ca7b4fa731ce3a61e686a From 4150746b65fb3feb4f3fd84d43e3b55c1053e0b5 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 15:14:21 +0200 Subject: [PATCH 34/62] fix(arc): observe native empty lists and retained runner children --- .../controllers/kustomization.yaml | 3 +- scripts/guard-arc-recovery/main.go | 99 ++++++++++++++++--- scripts/guard-arc-recovery/main_test.go | 47 ++++++++- 3 files changed, 128 insertions(+), 21 deletions(-) diff --git a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml index a47f5cfc03..79bb5c3979 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml @@ -9,7 +9,8 @@ resources: # virt-handler crash-looped (1000+ restarts) with zero VMs. - ../../../../bases/infrastructure/controllers/ # Retain the protected ARC namespaces and release in Flux inventory while - # registration is repaired. The declared HelmRelease remains suspended. + # registration is repaired. Flux reconciles the controller with its watch + # scope limited to arc-runners; the organization pool stays outside aggregates. - ../../../../bases/infrastructure/controllers/actions-runner-controller/ # Prod-only: a minimal Alertmanager scoped to Kubescape runtime-detection # alerts. The Headlamp Kubescape plugin's "Runtime Detection > Alerts" tab diff --git a/scripts/guard-arc-recovery/main.go b/scripts/guard-arc-recovery/main.go index c0aa9ea400..365bf42383 100644 --- a/scripts/guard-arc-recovery/main.go +++ b/scripts/guard-arc-recovery/main.go @@ -24,6 +24,7 @@ import ( const metadataAccept = "application/json;as=PartialObjectMetadataList;g=meta.k8s.io;v=v1" const controllerDeploymentPath = "/apis/apps/v1/namespaces/arc-systems/deployments/arc-controller" +const retainedPoolPath = "/apis/actions.github.com/v1alpha1/namespaces/arc-ksail-analysis/autoscalingrunnersets" // RE2 has no negative lookahead; these alternatives match every nonempty // method except the exact GET spelling, including nonstandard methods. @@ -34,6 +35,10 @@ var recoveryEndpoints = []string{ "/apis/actions.github.com/v1alpha1/namespaces/arc-runners/autoscalinglisteners", "/apis/actions.github.com/v1alpha1/namespaces/arc-runners/ephemeralrunnersets", "/apis/actions.github.com/v1alpha1/namespaces/arc-runners/ephemeralrunners", + retainedPoolPath, + "/apis/actions.github.com/v1alpha1/namespaces/arc-ksail-analysis/autoscalinglisteners", + "/apis/actions.github.com/v1alpha1/namespaces/arc-ksail-analysis/ephemeralrunnersets", + "/apis/actions.github.com/v1alpha1/namespaces/arc-ksail-analysis/ephemeralrunners", "/apis/actions.github.com/v1alpha1/namespaces/arc-systems/autoscalingrunnersets", "/apis/actions.github.com/v1alpha1/namespaces/arc-systems/autoscalinglisteners", "/apis/actions.github.com/v1alpha1/namespaces/arc-systems/ephemeralrunnersets", @@ -42,6 +47,7 @@ var recoveryEndpoints = []string{ "/apis/external-secrets.io/v1/namespaces/arc-runners/externalsecrets", "/api/v1/namespaces/arc-runners/pods", "/api/v1/namespaces/arc-ksail-analysis/pods", + "/api/v1/namespaces/arc-systems/pods?labelSelector=platform.devantler.tech%2Farc-role%3Dlistener", } type release struct { @@ -102,23 +108,23 @@ func metadataClient() *http.Client { } } -func requireEmptyMetadata(ctx context.Context, client *http.Client, endpoint string) error { +func readMetadata(ctx context.Context, client *http.Client, endpoint string) ([]json.RawMessage, error) { request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { - return errors.New("metadata request invalid") + return nil, errors.New("metadata request invalid") } request.Header.Set("Accept", metadataAccept) response, err := client.Do(request) if err != nil { - return errors.New("metadata read failed") + return nil, errors.New("metadata read failed") } defer response.Body.Close() if response.StatusCode != http.StatusOK { - return errors.New("metadata read refused or redirected") + return nil, errors.New("metadata read refused or redirected") } data, err := io.ReadAll(io.LimitReader(response.Body, (1<<20)+1)) if err != nil || len(data) > 1<<20 { - return errors.New("metadata response incomplete or oversized") + return nil, errors.New("metadata response incomplete or oversized") } var list struct { APIVersion string `json:"apiVersion"` @@ -128,26 +134,81 @@ func requireEmptyMetadata(ctx context.Context, client *http.Client, endpoint str Continue string `json:"continue"` Remaining *int64 `json:"remainingItemCount"` } `json:"metadata"` - Items []struct { - APIVersion string `json:"apiVersion"` - Kind string `json:"kind"` - Metadata map[string]json.RawMessage `json:"metadata"` - } `json:"items"` + Items json.RawMessage `json:"items"` } decoder := json.NewDecoder(bytes.NewReader(data)) decoder.DisallowUnknownFields() - if decoder.Decode(&list) != nil || list.APIVersion != "meta.k8s.io/v1" || list.Kind != "PartialObjectMetadataList" || list.Metadata.ResourceVersion == "" || list.Metadata.Continue != "" || list.Items == nil || (list.Metadata.Remaining != nil && *list.Metadata.Remaining != 0) { - return errors.New("metadata response is not a complete metadata list") + if decoder.Decode(&list) != nil || list.APIVersion != "meta.k8s.io/v1" || list.Kind != "PartialObjectMetadataList" || list.Metadata.ResourceVersion == "" || list.Metadata.Continue != "" || len(list.Items) == 0 || (list.Metadata.Remaining != nil && *list.Metadata.Remaining != 0) { + return nil, errors.New("metadata response is not a complete metadata list") } if decoder.Decode(new(any)) != io.EOF { - return errors.New("metadata response has trailing data") + return nil, errors.New("metadata response has trailing data") + } + // Kubernetes encodes a native empty metadata list as null. Keep the field + // presence check above so an omitted list cannot prove absence. + if bytes.Equal(bytes.TrimSpace(list.Items), []byte("null")) { + return nil, nil + } + var items []json.RawMessage + if json.Unmarshal(list.Items, &items) != nil || items == nil { + return nil, errors.New("metadata response items is not an array or native null") + } + return items, nil +} + +func requireEmptyMetadata(ctx context.Context, client *http.Client, endpoint string) error { + items, err := readMetadata(ctx, client, endpoint) + if err != nil { + return err } - if len(list.Items) != 0 { + if len(items) != 0 { return errors.New("ARC recovery found a live pool, credential-sync resource or pod") } return nil } +// Helm retains its declared AutoscalingRunnerSet even with both bounds at zero. +// Observe only that exact declaration; listeners, runner sets and pods must be +// absent, and after-reconcile independently proves the controller excludes it. +func requireRetainedMetadata(ctx context.Context, client *http.Client, endpoint string) error { + items, err := readMetadata(ctx, client, endpoint) + if err != nil { + return err + } + if len(items) == 0 { + return nil + } + if len(items) != 1 { + return errors.New("retained namespace contains unexpected pools") + } + var item struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Metadata map[string]json.RawMessage `json:"metadata"` + } + decoder := json.NewDecoder(bytes.NewReader(items[0])) + decoder.DisallowUnknownFields() + if decoder.Decode(&item) != nil || item.APIVersion != "meta.k8s.io/v1" || item.Kind != "PartialObjectMetadata" { + return errors.New("retained pool response is not metadata") + } + for key, expected := range map[string]string{"name": "ksail-analysis-runners", "namespace": "arc-ksail-analysis"} { + var actual string + if json.Unmarshal(item.Metadata[key], &actual) != nil || actual != expected { + return errors.New("retained pool identity does not match its declaration") + } + } + for _, key := range []string{"uid", "resourceVersion"} { + var value string + if json.Unmarshal(item.Metadata[key], &value) != nil || value == "" { + return errors.New("retained pool identity is incomplete") + } + } + if value, exists := item.Metadata["deletionTimestamp"]; exists && !bytes.Equal(bytes.TrimSpace(value), []byte("null")) { + return errors.New("retained pool is being deleted") + } + return nil +} + func requireControllerScope(ctx context.Context, client *http.Client, endpoint string) error { request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { @@ -230,7 +291,7 @@ func startMetadataProxy(ctx context.Context, command func(context.Context, ...st childCtx, cancel := context.WithCancel(ctx) paths := make([]string, len(recoveryEndpoints)+1) for i, path := range recoveryEndpoints { - paths[i] = regexp.QuoteMeta(path) + paths[i] = regexp.QuoteMeta(strings.SplitN(path, "?", 2)[0]) } paths[len(recoveryEndpoints)] = regexp.QuoteMeta(controllerDeploymentPath) cmd := command(childCtx, "--context", "admin@prod", "proxy", "--address=127.0.0.1", "--port=0", "--accept-hosts=^127\\.0\\.0\\.1$", "--accept-paths=^("+strings.Join(paths, "|")+")$", "--reject-methods="+rejectNonGET) @@ -311,11 +372,17 @@ func run(stage string) error { } } for _, endpoint := range recoveryEndpoints { + if endpoint == retainedPoolPath { + if err := requireRetainedMetadata(ctx, client, base+endpoint); err != nil { + return err + } + continue + } if err := requireEmptyMetadata(ctx, client, base+endpoint); err != nil { return err } } - fmt.Printf("ARC drain recovery %s: %d complete metadata lists prove zero active organization resources or ARC pods\n", stage, len(recoveryEndpoints)) + fmt.Printf("ARC drain recovery %s: %d complete metadata lists prove no active organization resources, retained runner children or ARC pods\n", stage, len(recoveryEndpoints)) return nil } diff --git a/scripts/guard-arc-recovery/main_test.go b/scripts/guard-arc-recovery/main_test.go index fda9de707b..8fc1991f46 100644 --- a/scripts/guard-arc-recovery/main_test.go +++ b/scripts/guard-arc-recovery/main_test.go @@ -27,7 +27,10 @@ func TestRecoveryMetadataReadsRequireCompleteEmptyMetadataLists(t *testing.T) { {"full pod list", strings.Replace(empty, "PartialObjectMetadataList", "PodList", 1), 200, false}, {"full data masquerading as metadata", strings.Replace(empty, `"items":[]`, `"spec":{"token":"synthetic"},"items":[]`, 1), 200, false}, {"missing items", strings.Replace(empty, `,"items":[]`, "", 1), 200, false}, - {"null items", strings.Replace(empty, `"items":[]`, `"items":null`, 1), 200, false}, + {"native empty null items", strings.Replace(empty, `"items":[]`, `"items":null`, 1), 200, true}, + {"null items without revision", strings.Replace(strings.Replace(empty, `"items":[]`, `"items":null`, 1), `"resourceVersion":"123"`, `"resourceVersion":""`, 1), 200, false}, + {"paginated null items", strings.Replace(strings.Replace(empty, `"items":[]`, `"items":null`, 1), `"resourceVersion":"123"`, `"resourceVersion":"123","continue":"next"`, 1), 200, false}, + {"object items", strings.Replace(empty, `"items":[]`, `"items":{}`, 1), 200, false}, {"pagination", strings.Replace(empty, `"resourceVersion":"123"`, `"resourceVersion":"123","continue":"next"`, 1), 200, false}, {"missing revision", strings.Replace(empty, `"resourceVersion":"123"`, `"resourceVersion":""`, 1), 200, false}, {"wrong api", strings.Replace(empty, "meta.k8s.io/v1", "v1", 1), 200, false}, @@ -57,6 +60,39 @@ func TestRecoveryMetadataReadsRequireCompleteEmptyMetadataLists(t *testing.T) { } } +func TestRecoveryObservesRetainedPoolWithoutConfusingItWithActiveRunners(t *testing.T) { + const item = `{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadata","metadata":{"name":"ksail-analysis-runners","namespace":"arc-ksail-analysis","uid":"retained","resourceVersion":"123"}}` + const prefix = `{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{"resourceVersion":"123"},"items":` + for _, tc := range []struct { + name, body string + ok bool + }{ + {"not yet installed", prefix + `null}`, true}, + {"retained declaration", prefix + `[` + item + `]}`, true}, + {"unexpected pool", prefix + `[` + strings.Replace(item, "ksail-analysis-runners", "other", 1) + `]}`, false}, + {"foreign namespace", prefix + `[` + strings.Replace(item, "arc-ksail-analysis", "arc-runners", 1) + `]}`, false}, + {"missing UID", prefix + `[` + strings.Replace(item, `"uid":"retained"`, `"uid":""`, 1) + `]}`, false}, + {"missing revision", prefix + `[` + strings.Replace(item, `"resourceVersion":"123"`, `"resourceVersion":""`, 1) + `]}`, false}, + {"multiple pools", prefix + `[` + item + `,` + item + `]}`, false}, + {"full resource", prefix + `[` + strings.Replace(item, `"metadata":`, `"spec":{"maxRunners":0},"metadata":`, 1) + `]}`, false}, + {"pagination", strings.Replace(prefix, `"resourceVersion":"123"`, `"resourceVersion":"123","continue":"next"`, 1) + `null}`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" || r.Header.Get("Accept") != metadataAccept { + t.Error("retained pool observation widened") + } + _, _ = w.Write([]byte(tc.body)) + })) + defer server.Close() + err := requireRetainedMetadata(context.Background(), metadataClient(), server.URL) + if (err == nil) != tc.ok { + t.Fatalf("accepted=%v, want %v: %v", err == nil, tc.ok, err) + } + }) + } +} + func TestRecoveryDeclarationRefusesSuspensionAndImplicitRunnerBounds(t *testing.T) { for _, tc := range []struct { name, controller, legacy string @@ -99,14 +135,17 @@ func TestNativeMetadataProxyPreservesAcceptAndRejectsWrites(t *testing.T) { if err != nil { t.Fatal("native metadata-proxy regression requires kubectl") } - requests := make(chan string, 16) + requests := make(chan string, len(recoveryEndpoints)+1) server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests <- r.Method + " " + r.URL.Path if r.Method != "GET" || r.Header.Get("Accept") != metadataAccept { t.Error("native proxy widened the read") } + if r.URL.Path == "/api/v1/namespaces/arc-systems/pods" && r.URL.Query().Get("labelSelector") != "platform.devantler.tech/arc-role=listener" { + t.Error("native proxy lost the retained listener selector") + } w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{"resourceVersion":"123"},"items":[]}`)) + _, _ = w.Write([]byte(`{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{"resourceVersion":"123"},"items":null}`)) })) defer server.Close() cert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) @@ -169,7 +208,7 @@ func TestNativeMetadataProxyPreservesAcceptAndRejectsWrites(t *testing.T) { } func TestRecoveryUsesOnlyMetadataEndpoints(t *testing.T) { - if len(recoveryEndpoints) != 12 { + if len(recoveryEndpoints) != 17 { t.Fatal("incomplete recovery observation set") } for _, path := range recoveryEndpoints { From 8d73c64ba32887f1d8112ace30362faba48651b1 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 15:15:36 +0200 Subject: [PATCH 35/62] test(arc): exercise retained pool metadata through native proxy --- docs/operations/arc-runners.md | 10 ++++++++-- scripts/guard-arc-recovery/main_test.go | 11 +++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/operations/arc-runners.md b/docs/operations/arc-runners.md index 9cc4fed599..a67e0d2911 100644 --- a/docs/operations/arc-runners.md +++ b/docs/operations/arc-runners.md @@ -8,8 +8,14 @@ zero minimum and maximum runner bounds. Suspension stops Helm reconciliation, does not drain an installed scale set, and prevents a failed release from recovering its readiness status. Native Flux and Helm health checks stay enabled. The protected deploy proves complete, empty metadata lists for organization runner sets, releases, -credential-sync resources and both runner namespaces' pods before publication and -after reconciliation. It requests no App credentials, JIT configuration or full +credential-sync resources, retained runner children, both runner namespaces' pods +and labeled listeners in the controller namespace before publication and after +reconciliation. The retained namespace may contain only the exact chart-declared +AutoscalingRunnerSet; Helm keeps that declaration with both runner bounds at zero. +After reconciliation the guard also proves the installed controller is fully +rolled out and excludes that retained namespace. Native empty `items: null` is +accepted only with a complete current-revision list; missing or paginated items +cannot prove absence. It requests no App credentials, JIT configuration or full Pod responses. Registration, execution and cleanup still need the separate proofs below before KSail #7131 can close. diff --git a/scripts/guard-arc-recovery/main_test.go b/scripts/guard-arc-recovery/main_test.go index 8fc1991f46..076035fdfc 100644 --- a/scripts/guard-arc-recovery/main_test.go +++ b/scripts/guard-arc-recovery/main_test.go @@ -73,6 +73,7 @@ func TestRecoveryObservesRetainedPoolWithoutConfusingItWithActiveRunners(t *test {"foreign namespace", prefix + `[` + strings.Replace(item, "arc-ksail-analysis", "arc-runners", 1) + `]}`, false}, {"missing UID", prefix + `[` + strings.Replace(item, `"uid":"retained"`, `"uid":""`, 1) + `]}`, false}, {"missing revision", prefix + `[` + strings.Replace(item, `"resourceVersion":"123"`, `"resourceVersion":""`, 1) + `]}`, false}, + {"being deleted", prefix + `[` + strings.Replace(item, `"uid":"retained"`, `"uid":"retained","deletionTimestamp":"2026-10-06T00:00:00Z"`, 1) + `]}`, false}, {"multiple pools", prefix + `[` + item + `,` + item + `]}`, false}, {"full resource", prefix + `[` + strings.Replace(item, `"metadata":`, `"spec":{"maxRunners":0},"metadata":`, 1) + `]}`, false}, {"pagination", strings.Replace(prefix, `"resourceVersion":"123"`, `"resourceVersion":"123","continue":"next"`, 1) + `null}`, false}, @@ -145,6 +146,10 @@ func TestNativeMetadataProxyPreservesAcceptAndRejectsWrites(t *testing.T) { t.Error("native proxy lost the retained listener selector") } w.Header().Set("Content-Type", "application/json") + if r.URL.Path == retainedPoolPath { + _, _ = w.Write([]byte(`{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{"resourceVersion":"123"},"items":[{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadata","metadata":{"name":"ksail-analysis-runners","namespace":"arc-ksail-analysis","uid":"retained","resourceVersion":"123"}}]}`)) + return + } _, _ = w.Write([]byte(`{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{"resourceVersion":"123"},"items":null}`)) })) defer server.Close() @@ -169,6 +174,12 @@ func TestNativeMetadataProxyPreservesAcceptAndRejectsWrites(t *testing.T) { client := metadataClient() defer client.CloseIdleConnections() for _, path := range recoveryEndpoints { + if path == retainedPoolPath { + if err := requireRetainedMetadata(ctx, client, base+path); err != nil { + t.Fatal(err) + } + continue + } if err := requireEmptyMetadata(ctx, client, base+path); err != nil { t.Fatal(err) } From e2501abca19948a9f37f42e422fd42665965a34e Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 15:47:48 +0200 Subject: [PATCH 36/62] fix(arc): reject ambiguous recovery proofs and argument grammar --- scripts/guard-arc-recovery/main.go | 88 +++++++++++++++++++++++++ scripts/guard-arc-recovery/main_test.go | 17 +++++ 2 files changed, 105 insertions(+) diff --git a/scripts/guard-arc-recovery/main.go b/scripts/guard-arc-recovery/main.go index 365bf42383..3353dc1bd3 100644 --- a/scripts/guard-arc-recovery/main.go +++ b/scripts/guard-arc-recovery/main.go @@ -108,6 +108,84 @@ func metadataClient() *http.Client { } } +// encoding/json accepts duplicate object keys and case-insensitive struct field +// aliases. Neither can establish absence or an effective controller scope. +// Check the complete bounded document before any typed decoding; unrelated +// Deployment fields remain readable without making their spelling a proof gate. +func requireUnambiguousJSON(data []byte) error { + fields := map[string][]string{ + "": {"apiVersion", "kind", "metadata", "items", "spec", "status"}, + "metadata": {"name", "namespace", "uid", "generation", "deletionTimestamp", "resourceVersion", "continue", "remainingItemCount"}, + "items[]": {"apiVersion", "kind", "metadata"}, + "items[].metadata": {"name", "namespace", "uid", "resourceVersion", "deletionTimestamp"}, + "spec": {"replicas", "template"}, + "spec.template": {"spec"}, + "spec.template.spec": {"containers"}, + "spec.template.spec.containers[]": {"name", "command", "args"}, + "status": {"observedGeneration", "replicas", "updatedReplicas", "readyReplicas", "availableReplicas", "unavailableReplicas"}, + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + var walk func(string, int) error + walk = func(path string, depth int) error { + if depth > 64 { + return errors.New("JSON nesting exceeds the proof bound") + } + token, err := decoder.Token() + if err != nil { + return errors.New("JSON proof is incomplete") + } + delimiter, composite := token.(json.Delim) + if !composite { + return nil + } + switch delimiter { + case '{': + seen := map[string]bool{} + for decoder.More() { + token, err := decoder.Token() + key, ok := token.(string) + if err != nil || !ok || seen[strings.ToLower(key)] { + return errors.New("JSON proof contains ambiguous object fields") + } + seen[strings.ToLower(key)] = true + for _, canonical := range fields[path] { + if key != canonical && strings.EqualFold(key, canonical) { + return errors.New("JSON proof contains a case-aliased field") + } + } + child := key + if path != "" { + child = path + "." + key + } + if err := walk(child, depth+1); err != nil { + return err + } + } + case '[': + for decoder.More() { + if err := walk(path+"[]", depth+1); err != nil { + return err + } + } + default: + return errors.New("JSON proof contains an unexpected delimiter") + } + end, err := decoder.Token() + if err != nil || (delimiter == '{' && end != json.Delim('}')) || (delimiter == '[' && end != json.Delim(']')) { + return errors.New("JSON proof is incomplete") + } + return nil + } + if err := walk("", 0); err != nil { + return err + } + if _, err := decoder.Token(); err != io.EOF { + return errors.New("JSON proof has trailing data") + } + return nil +} + func readMetadata(ctx context.Context, client *http.Client, endpoint string) ([]json.RawMessage, error) { request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { @@ -126,6 +204,9 @@ func readMetadata(ctx context.Context, client *http.Client, endpoint string) ([] if err != nil || len(data) > 1<<20 { return nil, errors.New("metadata response incomplete or oversized") } + if err := requireUnambiguousJSON(data); err != nil { + return nil, err + } var list struct { APIVersion string `json:"apiVersion"` Kind string `json:"kind"` @@ -227,6 +308,9 @@ func requireControllerScope(ctx context.Context, client *http.Client, endpoint s if err != nil || len(data) > 1<<20 { return errors.New("controller scope response incomplete or oversized") } + if err := requireUnambiguousJSON(data); err != nil { + return err + } var deployment struct { APIVersion string `json:"apiVersion"` Kind string `json:"kind"` @@ -263,7 +347,11 @@ func requireControllerScope(ctx context.Context, client *http.Client, endpoint s return errors.New("controller executable differs from the pinned chart") } scope, mode := 0, 0 + option := regexp.MustCompile(`^--[a-z][a-z0-9-]*(=.*)?$`) for _, arg := range container.Args { + if !option.MatchString(arg) { + return errors.New("controller arguments contain an option terminator or operand") + } switch { case strings.HasPrefix(arg, "--watch-single-namespace"): if arg != "--watch-single-namespace=arc-runners" { diff --git a/scripts/guard-arc-recovery/main_test.go b/scripts/guard-arc-recovery/main_test.go index 076035fdfc..6a98058036 100644 --- a/scripts/guard-arc-recovery/main_test.go +++ b/scripts/guard-arc-recovery/main_test.go @@ -28,6 +28,12 @@ func TestRecoveryMetadataReadsRequireCompleteEmptyMetadataLists(t *testing.T) { {"full data masquerading as metadata", strings.Replace(empty, `"items":[]`, `"spec":{"token":"synthetic"},"items":[]`, 1), 200, false}, {"missing items", strings.Replace(empty, `,"items":[]`, "", 1), 200, false}, {"native empty null items", strings.Replace(empty, `"items":[]`, `"items":null`, 1), 200, true}, + {"duplicate live items hidden by null", strings.Replace(empty, `"items":[]`, `"items":[{"metadata":{"name":"active"}}],"items":null`, 1), 200, false}, + {"duplicate continuation hidden by empty", strings.Replace(empty, `"resourceVersion":"123"`, `"resourceVersion":"123","continue":"next","continue":""`, 1), 200, false}, + {"duplicate remaining count hidden by zero", strings.Replace(empty, `"resourceVersion":"123"`, `"resourceVersion":"123","remainingItemCount":1,"remainingItemCount":0`, 1), 200, false}, + {"case aliased items", strings.Replace(empty, `"items":[]`, `"Items":[]`, 1), 200, false}, + {"case aliased revision", strings.Replace(empty, `"resourceVersion"`, `"ResourceVersion"`, 1), 200, false}, + {"case alias hiding live items", strings.Replace(empty, `"items":[]`, `"items":[{"metadata":{"name":"active"}}],"Items":null`, 1), 200, false}, {"null items without revision", strings.Replace(strings.Replace(empty, `"items":[]`, `"items":null`, 1), `"resourceVersion":"123"`, `"resourceVersion":""`, 1), 200, false}, {"paginated null items", strings.Replace(strings.Replace(empty, `"items":[]`, `"items":null`, 1), `"resourceVersion":"123"`, `"resourceVersion":"123","continue":"next"`, 1), 200, false}, {"object items", strings.Replace(empty, `"items":[]`, `"items":{}`, 1), 200, false}, @@ -74,6 +80,8 @@ func TestRecoveryObservesRetainedPoolWithoutConfusingItWithActiveRunners(t *test {"missing UID", prefix + `[` + strings.Replace(item, `"uid":"retained"`, `"uid":""`, 1) + `]}`, false}, {"missing revision", prefix + `[` + strings.Replace(item, `"resourceVersion":"123"`, `"resourceVersion":""`, 1) + `]}`, false}, {"being deleted", prefix + `[` + strings.Replace(item, `"uid":"retained"`, `"uid":"retained","deletionTimestamp":"2026-10-06T00:00:00Z"`, 1) + `]}`, false}, + {"duplicate retained identity", prefix + `[` + strings.Replace(item, `"name":"ksail-analysis-runners"`, `"name":"foreign","name":"ksail-analysis-runners"`, 1) + `]}`, false}, + {"aliased retained kind", prefix + `[` + strings.Replace(item, `"kind"`, `"Kind"`, 1) + `]}`, false}, {"multiple pools", prefix + `[` + item + `,` + item + `]}`, false}, {"full resource", prefix + `[` + strings.Replace(item, `"metadata":`, `"spec":{"maxRunners":0},"metadata":`, 1) + `]}`, false}, {"pagination", strings.Replace(prefix, `"resourceVersion":"123"`, `"resourceVersion":"123","continue":"next"`, 1) + `null}`, false}, @@ -244,6 +252,15 @@ func TestRecoveryRequiresAppliedControllerScope(t *testing.T) { {"duplicate scope", strings.Replace(recoveredDeployment, `"--watch-single-namespace=arc-runners"`, `"--watch-single-namespace=arc-runners","--watch-single-namespace=arc-ksail-analysis"`, 1), false}, {"broad scope", strings.Replace(recoveredDeployment, "--watch-single-namespace=arc-runners", "--watch-namespace=", 1), false}, {"legacy mode", strings.Replace(recoveredDeployment, "--auto-scaling-runner-set-only", "--other-mode", 1), false}, + {"option terminator", strings.Replace(recoveredDeployment, `"args":[`, `"args":["--",`, 1), false}, + {"positional before scope", strings.Replace(recoveredDeployment, `"args":[`, `"args":["operand",`, 1), false}, + {"positional after scope", strings.Replace(recoveredDeployment, `"--watch-single-namespace=arc-runners"`, `"--watch-single-namespace=arc-runners","operand"`, 1), false}, + {"short option", strings.Replace(recoveredDeployment, `"args":[`, `"args":["-x",`, 1), false}, + {"duplicate stale observed generation", strings.Replace(recoveredDeployment, `"observedGeneration":2`, `"observedGeneration":1,"observedGeneration":2`, 1), false}, + {"duplicate controller arguments", strings.Replace(recoveredDeployment, `"args":[`, `"args":["--watch-single-namespace=foreign"],"args":[`, 1), false}, + {"case aliased ready count", strings.Replace(recoveredDeployment, `"readyReplicas"`, `"ReadyReplicas"`, 1), false}, + {"case aliased args", strings.Replace(recoveredDeployment, `"args"`, `"Args"`, 1), false}, + {"case aliased UID", strings.Replace(recoveredDeployment, `"uid"`, `"UID"`, 1), false}, {"stale generation", strings.Replace(recoveredDeployment, `"observedGeneration":2`, `"observedGeneration":1`, 1), false}, {"old replica", strings.Replace(recoveredDeployment, `"replicas":1,"updatedReplicas"`, `"replicas":2,"updatedReplicas"`, 1), false}, {"unready", strings.Replace(recoveredDeployment, `"readyReplicas":1`, `"readyReplicas":0`, 1), false}, From daa62aa3c42ae9af3f168547366fd0ead0ecadf5 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 15:57:09 +0200 Subject: [PATCH 37/62] fix(arc): preserve case-sensitive Kubernetes metadata keys --- scripts/guard-arc-recovery/main.go | 4 ++-- scripts/guard-arc-recovery/main_test.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/guard-arc-recovery/main.go b/scripts/guard-arc-recovery/main.go index 3353dc1bd3..ac5fbafcfa 100644 --- a/scripts/guard-arc-recovery/main.go +++ b/scripts/guard-arc-recovery/main.go @@ -145,10 +145,10 @@ func requireUnambiguousJSON(data []byte) error { for decoder.More() { token, err := decoder.Token() key, ok := token.(string) - if err != nil || !ok || seen[strings.ToLower(key)] { + if err != nil || !ok || seen[key] { return errors.New("JSON proof contains ambiguous object fields") } - seen[strings.ToLower(key)] = true + seen[key] = true for _, canonical := range fields[path] { if key != canonical && strings.EqualFold(key, canonical) { return errors.New("JSON proof contains a case-aliased field") diff --git a/scripts/guard-arc-recovery/main_test.go b/scripts/guard-arc-recovery/main_test.go index 6a98058036..b3a04cfbba 100644 --- a/scripts/guard-arc-recovery/main_test.go +++ b/scripts/guard-arc-recovery/main_test.go @@ -248,6 +248,7 @@ func TestRecoveryRequiresAppliedControllerScope(t *testing.T) { ok bool }{ {"recovered", recoveredDeployment, true}, + {"valid case-sensitive metadata keys", strings.Replace(recoveredDeployment, `"generation":2`, `"generation":2,"labels":{"App":"one","app":"two"},"annotations":{"Description":"one","description":"two"}`, 1), true}, {"legacy scope", strings.Replace(recoveredDeployment, "--watch-single-namespace=arc-runners", "--watch-single-namespace=arc-ksail-analysis", 1), false}, {"duplicate scope", strings.Replace(recoveredDeployment, `"--watch-single-namespace=arc-runners"`, `"--watch-single-namespace=arc-runners","--watch-single-namespace=arc-ksail-analysis"`, 1), false}, {"broad scope", strings.Replace(recoveredDeployment, "--watch-single-namespace=arc-runners", "--watch-namespace=", 1), false}, From 677e5bf7b5fe56095024120e336505b69b46f32f Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 16:05:13 +0200 Subject: [PATCH 38/62] fix(arc): enforce public listener content for credential scan disposition --- .github/workflows/ci.yaml | 3 + docs/operations/arc-runners.md | 9 ++ .../restrict-arc-openbao-listener.yaml | 46 +++++++++ .../cluster-policies/kustomization.yaml | 1 + .../arc-openbao-listener.yaml | 20 ++++ .../kustomization.yaml | 1 + .../config-map-headlamp-exceptions.yaml | 22 +++++ scripts/tests/arc-staging/transport_test.go | 98 +++++++++++++++++++ .../tests/test-arc-listener-public-content.sh | 50 ++++++++++ scripts/tests/test-verify-arc-transport.sh | 22 ++++- scripts/verify-arc-transport.sh | 31 ++++++ 11 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 k8s/bases/infrastructure/cluster-policies/best-practices/restrict-arc-openbao-listener.yaml create mode 100644 k8s/bases/infrastructure/cluster-security-exceptions/arc-openbao-listener.yaml create mode 100644 scripts/tests/test-arc-listener-public-content.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e1310004ff..7b142677f1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -530,6 +530,7 @@ jobs: - 'scripts/tests/test-headlamp-plugin-removal.sh' - 'scripts/tests/test-github-config-role-activation-parity.sh' - 'scripts/tests/test-restrict-tenant-secret-stores.sh' + - 'scripts/tests/test-arc-listener-public-content.sh' - 'scripts/validate-kyverno-fixture-evaluation.sh' - 'scripts/tests/test-validate-kyverno-fixture-evaluation.sh' # Every kyverno fixture, so a change to one alone still runs kyverno test @@ -1677,6 +1678,8 @@ jobs: shellcheck scripts/validate-kyverno-fixture-evaluation.sh scripts/tests/test-validate-kyverno-fixture-evaluation.sh bash scripts/tests/test-validate-kyverno-fixture-evaluation.sh bash scripts/validate-kyverno-fixture-evaluation.sh tests + shellcheck scripts/tests/test-arc-listener-public-content.sh + bash scripts/tests/test-arc-listener-public-content.sh shellcheck scripts/tests/test-hcloud-volume-priority.sh bash scripts/tests/test-hcloud-volume-priority.sh shellcheck scripts/tests/test-audit-privileged-rbac.sh diff --git a/docs/operations/arc-runners.md b/docs/operations/arc-runners.md index 7b2b006eaa..0bfb8b2cc3 100644 --- a/docs/operations/arc-runners.md +++ b/docs/operations/arc-runners.md @@ -57,6 +57,15 @@ authenticated cluster connection to reach the leader. The existing API listener, Raft storage, audit configuration and unseal hook remain in the base configuration. An additional listener file is appended through the pinned chart. +That ConfigMap contains only public settings and paths into a mounted Secret. +Kubescape's generic credential-text rule matches the mandatory `tls_key_file` +setting, so its disposition covers only the named ConfigMap and that control. +Both platform variants enforce its exact public content through admission; source +tests reject drift, and native scanner controls prove genuine credentials remain +findings. The protected transport challenge verifies the installed policy and +configuration, then proves server admission denies synthetic credentials without +writing them to the cluster. + The dedicated certificate authority is trusted only in the runner namespace. Its issuance policy denies use outside OpenBao. The credential store requires HTTPS and that authority, with hostname verification and no plaintext fallback. diff --git a/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-arc-openbao-listener.yaml b/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-arc-openbao-listener.yaml new file mode 100644 index 0000000000..7501d3e7b3 --- /dev/null +++ b/k8s/bases/infrastructure/cluster-policies/best-practices/restrict-arc-openbao-listener.yaml @@ -0,0 +1,46 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: restrict-arc-openbao-listener + annotations: + policies.kyverno.io/title: Keep the ARC OpenBao Listener Configuration Public + policies.kyverno.io/category: Security + policies.kyverno.io/severity: high +spec: + background: true + rules: + - name: public-listener-settings-only + match: + any: + - resources: + kinds: + - ConfigMap + names: + - arc-openbao-listener + validate: + failureAction: Enforce + message: This listener ConfigMap may contain only audited public settings and Secret-mounted file paths. + deny: + conditions: + any: + - key: "{{ request.object.metadata.namespace || '' }}" + operator: NotEquals + value: openbao + - key: "{{ length(keys(request.object.data || `{}`)) }}" + operator: NotEquals + value: 1 + - key: "{{ length(keys(request.object.binaryData || `{}`)) }}" + operator: NotEquals + value: 0 + - key: >- + {{ request.object.data."listener.hcl" || '' }} + operator: NotEquals + value: | + listener "tcp" { + address = "0.0.0.0:8204" + cluster_address = "127.0.0.1:8205" + tls_cert_file = "/openbao/arc-tls/tls.crt" + tls_key_file = "/openbao/arc-tls/tls.key" + tls_min_version = "tls12" + tls_disable_client_certs = true + } diff --git a/k8s/bases/infrastructure/cluster-policies/kustomization.yaml b/k8s/bases/infrastructure/cluster-policies/kustomization.yaml index 3b74201d38..190ae1c219 100644 --- a/k8s/bases/infrastructure/cluster-policies/kustomization.yaml +++ b/k8s/bases/infrastructure/cluster-policies/kustomization.yaml @@ -15,6 +15,7 @@ resources: - best-practices/propagate-reloader-to-flagger-primary.yaml - best-practices/restrict-github-team-external-identity.yaml - best-practices/restrict-github-team-management.yaml + - best-practices/restrict-arc-openbao-listener.yaml - best-practices/restrict-homepage-service-groups.yaml - best-practices/restrict-tenant-issuer-refs.yaml - best-practices/restrict-tenant-network-policies.yaml diff --git a/k8s/bases/infrastructure/cluster-security-exceptions/arc-openbao-listener.yaml b/k8s/bases/infrastructure/cluster-security-exceptions/arc-openbao-listener.yaml new file mode 100644 index 0000000000..ebc50f4454 --- /dev/null +++ b/k8s/bases/infrastructure/cluster-security-exceptions/arc-openbao-listener.yaml @@ -0,0 +1,20 @@ +apiVersion: kubescape.io/v1beta1 +kind: ClusterSecurityException +metadata: + name: arc-openbao-listener +spec: + reason: >- + C-0012's generic _key_ text pattern matches the mandatory native OpenBao + tls_key_file setting. Its value is a public file path to a mounted Secret, + never credential content. Enforced restrict-arc-openbao-listener admission + and unconditional source tests permit exactly the audited public listener + in openbao, reject extra or binary data and reject this name elsewhere. + Other ConfigMaps, controls and genuine credentials remain scanned. + match: + resources: + - apiGroup: "" + kind: ConfigMap + name: ^arc-openbao-listener$ + posture: + - controlID: C-0012 + action: ignore diff --git a/k8s/bases/infrastructure/cluster-security-exceptions/kustomization.yaml b/k8s/bases/infrastructure/cluster-security-exceptions/kustomization.yaml index e8c276485e..794bdd946d 100644 --- a/k8s/bases/infrastructure/cluster-security-exceptions/kustomization.yaml +++ b/k8s/bases/infrastructure/cluster-security-exceptions/kustomization.yaml @@ -3,6 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - admission-controllers.yaml + - arc-openbao-listener.yaml - batch-workloads.yaml - cilium-network-policies.yaml - controller-rbac.yaml diff --git a/k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml b/k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml index 93be3d39ec..d377069664 100644 --- a/k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml +++ b/k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml @@ -91,6 +91,28 @@ data: ], "reason": "Admission webhook timeouts are configured by upstream Helm charts (Kyverno, cert-manager, VPA, KEDA) at appropriate values for their operation. Changing these would cause admission failures." }, + { + "name": "arc-openbao-listener", + "policyType": "postureExceptionPolicy", + "actions": [ + "disable" + ], + "resources": [ + { + "designatorType": "Attributes", + "attributes": { + "kind": "^ConfigMap$", + "name": "^arc-openbao-listener$" + } + } + ], + "posturePolicies": [ + { + "controlID": "^C-0012$" + } + ], + "reason": "C-0012's generic _key_ text pattern matches the mandatory native OpenBao tls_key_file setting. Its value is a public file path to a mounted Secret, never credential content. Enforced restrict-arc-openbao-listener admission and unconditional source tests permit exactly the audited public listener in openbao, reject extra or binary data and reject this name elsewhere. Other ConfigMaps, controls and genuine credentials remain scanned." + }, { "name": "batch-workloads", "policyType": "postureExceptionPolicy", diff --git a/scripts/tests/arc-staging/transport_test.go b/scripts/tests/arc-staging/transport_test.go index e7e66de8d4..5474ca41fe 100644 --- a/scripts/tests/arc-staging/transport_test.go +++ b/scripts/tests/arc-staging/transport_test.go @@ -7,6 +7,104 @@ import ( const transportPath = "k8s/providers/hetzner/infrastructure/controllers/openbao/transport/" +const publicListenerHCL = `listener "tcp" { + address = "0.0.0.0:8204" + cluster_address = "127.0.0.1:8205" + tls_cert_file = "/openbao/arc-tls/tls.crt" + tls_key_file = "/openbao/arc-tls/tls.key" + tls_min_version = "tls12" + tls_disable_client_certs = true +} +` + +func TestListenerCredentialFalsePositiveRequiresPublicContentEnforcement(t *testing.T) { + config := readYAML(t, transportPath+"config-map-listener.yaml") + equal(t, field(t, config, "metadata", "name"), "arc-openbao-listener") + equal(t, field(t, config, "metadata", "namespace"), "openbao") + data := field(t, config, "data").(map[string]any) + if len(data) != 1 { + t.Fatal("the excepted ConfigMap may contain only the public listener") + } + equal(t, data["listener.hcl"], publicListenerHCL) + if _, exists := config["binaryData"]; exists { + t.Fatal("the excepted ConfigMap must not contain opaque data") + } + exception := readYAML(t, "k8s/bases/infrastructure/cluster-security-exceptions/arc-openbao-listener.yaml") + resources := field(t, exception, "spec", "match", "resources").([]any) + controls := field(t, exception, "spec", "posture").([]any) + if len(resources) != 1 || len(controls) != 1 { + t.Fatal("only this ConfigMap's credential-text false positive may be excepted") + } + equal(t, field(t, resources[0], "kind"), "ConfigMap") + equal(t, field(t, resources[0], "name"), "^arc-openbao-listener$") + equal(t, field(t, controls[0], "controlID"), "C-0012") + equal(t, field(t, controls[0], "action"), "ignore") + policy := readYAML(t, "k8s/bases/infrastructure/cluster-policies/best-practices/restrict-arc-openbao-listener.yaml") + spec := field(t, policy, "spec").(map[string]any) + if len(spec) != 2 || spec["background"] != true { + t.Fatal("listener admission must retain its default admission mode and background audit") + } + rules := field(t, policy, "spec", "rules").([]any) + if len(rules) != 1 { + t.Fatal("public listener admission must have exactly one enforced rule") + } + rule := rules[0].(map[string]any) + if len(rule) != 3 || len(field(t, rule, "validate").(map[string]any)) != 3 { + t.Fatal("listener admission must not add a bypass or action override") + } + for _, forbidden := range []string{"exclude", "preconditions"} { + if _, exists := rule[forbidden]; exists { + t.Fatalf("public listener admission must not bypass %s", forbidden) + } + } + equal(t, field(t, rule, "validate", "failureAction"), "Enforce") + match := field(t, rule, "match", "any").([]any) + if len(match) != 1 { + t.Fatal("listener admission must have one global resource match") + } + selector := field(t, match[0], "resources").(map[string]any) + if len(selector) != 2 { + t.Fatal("listener admission must cover its name in every namespace") + } + kinds := field(t, selector, "kinds").([]any) + names := field(t, selector, "names").([]any) + if len(kinds) != 1 || len(names) != 1 { + t.Fatal("listener admission resource scope differs from its disposition") + } + equal(t, kinds[0], "ConfigMap") + equal(t, names[0], "arc-openbao-listener") + conditions := field(t, rule, "validate", "deny", "conditions", "any").([]any) + expected := map[string]any{ + "{{ request.object.metadata.namespace || '' }}": "openbao", + "{{ length(keys(request.object.data || `{}`)) }}": 1, + "{{ length(keys(request.object.binaryData || `{}`)) }}": 0, + "{{ request.object.data.\"listener.hcl\" || '' }}": publicListenerHCL, + } + if len(conditions) != len(expected) { + t.Fatal("listener admission must enforce every public-content premise") + } + for _, condition := range conditions { + key := field(t, condition, "key").(string) + want, exists := expected[key] + if !exists { + t.Fatal("listener admission has an unexpected or duplicate condition") + } + equal(t, field(t, condition, "operator"), "NotEquals") + equal(t, field(t, condition, "value"), want) + delete(expected, key) + } + aggregate := readYAML(t, "k8s/bases/infrastructure/cluster-policies/kustomization.yaml") + count := 0 + for _, resource := range field(t, aggregate, "resources").([]any) { + if resource == "best-practices/restrict-arc-openbao-listener.yaml" { + count++ + } + } + if count != 1 { + t.Fatal("both providers must deploy the public-content admission policy once") + } +} + func TestCredentialStoreRequiresDedicatedVerifiedTLS(t *testing.T) { store := readYAML(t, "k8s/bases/infrastructure/actions-runners/credentials/secret-store.yaml") vault := field(t, store, "spec", "provider", "vault") diff --git a/scripts/tests/test-arc-listener-public-content.sh b/scripts/tests/test-arc-listener-public-content.sh new file mode 100644 index 0000000000..3d3b44692c --- /dev/null +++ b/scripts/tests/test-arc-listener-public-content.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Exercise native admission and the real scanner; no cluster or credentials. +set -euo pipefail +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +cd "$repo_root" +policy=k8s/bases/infrastructure/cluster-policies/best-practices/restrict-arc-openbao-listener.yaml +listener=k8s/providers/hetzner/infrastructure/controllers/openbao/transport/config-map-listener.yaml +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +fail() { printf 'ARC public listener: FAIL %s\n' "$1" >&2; exit 1; } +yq -o=json '.' "$listener" >"$scratch/public.json" +kyverno apply "$policy" --resource "$scratch/public.json" --remove-color >"$scratch/admission" 2>&1 || fail public-settings +grep -Fq 'pass: 1, fail: 0, warn: 0, error: 0, skip: 0' "$scratch/admission" || fail unevaluated-positive + +for change in \ + '.data["listener.hcl"] += "synthetic PRIVATE KEY content\n"' \ + '.data["listener.hcl"] |= sub("/openbao/arc-tls/tls.key"; "synthetic PRIVATE KEY")' \ + '.data.password = "synthetic"' \ + '.binaryData = {credential:"c3ludGhldGlj"}' \ + '.metadata.namespace = "elsewhere"' \ + 'del(.data)' \ + '.data["listener.hcl"] += "\n"' \ + '.data["listener.hcl"] |= sub("tls12"; "tls10")'; do + jq "$change" "$scratch/public.json" >"$scratch/denied.json" + if kyverno apply "$policy" --resource "$scratch/denied.json" --remove-color >"$scratch/admission" 2>&1; then + fail admitted-negative + fi + grep -Fq 'pass: 0, fail: 1, warn: 0, error: 0, skip: 0' "$scratch/admission" || fail unevaluated-negative +done + +go run ./scripts/generate-kubescape-exceptions -o "$scratch/exceptions.json" >"$scratch/generator" 2>&1 +scan() { + ksail workload scan --no-render --framework nsa,mitre --format json --output "$scratch/$1.json" \ + "${@:2}" >"$scratch/scanner" 2>&1 || fail scanner-execution +} +scan baseline "$listener" +jq -e '.summaryDetails.controls[] | select(.controlID == "C-0012") | + .ResourceCounters.failedResources == 1 and .subStatusCounters.ignoredResources == 0' \ + "$scratch/baseline.json" >/dev/null || fail missing-false-positive +scan justified --exceptions "$scratch/exceptions.json" "$listener" +jq -e '.summaryDetails.controls[] | select(.controlID == "C-0012") | + .ResourceCounters.failedResources == 0 and .subStatusCounters.ignoredResources == 1' \ + "$scratch/justified.json" >/dev/null || fail unapplied-disposition +jq '.metadata.name = "synthetic-credential-control" | .data = {password:"synthetic PRIVATE KEY"}' \ + "$scratch/public.json" >"$scratch/credential-input.json" +scan credential --exceptions "$scratch/exceptions.json" "$scratch/credential-input.json" +jq -e '.summaryDetails.controls[] | select(.controlID == "C-0012") | + .ResourceCounters.failedResources == 1 and .subStatusCounters.ignoredResources == 0' \ + "$scratch/credential.json" >/dev/null || fail hidden-genuine-credential +printf 'ARC public listener: native admission 1 pass/8 denies; scanner false positive, exact disposition and credential control verified.\n' diff --git a/scripts/tests/test-verify-arc-transport.sh b/scripts/tests/test-verify-arc-transport.sh index 4cdb3db9b4..3c38067cab 100755 --- a/scripts/tests/test-verify-arc-transport.sh +++ b/scripts/tests/test-verify-arc-transport.sh @@ -11,6 +11,10 @@ readonly head=1111111111111111111111111111111111111111 export GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform GITHUB_REF=refs/heads/main GITHUB_SHA="$head" GITHUB_RUN_ID=12345 GITHUB_RUN_ATTEMPT=1 yq -o=json '.spec.rules[0]' k8s/providers/hetzner/infrastructure/cluster-policies/restrict-arc-openbao-certificate.yaml \ | jq '{spec:{rules:[.]},status:{conditions:[{type:"Ready",status:"True"}]}}' >"$scratch/policy.json" +yq -o=json '.spec' k8s/bases/infrastructure/cluster-policies/best-practices/restrict-arc-openbao-listener.yaml \ + | jq '{metadata:{uid:"policy-uid"},spec:.,status:{conditions:[{type:"Ready",status:"True"}]}}' >"$scratch/listener-policy.json" +yq -o=json '.' k8s/providers/hetzner/infrastructure/controllers/openbao/transport/config-map-listener.yaml \ + | jq '.metadata.uid="listener-uid" | .metadata.resourceVersion="123"' >"$scratch/listener.json" cat >"$scratch/bin/gh" <<'MOCK' #!/usr/bin/env bash printf '%s\n' "${GITHUB_SHA}" @@ -47,6 +51,13 @@ case "$args" in 'get secretstore '*) jq -n '{spec:{provider:{vault:{server:"https://openbao-arc.openbao.svc.cluster.local:8204",caProvider:{type:"ConfigMap",name:"arc-openbao-ca",key:"ca.crt"},path:"secret",version:"v2",auth:{kubernetes:{mountPath:"kubernetes",role:"arc-secret-reader",serviceAccountRef:{name:"arc-secret-reader"}}}}}}}' ;; 'get configmap arc-openbao-ca '*) printf '{"data":{"ca.crt":"-----BEGIN CERTIFICATE-----fixture"}}' ;; 'get configmap cilium-config '*) printf '{"data":{"cluster-name":"prod"}}' ;; + 'get configmap arc-openbao-listener '*) + if [[ "${PROOF_CASE:-}" == listener_content ]]; then jq '.data.password="synthetic"' "$PROOF_TEST_ROOT/listener.json" + elif [[ "${PROOF_CASE:-}" == listener_churn && -e "$state/attempt" ]]; then jq '.metadata.uid="replacement-listener"' "$PROOF_TEST_ROOT/listener.json" + else cat "$PROOF_TEST_ROOT/listener.json"; fi ;; + 'get clusterpolicy restrict-arc-openbao-listener '*) + if [[ "${PROOF_CASE:-}" == listener_audit ]]; then jq '.spec.rules[0].validate.failureAction="Audit"' "$PROOF_TEST_ROOT/listener-policy.json" + else cat "$PROOF_TEST_ROOT/listener-policy.json"; fi ;; 'get clusterpolicy '*) cat "$PROOF_TEST_ROOT/policy.json" ;; 'get kustomization '*) printf '{"metadata":{"generation":1},"status":{"observedGeneration":1,"lastAppliedRevision":"latest@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","conditions":[{"type":"Ready","status":"True"}]}}' ;; 'get service '*) printf '{"spec":{"selector":{"app.kubernetes.io/name":"openbao","app.kubernetes.io/instance":"openbao","statefulset.kubernetes.io/pod-name":"openbao-2"},"ports":[{"name":"arc-tls","port":8204,"targetPort":8204,"protocol":"TCP"}]}}' ;; @@ -62,6 +73,13 @@ case "$args" in 'get pod arc-transport-'*) [[ -e "$state/created" ]] || exit 0 if [[ "${PROOF_CASE:-}" == replacement && -e "$state/attempt" ]]; then jq '.metadata.uid="replacement-uid"' "$state/pod.json"; else cat "$state/pod.json"; fi ;; + 'replace --dry-run=server '*) + file=${args##* -f } + if jq -e --slurpfile public "$PROOF_TEST_ROOT/listener.json" '.data == $public[0].data and ((.binaryData // {}) | length) == 0' "$file" >/dev/null; then exit 0; fi + [[ "${PROOF_CASE:-}" != listener_admit ]] || exit 0 + if [[ "${PROOF_CASE:-}" == listener_api_error ]]; then printf 'connection refused\n' >&2 + else printf 'restrict-arc-openbao-listener public-listener-settings-only denied\n' >&2; fi + exit 1 ;; 'create --dry-run=server '*) file=${args##* -f } if jq -e '.spec.containers[0].securityContext.privileged == true' "$file" >/dev/null; then problem=privileged @@ -106,13 +124,15 @@ run_case() { [[ "$result" == "$expected" ]] || { printf 'FAIL %s: exit %s expected %s\n' "$name" "$result" "$expected"; cat "$scratch/error"; exit 1; } # No secret, TokenRequest, JIT, host exec, policy change or node deletion is permitted. if [[ -f "$scratch/state/calls" ]] && grep -Eqi 'get secret |tokenrequest|create token|ephemeralrunner|delete node|patch |apply ' "$scratch/state/calls"; then exit 1; fi + if [[ -f "$scratch/state/calls" ]] && grep 'replace ' "$scratch/state/calls" | grep -vq 'replace --dry-run=server '; then exit 1; fi if [[ "$name" == replacement ]] && grep -q 'delete --raw=' "$scratch/state/calls"; then printf 'FAIL replacement delete\n'; exit 1; fi cases=$((cases+1)) } GITHUB_REF=refs/heads/feature run_case unreviewed_ref 1 run_case healthy_denied 0 run_case tls_error 60 -for name in sealed stale_probe admit_privilege api_failure wrong_node reachable cert_error wrong_reason lost_events; do run_case "$name" 1; done +for name in sealed stale_probe admit_privilege api_failure wrong_node reachable cert_error wrong_reason lost_events \ + listener_content listener_churn listener_audit listener_admit listener_api_error; do run_case "$name" 1; done run_case replacement 4 run_case cleanup_failure 4 printf 'PASS: %s transport orchestration cases\n' "$cases" diff --git a/scripts/verify-arc-transport.sh b/scripts/verify-arc-transport.sh index 55b7d83fd5..bbebb1b67b 100755 --- a/scripts/verify-arc-transport.sh +++ b/scripts/verify-arc-transport.sh @@ -51,7 +51,33 @@ trap 'printf "ARC transport: FAIL at %s\n" "$stage" >&2' ERR trap 'exit 143' INT TERM go build -trimpath -o "$scratch/evidence" ./scripts/verify-arc-transport >"$scratch/build-output" 2>"$scratch/build-error" +public_listener_boundary() { + kc get clusterpolicy restrict-arc-openbao-listener -o json >"$scratch/listener-policy.json" + yq -o=json '.spec' k8s/bases/infrastructure/cluster-policies/best-practices/restrict-arc-openbao-listener.yaml >"$scratch/listener-spec.json" + jq -e --slurpfile expected "$scratch/listener-spec.json" '.spec == $expected[0] and + any(.status.conditions[]; .type == "Ready" and .status == "True")' "$scratch/listener-policy.json" >/dev/null + kc -n openbao get configmap arc-openbao-listener -o json >"$scratch/listener.json" + yq -o=json '.data' k8s/providers/hetzner/infrastructure/controllers/openbao/transport/config-map-listener.yaml >"$scratch/listener-data.json" + jq -e --slurpfile expected "$scratch/listener-data.json" '.metadata.name == "arc-openbao-listener" and + .metadata.namespace == "openbao" and (.metadata.resourceVersion | type) == "string" and + (.metadata.resourceVersion | length) > 0 and .data == $expected[0] and + ((.binaryData // {}) | length) == 0' "$scratch/listener.json" >/dev/null + quiet kc replace --dry-run=server -f "$scratch/listener.json" + for change in '.data["listener.hcl"] += "synthetic PRIVATE KEY content\n"' \ + '.data.password = "synthetic"' '.binaryData = {credential:"c3ludGhldGlj"}'; do + jq "$change" "$scratch/listener.json" >"$scratch/credential-negative.json" + if kc replace --dry-run=server -f "$scratch/credential-negative.json" >"$scratch/admission-output" 2>"$scratch/admission-error"; then + fail public-listener-admitted-credential + fi + grep -Fq restrict-arc-openbao-listener "$scratch/kubectl-error" || fail public-listener-admission-unknown + grep -Fq public-listener-settings-only "$scratch/kubectl-error" || fail public-listener-admission-unknown + done +} + stage=inactive-credential-boundary +public_listener_boundary +jq -Sc '{uid:.metadata.uid,spec:.spec}' "$scratch/listener-policy.json" >"$scratch/listener-policy-before" +jq -Sc '{uid:.metadata.uid,data:.data,binaryData:.binaryData}' "$scratch/listener.json" >"$scratch/listener-before" # Inspect only metadata counts: neither existing App credentials nor JIT data is read. kc -n "$namespace" get externalsecrets -o json | jq -e '.items | length == 0' >/dev/null kc -n "$namespace" get helmreleases -o json | jq -e 'all(.items[]; .spec.suspend == true)' >/dev/null @@ -204,6 +230,11 @@ while IFS= read -r node; do cleanup_probe || { printf 'ARC transport: FAIL cleanup\n' >&2; exit 4; } done <"$scratch/nodes" stage=final-bindings +public_listener_boundary +jq -Sc '{uid:.metadata.uid,spec:.spec}' "$scratch/listener-policy.json" >"$scratch/after" +cmp -s "$scratch/listener-policy-before" "$scratch/after" || fail listener-policy-churn +jq -Sc '{uid:.metadata.uid,data:.data,binaryData:.binaryData}' "$scratch/listener.json" >"$scratch/after" +cmp -s "$scratch/listener-before" "$scratch/after" || fail listener-configuration-churn for object in 'secretstore openbao' 'configmap arc-openbao-ca'; do read -r kind name <<<"$object" kc -n "$namespace" get "$kind" "$name" -o json | jq -Sc '{uid:.metadata.uid,spec:.spec,data:.data}' >"$scratch/after" From 2aa5f57da2973884e8fc21fc05fe6e2fa08f3838 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 16:16:34 +0200 Subject: [PATCH 39/62] fix(arc): keep generated exception explanations free of credential markers --- .../cluster-security-exceptions/arc-openbao-listener.yaml | 4 ++-- .../kubescape/config-map-headlamp-exceptions.yaml | 2 +- scripts/tests/test-arc-listener-public-content.sh | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/k8s/bases/infrastructure/cluster-security-exceptions/arc-openbao-listener.yaml b/k8s/bases/infrastructure/cluster-security-exceptions/arc-openbao-listener.yaml index ebc50f4454..390673a652 100644 --- a/k8s/bases/infrastructure/cluster-security-exceptions/arc-openbao-listener.yaml +++ b/k8s/bases/infrastructure/cluster-security-exceptions/arc-openbao-listener.yaml @@ -4,8 +4,8 @@ metadata: name: arc-openbao-listener spec: reason: >- - C-0012's generic _key_ text pattern matches the mandatory native OpenBao - tls_key_file setting. Its value is a public file path to a mounted Secret, + C-0012's generic credential-text pattern matches the mandatory native OpenBao + TLS private-key file setting. Its value is a public path to a mounted Secret, never credential content. Enforced restrict-arc-openbao-listener admission and unconditional source tests permit exactly the audited public listener in openbao, reject extra or binary data and reject this name elsewhere. diff --git a/k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml b/k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml index d377069664..6146c3eb26 100644 --- a/k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml +++ b/k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml @@ -111,7 +111,7 @@ data: "controlID": "^C-0012$" } ], - "reason": "C-0012's generic _key_ text pattern matches the mandatory native OpenBao tls_key_file setting. Its value is a public file path to a mounted Secret, never credential content. Enforced restrict-arc-openbao-listener admission and unconditional source tests permit exactly the audited public listener in openbao, reject extra or binary data and reject this name elsewhere. Other ConfigMaps, controls and genuine credentials remain scanned." + "reason": "C-0012's generic credential-text pattern matches the mandatory native OpenBao TLS private-key file setting. Its value is a public path to a mounted Secret, never credential content. Enforced restrict-arc-openbao-listener admission and unconditional source tests permit exactly the audited public listener in openbao, reject extra or binary data and reject this name elsewhere. Other ConfigMaps, controls and genuine credentials remain scanned." }, { "name": "batch-workloads", diff --git a/scripts/tests/test-arc-listener-public-content.sh b/scripts/tests/test-arc-listener-public-content.sh index 3d3b44692c..c02e9a8655 100644 --- a/scripts/tests/test-arc-listener-public-content.sh +++ b/scripts/tests/test-arc-listener-public-content.sh @@ -41,10 +41,14 @@ scan justified --exceptions "$scratch/exceptions.json" "$listener" jq -e '.summaryDetails.controls[] | select(.controlID == "C-0012") | .ResourceCounters.failedResources == 0 and .subStatusCounters.ignoredResources == 1' \ "$scratch/justified.json" >/dev/null || fail unapplied-disposition +scan mirror k8s/bases/infrastructure/controllers/kubescape/config-map-headlamp-exceptions.yaml +jq -e '.summaryDetails.controls[] | select(.controlID == "C-0012") | + .ResourceCounters.passedResources == 1 and .ResourceCounters.failedResources == 0 and .subStatusCounters.ignoredResources == 0' \ + "$scratch/mirror.json" >/dev/null || fail new-mirror-false-positive jq '.metadata.name = "synthetic-credential-control" | .data = {password:"synthetic PRIVATE KEY"}' \ "$scratch/public.json" >"$scratch/credential-input.json" scan credential --exceptions "$scratch/exceptions.json" "$scratch/credential-input.json" jq -e '.summaryDetails.controls[] | select(.controlID == "C-0012") | .ResourceCounters.failedResources == 1 and .subStatusCounters.ignoredResources == 0' \ "$scratch/credential.json" >/dev/null || fail hidden-genuine-credential -printf 'ARC public listener: native admission 1 pass/8 denies; scanner false positive, exact disposition and credential control verified.\n' +printf 'ARC public listener: native admission 1 pass/8 denies; scanner false positive, exact disposition, unexcepted mirror and credential control verified.\n' From 87e446c6dd16037fd65ae2c443cf195280a43ff2 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 16:35:45 +0200 Subject: [PATCH 40/62] fix(arc): preserve enforced listener policy action --- tests/policy-failure-actions.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/policy-failure-actions.json b/tests/policy-failure-actions.json index 890a87b041..7d3206055c 100644 --- a/tests/policy-failure-actions.json +++ b/tests/policy-failure-actions.json @@ -1,5 +1,6 @@ { "restrict-arc-openbao-certificate/only-openbao-may-use-credential-ca": "Enforce", + "restrict-arc-openbao-listener/public-listener-settings-only": "Enforce", "require-baseline-workers-for-hcloud-volumes/require-baseline-worker-affinity": "Enforce", "require-baseline-workers-for-hcloud-volumes/report-claim-lookup-failure": "Audit", "audit-privileged-rbac/privileged-rules": "Enforce", From 847c5e9eb77910a9700b50162ebc02912fa84205 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 16:35:45 +0200 Subject: [PATCH 41/62] fix(arc): reject ambiguous transport evidence --- scripts/verify-arc-transport/json_proof.go | 88 ++++++++++++++++++++++ scripts/verify-arc-transport/main.go | 20 ++++- scripts/verify-arc-transport/proof_test.go | 44 +++++++++++ 3 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 scripts/verify-arc-transport/json_proof.go diff --git a/scripts/verify-arc-transport/json_proof.go b/scripts/verify-arc-transport/json_proof.go new file mode 100644 index 0000000000..4b610dceac --- /dev/null +++ b/scripts/verify-arc-transport/json_proof.go @@ -0,0 +1,88 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "strings" +) + +// Validate each bounded native document before typed decoding can overwrite +// duplicate members or accept case aliases. Unrelated native fields retain +// their spelling and values; only fields used by this proof have canonical names. +func requireTransportJSON(data []byte, health bool) error { + fields := map[string][]string{ + "": {"flow", "lost_events", "lostEvents", "node_status", "nodeStatus"}, + "flow": {"verdict", "drop_reason_desc", "traffic_direction", "source", "destination", "IP", "l4", "node_name", "time"}, + "flow.source": {"namespace", "pod_name"}, + "flow.destination": {"namespace", "pod_name"}, + "flow.IP": {"source", "destination"}, + "flow.l4": {"TCP"}, + "flow.l4.TCP": {"destination_port", "flags"}, + "flow.l4.TCP.flags": {"SYN"}, + } + if health { + fields = map[string][]string{"": {"initialized", "sealed", "version", "cluster_id"}} + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + var walk func(string, int) error + walk = func(path string, depth int) error { + if depth > 64 { + return errors.New("native evidence nesting exceeds bound") + } + token, err := decoder.Token() + if err != nil { + return errors.New("incomplete native evidence") + } + delimiter, composite := token.(json.Delim) + if !composite { + return nil + } + switch delimiter { + case '{': + seen := map[string]bool{} + for decoder.More() { + token, err := decoder.Token() + key, ok := token.(string) + if err != nil || !ok || seen[key] { + return errors.New("ambiguous native evidence fields") + } + seen[key] = true + for _, canonical := range fields[path] { + if key != canonical && strings.EqualFold(key, canonical) { + return errors.New("case-aliased native evidence field") + } + } + child := key + if path != "" { + child = path + "." + key + } + if err := walk(child, depth+1); err != nil { + return err + } + } + case '[': + for decoder.More() { + if err := walk(path+"[]", depth+1); err != nil { + return err + } + } + default: + return errors.New("unexpected native evidence delimiter") + } + end, err := decoder.Token() + if err != nil || (delimiter == '{' && end != json.Delim('}')) || (delimiter == '[' && end != json.Delim(']')) { + return errors.New("incomplete native evidence") + } + return nil + } + if err := walk("", 0); err != nil { + return err + } + if _, err := decoder.Token(); err != io.EOF { + return errors.New("trailing native evidence") + } + return nil +} diff --git a/scripts/verify-arc-transport/main.go b/scripts/verify-arc-transport/main.go index 70494f100e..4a8fe1b284 100644 --- a/scripts/verify-arc-transport/main.go +++ b/scripts/verify-arc-transport/main.go @@ -47,13 +47,20 @@ type flow struct { func verifyHealth(input io.Reader) error { bounded := &io.LimitedReader{R: input, N: 1 << 20} decoder := json.NewDecoder(bounded) + var raw json.RawMessage + if err := decoder.Decode(&raw); err != nil { + return errors.New("invalid health response") + } + if err := requireTransportJSON(raw, true); err != nil { + return err + } var health struct { Initialized *bool `json:"initialized"` Sealed *bool `json:"sealed"` Version string `json:"version"` Cluster string `json:"cluster_id"` } - if err := decoder.Decode(&health); err != nil { + if err := json.Unmarshal(raw, &health); err != nil { return errors.New("invalid health response") } var extra any @@ -71,13 +78,20 @@ func verifyDenial(input io.Reader, expected target) error { decoder := json.NewDecoder(bounded) matched := false for { - var response map[string]json.RawMessage - if err := decoder.Decode(&response); err != nil { + var document json.RawMessage + if err := decoder.Decode(&document); err != nil { if errors.Is(err, io.EOF) { break } return errors.New("malformed observer response") } + if err := requireTransportJSON(document, false); err != nil { + return err + } + var response map[string]json.RawMessage + if err := json.Unmarshal(document, &response); err != nil { + return errors.New("malformed observer response") + } for _, field := range []string{"lost_events", "lostEvents", "node_status", "nodeStatus"} { if _, ok := response[field]; ok { return errors.New("incomplete observer response") diff --git a/scripts/verify-arc-transport/proof_test.go b/scripts/verify-arc-transport/proof_test.go index 2392006ff0..48f0c9d401 100644 --- a/scripts/verify-arc-transport/proof_test.go +++ b/scripts/verify-arc-transport/proof_test.go @@ -19,6 +19,14 @@ func TestHealthRequiresActualUnsealedCanary(t *testing.T) { {`{"initialized":true,"version":"2.6.3","cluster_id":"canary"}`, false}, {`{"initialized":true,"sealed":false,"version":"2.6.3","cluster_id":""}`, false}, {`{"initialized":true,"sealed":false,"version":"2.6.3","cluster_id":"canary"} {}`, false}, + {`{"initialized":true,"sealed":true,"sealed":false,"version":"2.6.3","cluster_id":"canary"}`, false}, + {`{"initialized":true,"sealed":true,"\u0073ealed":false,"version":"2.6.3","cluster_id":"canary"}`, false}, + {`{"initialized":true,"sealed":true,"Sealed":false,"version":"2.6.3","cluster_id":"canary"}`, false}, + {`{"initialized":false,"initialized":true,"sealed":false,"version":"2.6.3","cluster_id":"canary"}`, false}, + {`{"Initialized":true,"sealed":false,"version":"2.6.3","cluster_id":"canary"}`, false}, + {`{"initialized":true,"sealed":false,"version":"2.5.4","version":"2.6.3","cluster_id":"canary"}`, false}, + {`{"initialized":true,"sealed":false,"version":"2.6.3","cluster_id":"other","cluster_id":"canary"}`, false}, + {`{"initialized":true,"sealed":false,"version":"2.6.3","cluster_id":"canary","native":{"Label":"one","label":"two"}}`, true}, } { if err := verifyHealth(strings.NewReader(test.body)); (err == nil) != test.ok { t.Errorf("health accepted=%v want=%v", err == nil, test.ok) @@ -26,6 +34,42 @@ func TestHealthRequiresActualUnsealedCanary(t *testing.T) { } } +func TestDenialRejectsAmbiguousProofFields(t *testing.T) { + start, _ := time.Parse(time.RFC3339Nano, "2026-10-06T12:00:00Z") + want := target{pod: "probe", source: "10.1.1.1", destination: "10.1.1.2", node: "prod/node", port: 8204, since: start, until: start.Add(5 * time.Second)} + valid := `{"flow":{"verdict":"DROPPED","drop_reason_desc":"POLICY_DENIED","traffic_direction":"EGRESS","time":"2026-10-06T12:00:01Z","node_name":"prod/node","source":{"namespace":"arc-runners","pod_name":"probe"},"destination":{"namespace":"openbao","pod_name":"openbao-2"},"IP":{"source":"10.1.1.1","destination":"10.1.1.2"},"l4":{"TCP":{"destination_port":8204,"flags":{"SYN":true}}}}}` + for _, body := range []string{ + strings.Replace(valid, `"flow":`, `"flow":{"verdict":"FORWARDED"},"flow":`, 1), + strings.Replace(valid, `"verdict":"DROPPED"`, `"verdict":"FORWARDED","verdict":"DROPPED"`, 1), + strings.Replace(valid, `"verdict":"DROPPED"`, `"verdict":"FORWARDED","Verdict":"DROPPED"`, 1), + strings.Replace(valid, `"verdict":`, `"Verdict":`, 1), + strings.Replace(valid, `"source":"10.1.1.1"`, `"source":"other","source":"10.1.1.1"`, 1), + strings.Replace(valid, `"destination_port":8204`, `"destination_port":8200,"destination_port":8204`, 1), + strings.Replace(valid, `"SYN":true`, `"SYN":false,"syn":true`, 1), + strings.Replace(valid, `"namespace":"arc-runners"`, `"Namespace":"arc-runners"`, 1), + strings.Replace(valid, `"IP":`, `"ip":`, 1), + strings.Replace(valid, `"TCP":`, `"tcp":`, 1), + strings.Replace(valid, `"node_name":"prod/node"`, `"node_name":"other","node_name":"prod/node"`, 1), + strings.Replace(valid, `"time":"2026-10-06T12:00:01Z"`, `"time":"2026-10-06T11:59:59Z","time":"2026-10-06T12:00:01Z"`, 1), + } { + if err := verifyDenial(strings.NewReader(body), want); err == nil { + t.Errorf("ambiguous proof accepted: %s", body) + } + } + withNative := strings.Replace(valid, `"flow":{`, `"flow":{"native":{"Label":"one","label":"two"},`, 1) + if err := verifyDenial(strings.NewReader(withNative), want); err != nil { + t.Fatalf("unrelated native fields rejected: %v", err) + } + deepNative := strings.Replace(valid, `"flow":{`, `"flow":{"native":`+strings.Repeat(`[`, 65)+`0`+strings.Repeat(`]`, 65)+`,`, 1) + if err := verifyDenial(strings.NewReader(deepNative), want); err == nil { + t.Fatal("excessively nested native evidence accepted") + } + ambiguous := strings.Replace(valid, `"verdict":"DROPPED"`, `"verdict":"FORWARDED","verdict":"DROPPED"`, 1) + if err := verifyDenial(strings.NewReader(valid+"\n"+ambiguous), want); err == nil { + t.Fatal("ambiguous later document hidden by a valid match") + } +} + func TestDenialMustJoinTheActualAttempt(t *testing.T) { start, _ := time.Parse(time.RFC3339Nano, "2026-10-06T12:00:00Z") want := target{pod: "probe", source: "10.1.1.1", destination: "10.1.1.2", node: "prod/node", port: 8204, since: start, until: start.Add(5 * time.Second)} From e2878a0d761f79d2fe716dddeb0bea5e8ffeb01d Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 16:40:45 +0200 Subject: [PATCH 42/62] fix(arc): track scanner regression as an executable entry point --- scripts/tests/test-arc-listener-public-content.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/tests/test-arc-listener-public-content.sh diff --git a/scripts/tests/test-arc-listener-public-content.sh b/scripts/tests/test-arc-listener-public-content.sh old mode 100644 new mode 100755 From d1dd366b6ee9b1fa7c237c4e65cd29b5fac4eda4 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 16:50:20 +0200 Subject: [PATCH 43/62] fix(arc): bind recovery to the rendered retained pool name --- scripts/guard-arc-recovery/main.go | 12 ++++++++---- scripts/guard-arc-recovery/main_test.go | 23 +++++++++++++---------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/scripts/guard-arc-recovery/main.go b/scripts/guard-arc-recovery/main.go index ac5fbafcfa..c23429bd60 100644 --- a/scripts/guard-arc-recovery/main.go +++ b/scripts/guard-arc-recovery/main.go @@ -25,6 +25,7 @@ import ( const metadataAccept = "application/json;as=PartialObjectMetadataList;g=meta.k8s.io;v=v1" const controllerDeploymentPath = "/apis/apps/v1/namespaces/arc-systems/deployments/arc-controller" const retainedPoolPath = "/apis/actions.github.com/v1alpha1/namespaces/arc-ksail-analysis/autoscalingrunnersets" +const retainedPoolName = "ksail-code-quality" // RE2 has no negative lookahead; these alternatives match every nonempty // method except the exact GET spelling, including nonstandard methods. @@ -89,12 +90,13 @@ func recoveryArmed(root string) (bool, error) { Spec struct { Suspend *bool `yaml:"suspend"` Values struct { - Min *int `yaml:"minRunners"` - Max *int `yaml:"maxRunners"` + Name string `yaml:"runnerScaleSetName"` + Min *int `yaml:"minRunners"` + Max *int `yaml:"maxRunners"` } `yaml:"values"` } `yaml:"spec"` } - if yaml.Unmarshal(data, &legacy) != nil || legacy.Spec.Suspend == nil || *legacy.Spec.Suspend || legacy.Spec.Values.Min == nil || legacy.Spec.Values.Max == nil || *legacy.Spec.Values.Min != 0 || *legacy.Spec.Values.Max != 0 { + if yaml.Unmarshal(data, &legacy) != nil || legacy.Spec.Suspend == nil || *legacy.Spec.Suspend || legacy.Spec.Values.Name != retainedPoolName || legacy.Spec.Values.Min == nil || legacy.Spec.Values.Max == nil || *legacy.Spec.Values.Min != 0 || *legacy.Spec.Values.Max != 0 { return false, errors.New("legacy pool is not explicitly drained and reconciling") } return true, nil @@ -272,7 +274,9 @@ func requireRetainedMetadata(ctx context.Context, client *http.Client, endpoint if decoder.Decode(&item) != nil || item.APIVersion != "meta.k8s.io/v1" || item.Kind != "PartialObjectMetadata" { return errors.New("retained pool response is not metadata") } - for key, expected := range map[string]string{"name": "ksail-analysis-runners", "namespace": "arc-ksail-analysis"} { + // The pinned chart names this object from runnerScaleSetName, not the + // HelmRelease name. recoveryArmed binds that same explicit source value. + for key, expected := range map[string]string{"name": retainedPoolName, "namespace": "arc-ksail-analysis"} { var actual string if json.Unmarshal(item.Metadata[key], &actual) != nil || actual != expected { return errors.New("retained pool identity does not match its declaration") diff --git a/scripts/guard-arc-recovery/main_test.go b/scripts/guard-arc-recovery/main_test.go index b3a04cfbba..8c7f8a5371 100644 --- a/scripts/guard-arc-recovery/main_test.go +++ b/scripts/guard-arc-recovery/main_test.go @@ -67,7 +67,7 @@ func TestRecoveryMetadataReadsRequireCompleteEmptyMetadataLists(t *testing.T) { } func TestRecoveryObservesRetainedPoolWithoutConfusingItWithActiveRunners(t *testing.T) { - const item = `{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadata","metadata":{"name":"ksail-analysis-runners","namespace":"arc-ksail-analysis","uid":"retained","resourceVersion":"123"}}` + const item = `{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadata","metadata":{"name":"ksail-code-quality","namespace":"arc-ksail-analysis","uid":"retained","resourceVersion":"123"}}` const prefix = `{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{"resourceVersion":"123"},"items":` for _, tc := range []struct { name, body string @@ -75,12 +75,13 @@ func TestRecoveryObservesRetainedPoolWithoutConfusingItWithActiveRunners(t *test }{ {"not yet installed", prefix + `null}`, true}, {"retained declaration", prefix + `[` + item + `]}`, true}, - {"unexpected pool", prefix + `[` + strings.Replace(item, "ksail-analysis-runners", "other", 1) + `]}`, false}, + {"Helm release name is not the pool name", prefix + `[` + strings.Replace(item, "ksail-code-quality", "ksail-analysis-runners", 1) + `]}`, false}, + {"unexpected pool", prefix + `[` + strings.Replace(item, "ksail-code-quality", "other", 1) + `]}`, false}, {"foreign namespace", prefix + `[` + strings.Replace(item, "arc-ksail-analysis", "arc-runners", 1) + `]}`, false}, {"missing UID", prefix + `[` + strings.Replace(item, `"uid":"retained"`, `"uid":""`, 1) + `]}`, false}, {"missing revision", prefix + `[` + strings.Replace(item, `"resourceVersion":"123"`, `"resourceVersion":""`, 1) + `]}`, false}, {"being deleted", prefix + `[` + strings.Replace(item, `"uid":"retained"`, `"uid":"retained","deletionTimestamp":"2026-10-06T00:00:00Z"`, 1) + `]}`, false}, - {"duplicate retained identity", prefix + `[` + strings.Replace(item, `"name":"ksail-analysis-runners"`, `"name":"foreign","name":"ksail-analysis-runners"`, 1) + `]}`, false}, + {"duplicate retained identity", prefix + `[` + strings.Replace(item, `"name":"ksail-code-quality"`, `"name":"foreign","name":"ksail-code-quality"`, 1) + `]}`, false}, {"aliased retained kind", prefix + `[` + strings.Replace(item, `"kind"`, `"Kind"`, 1) + `]}`, false}, {"multiple pools", prefix + `[` + item + `,` + item + `]}`, false}, {"full resource", prefix + `[` + strings.Replace(item, `"metadata":`, `"spec":{"maxRunners":0},"metadata":`, 1) + `]}`, false}, @@ -107,12 +108,14 @@ func TestRecoveryDeclarationRefusesSuspensionAndImplicitRunnerBounds(t *testing. name, controller, legacy string ok bool }{ - {"drained", "false", "suspend: false\n values: {minRunners: 0, maxRunners: 0}", true}, - {"suspended controller", "true", "suspend: false\n values: {minRunners: 0, maxRunners: 0}", false}, - {"suspended legacy", "false", "suspend: true\n values: {minRunners: 0, maxRunners: 0}", false}, - {"implicit maximum", "false", "suspend: false\n values: {minRunners: 0}", false}, - {"implicit minimum", "false", "suspend: false\n values: {maxRunners: 0}", false}, - {"admits one runner", "false", "suspend: false\n values: {minRunners: 0, maxRunners: 1}", false}, + {"drained", "false", "suspend: false\n values: {runnerScaleSetName: ksail-code-quality, minRunners: 0, maxRunners: 0}", true}, + {"suspended controller", "true", "suspend: false\n values: {runnerScaleSetName: ksail-code-quality, minRunners: 0, maxRunners: 0}", false}, + {"suspended legacy", "false", "suspend: true\n values: {runnerScaleSetName: ksail-code-quality, minRunners: 0, maxRunners: 0}", false}, + {"implicit maximum", "false", "suspend: false\n values: {runnerScaleSetName: ksail-code-quality, minRunners: 0}", false}, + {"implicit minimum", "false", "suspend: false\n values: {runnerScaleSetName: ksail-code-quality, maxRunners: 0}", false}, + {"admits one runner", "false", "suspend: false\n values: {runnerScaleSetName: ksail-code-quality, minRunners: 0, maxRunners: 1}", false}, + {"implicit pool name", "false", "suspend: false\n values: {minRunners: 0, maxRunners: 0}", false}, + {"different pool name", "false", "suspend: false\n values: {runnerScaleSetName: other, minRunners: 0, maxRunners: 0}", false}, } { t.Run(tc.name, func(t *testing.T) { root := t.TempDir() @@ -155,7 +158,7 @@ func TestNativeMetadataProxyPreservesAcceptAndRejectsWrites(t *testing.T) { } w.Header().Set("Content-Type", "application/json") if r.URL.Path == retainedPoolPath { - _, _ = w.Write([]byte(`{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{"resourceVersion":"123"},"items":[{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadata","metadata":{"name":"ksail-analysis-runners","namespace":"arc-ksail-analysis","uid":"retained","resourceVersion":"123"}}]}`)) + _, _ = w.Write([]byte(`{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{"resourceVersion":"123"},"items":[{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadata","metadata":{"name":"ksail-code-quality","namespace":"arc-ksail-analysis","uid":"retained","resourceVersion":"123"}}]}`)) return } _, _ = w.Write([]byte(`{"apiVersion":"meta.k8s.io/v1","kind":"PartialObjectMetadataList","metadata":{"resourceVersion":"123"},"items":null}`)) From cfe85aceeee3150c726b5a42cc4c1061d8b69e20 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 17:23:25 +0200 Subject: [PATCH 44/62] fix(ci): install native scanner before ARC listener regression --- .github/workflows/ci.yaml | 11 +++++-- scripts/tests/arc-staging/transport_test.go | 32 +++++++++++++++++++ .../tests/test-arc-listener-public-content.sh | 1 + 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7b142677f1..dffe1c6e81 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1678,8 +1678,6 @@ jobs: shellcheck scripts/validate-kyverno-fixture-evaluation.sh scripts/tests/test-validate-kyverno-fixture-evaluation.sh bash scripts/tests/test-validate-kyverno-fixture-evaluation.sh bash scripts/validate-kyverno-fixture-evaluation.sh tests - shellcheck scripts/tests/test-arc-listener-public-content.sh - bash scripts/tests/test-arc-listener-public-content.sh shellcheck scripts/tests/test-hcloud-volume-priority.sh bash scripts/tests/test-hcloud-volume-priority.sh shellcheck scripts/tests/test-audit-privileged-rbac.sh @@ -2423,6 +2421,15 @@ jobs: run: | .github/scripts/setup-ksail.sh + - name: 🔐 Validate ARC listener public-content disposition + if: needs.changes.outputs.k8s == 'true' + # Use the pinned native scanner installed above. The regression must + # prove both admission enforcement and the exact scanner disposition, + # including the unexcepted mirror and genuine-credential controls. + run: | + shellcheck scripts/tests/test-arc-listener-public-content.sh + bash scripts/tests/test-arc-listener-public-content.sh + - name: 📥 Restore kubeconform schema cache # ksail's kubeconform client fetches JSON schemas from # raw.githubusercontent.com/yannh/kubernetes-json-schema on every cold run diff --git a/scripts/tests/arc-staging/transport_test.go b/scripts/tests/arc-staging/transport_test.go index 5474ca41fe..0e92ffde72 100644 --- a/scripts/tests/arc-staging/transport_test.go +++ b/scripts/tests/arc-staging/transport_test.go @@ -17,6 +17,38 @@ const publicListenerHCL = `listener "tcp" { } ` +func TestPublicListenerNativeScanRunsAfterPinnedKSailSetup(t *testing.T) { + workflow := readYAML(t, ".github/workflows/ci.yaml") + steps := field(t, workflow, "jobs", "validate", "steps").([]any) + setup, scan, scanCount := -1, -1, 0 + for index, value := range steps { + step := value.(map[string]any) + run, _ := step["run"].(string) + if strings.TrimSpace(run) == ".github/scripts/setup-ksail.sh" { + if setup != -1 { + t.Fatal("manifest validation must install one pinned KSail release") + } + setup = index + version, ok := field(t, step, "env", "KSAIL_VERSION").(string) + if !ok || version == "" { + t.Fatal("the native scanner must use the job's pinned KSail release") + } + if _, conditional := step["if"]; conditional { + t.Fatal("the native scanner's KSail installation must be unconditional") + } + } + for _, line := range strings.Split(run, "\n") { + if strings.TrimSpace(line) == "bash scripts/tests/test-arc-listener-public-content.sh" { + scan, scanCount = index, scanCount+1 + equal(t, step["if"], "needs.changes.outputs.k8s == 'true'") + } + } + } + if setup < 0 || scanCount != 1 || scan <= setup { + t.Fatal("the mandatory public-listener native scan must run once after pinned KSail installation") + } +} + func TestListenerCredentialFalsePositiveRequiresPublicContentEnforcement(t *testing.T) { config := readYAML(t, transportPath+"config-map-listener.yaml") equal(t, field(t, config, "metadata", "name"), "arc-openbao-listener") diff --git a/scripts/tests/test-arc-listener-public-content.sh b/scripts/tests/test-arc-listener-public-content.sh index c02e9a8655..6105dea974 100755 --- a/scripts/tests/test-arc-listener-public-content.sh +++ b/scripts/tests/test-arc-listener-public-content.sh @@ -8,6 +8,7 @@ listener=k8s/providers/hetzner/infrastructure/controllers/openbao/transport/conf scratch=$(mktemp -d) trap 'rm -rf "$scratch"' EXIT fail() { printf 'ARC public listener: FAIL %s\n' "$1" >&2; exit 1; } +command -v ksail >/dev/null 2>&1 || fail missing-native-scanner yq -o=json '.' "$listener" >"$scratch/public.json" kyverno apply "$policy" --resource "$scratch/public.json" --remove-color >"$scratch/admission" 2>&1 || fail public-settings grep -Fq 'pass: 1, fail: 0, warn: 0, error: 0, skip: 0' "$scratch/admission" || fail unevaluated-positive From f1e0d24e6913369ae8522b2d99f0d9217bf923b6 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 18:20:24 +0200 Subject: [PATCH 45/62] fix(arc): validate activation against recovered drain state --- .../publication_test.go | 33 +++++++++++------ .../tests/test-verify-ksail-arc-runtime.sh | 36 +++++++++++++++++++ scripts/verify-ksail-arc-runtime.sh | 11 ++++-- 3 files changed, 67 insertions(+), 13 deletions(-) diff --git a/scripts/generate-ksail-arc-job-metrics/publication_test.go b/scripts/generate-ksail-arc-job-metrics/publication_test.go index 6fc7fabb96..b25e9cd68d 100644 --- a/scripts/generate-ksail-arc-job-metrics/publication_test.go +++ b/scripts/generate-ksail-arc-job-metrics/publication_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "io" + "io/fs" "os" "os/exec" "path/filepath" @@ -41,21 +42,33 @@ func TestMetricsHookSurvivesPublishedArtifactRender(t *testing.T) { // Render that artifact boundary so a checkout-only dependency cannot pass. component := filepath.Join(repoRoot, metricsComponent) published := t.TempDir() - files, err := os.ReadDir(component) - if err != nil { - t.Fatal(err) - } - for _, file := range files { - switch strings.ToLower(filepath.Ext(file.Name())) { + err := filepath.WalkDir(component, func(path string, file fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if file.IsDir() { + return nil + } + switch strings.ToLower(filepath.Ext(path)) { case ".yaml", ".yml", ".json": - data, err := os.ReadFile(filepath.Join(component, file.Name())) + data, err := os.ReadFile(path) if err != nil { - t.Fatal(err) + return err } - if err := os.WriteFile(filepath.Join(published, file.Name()), data, 0600); err != nil { - t.Fatal(err) + relative, err := filepath.Rel(component, path) + if err != nil { + return err + } + destination := filepath.Join(published, relative) + if err := os.MkdirAll(filepath.Dir(destination), 0700); err != nil { + return err } + return os.WriteFile(destination, data, 0600) } + return nil + }) + if err != nil { + t.Fatal(err) } rendered, err := exec.Command("kubectl", "kustomize", published).CombinedOutput() if err != nil { diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh index e79122d214..1fa66cd763 100755 --- a/scripts/tests/test-verify-ksail-arc-runtime.sh +++ b/scripts/tests/test-verify-ksail-arc-runtime.sh @@ -16,6 +16,8 @@ cp "$root/k8s/bases/infrastructure/actions-runners/helm-release.yaml" \ cp "$root/scripts/ksail-arc-job-metrics.sh" "$scratch/scripts/" cp "$root/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml" \ "$scratch/k8s/bases/infrastructure/controllers/actions-runner-controller/" +cp "$root/k8s/providers/hetzner/infrastructure/retained-ksail-analysis/helm-release.yaml" \ + "$scratch/retained-release.yaml" yq -i '.spec.suspend=false' "$scratch/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml" go build -o "$scratch/verifier" "$root/scripts/verify-ksail-arc-runtime" export ARC_TEST_ROOT="$scratch" @@ -94,9 +96,27 @@ if [[ "${1:-}" == kustomize ]]; then *) exit 0 ;; esac references=$(yq -o=json '(.resources // []) + (.bases // [])' "$aggregate") + if [[ "$2" == k8s/providers/hetzner/infrastructure ]]; then + case "${ARC_TEST_CASE:-}" in + rendered-retained-drain|rendered-drained-controller|rendered-drained-controller-wrong-scope|retained-active-min|retained-active-max|retained-suspended|retained-missing-bound|retained-wrong-scale-set) + printf '\n---\n' + case "$ARC_TEST_CASE" in + retained-active-min) yq '.spec.values.minRunners=1' "$ARC_TEST_ROOT/retained-release.yaml" ;; + retained-active-max) yq '.spec.values.maxRunners=1' "$ARC_TEST_ROOT/retained-release.yaml" ;; + retained-suspended) yq '.spec.suspend=true' "$ARC_TEST_ROOT/retained-release.yaml" ;; + retained-missing-bound) yq 'del(.spec.values.maxRunners)' "$ARC_TEST_ROOT/retained-release.yaml" ;; + retained-wrong-scale-set) yq '.spec.values.runnerScaleSetName="unapproved"' "$ARC_TEST_ROOT/retained-release.yaml" ;; + *) cat "$ARC_TEST_ROOT/retained-release.yaml" ;; + esac + printf '\n---\n' + ;; + esac + fi if jq -e --arg component "$reference" 'any(.[]; contains($component))' <<<"$references" >/dev/null; then case "${ARC_TEST_CASE:-}" in rendered-suspended) yq '.spec.suspend=true' "$release" ;; + rendered-drained-controller) yq '.metadata.annotations."platform.devantler.tech/arc-recovery"="drain-only"' "$release" ;; + rendered-drained-controller-wrong-scope) yq '.metadata.annotations."platform.devantler.tech/arc-recovery"="drain-only" | .spec.values.flags.watchSingleNamespace="unapproved"' "$release" ;; rendered-wrong-image) yq '.spec.values.template.spec.containers[0].image="unverified:latest"' "$release" ;; *) cat "$release" ;; esac @@ -306,6 +326,16 @@ printf 'resources: ["../../../../bases/infrastructure/controllers/actions-runner ) >"$scratch/retained-controller" grep -Fq 'inactive source; no runtime access' "$scratch/retained-controller" [[ ! -e "$scratch/runtime-access" ]] +( + cd "$scratch" + PATH="$scratch/bin:$PATH" ARC_TEST_CASE=rendered-drained-controller bash scripts/verify-ksail-arc-runtime.sh --if-active +) >"$scratch/drained-controller" +grep -Fq 'inactive source; no runtime access' "$scratch/drained-controller" +[[ ! -e "$scratch/runtime-access" ]] +if (cd "$scratch"; PATH="$scratch/bin:$PATH" ARC_TEST_CASE=rendered-drained-controller-wrong-scope bash scripts/verify-ksail-arc-runtime.sh --if-active) \ + >"$scratch/wrong-scope-out" 2>"$scratch/wrong-scope-error"; then exit 1; fi +grep -Fq 'FAIL at partial-activation' "$scratch/wrong-scope-error" +[[ ! -e "$scratch/runtime-access" ]] if (cd "$scratch"; PATH="$scratch/bin:$PATH" bash scripts/verify-ksail-arc-runtime.sh --if-active) \ >"$scratch/active-controller-out" 2>"$scratch/active-controller-error"; then exit 1; fi grep -Fq 'FAIL at partial-activation' "$scratch/active-controller-error" @@ -315,6 +345,12 @@ printf 'resources: ["../../../bases/infrastructure/actions-runners/"]\n' \ if (cd "$scratch"; PATH="$scratch/bin:$PATH" GITHUB_ACTIONS=false bash scripts/verify-ksail-arc-runtime.sh --if-active) \ >"$scratch/unauthorized-out" 2>"$scratch/unauthorized-error"; then exit 1; fi grep -Fq 'FAIL at deployment-identity' "$scratch/unauthorized-error" +run_case rendered-retained-drain pass +for name in retained-active-min retained-active-max retained-suspended retained-missing-bound retained-wrong-scale-set; do + run_case "$name" fail + grep -Fq 'FAIL at retained-source-state' "$scratch/stderr" + [[ ! -e "$scratch/runtime-access" && ! -e "$scratch/live-pod" ]] +done run_case complete-proof pass run_case delayed-flux pass [[ $(cat "$scratch/flux-reads") -gt 2 && -e "$scratch/registration-budget" ]] diff --git a/scripts/verify-ksail-arc-runtime.sh b/scripts/verify-ksail-arc-runtime.sh index f4a34fa8c5..945e4f9789 100755 --- a/scripts/verify-ksail-arc-runtime.sh +++ b/scripts/verify-ksail-arc-runtime.sh @@ -22,20 +22,25 @@ done yq ea -o=json -I=0 '[select(.kind == "HelmRelease" and ((.metadata.namespace == "arc-systems" and .metadata.name == "arc-controller") or (.metadata.namespace == "arc-runners" and .metadata.name == "platform-runners") or - (.metadata.namespace == "arc-ksail-analysis" and .metadata.name == "ksail-analysis-runners")))]' \ + .metadata.namespace == "arc-ksail-analysis"))]' \ "$source_scratch/rendered.yaml" >"$source_scratch/releases.json" rendered_controller=$(jq '[.[] | select(.metadata.namespace == "arc-systems")] | length' "$source_scratch/releases.json") rendered_pool=$(jq '[.[] | select(.metadata.namespace == "arc-runners")] | length' "$source_scratch/releases.json") rendered_legacy=$(jq '[.[] | select(.metadata.namespace == "arc-ksail-analysis")] | length' "$source_scratch/releases.json") [[ "$rendered_legacy" -le 1 ]] || fail retained-source-state -jq -e 'all(.[] | select(.metadata.namespace == "arc-ksail-analysis"); .spec.suspend == true)' \ +jq -e 'all(.[] | select(.metadata.namespace == "arc-ksail-analysis"); + .metadata.name == "ksail-analysis-runners" and .spec.suspend == false and + .spec.values.runnerScaleSetName == "ksail-code-quality" and + .spec.values.minRunners == 0 and .spec.values.maxRunners == 0)' \ "$source_scratch/releases.json" >/dev/null || fail retained-source-state if [[ "$controller_active" == 0 && "$pool_active" == 0 && "$rendered_controller" == 0 && "$rendered_pool" == 0 ]]; then printf 'ARC acceptance: inactive source; no runtime access\n' exit 0 fi if [[ "$controller_active" == 1 && "$pool_active" == 0 && "$rendered_controller" == 1 && "$rendered_pool" == 0 ]] && - jq -e 'all(.[]; .spec.suspend == true)' "$source_scratch/releases.json" >/dev/null; then + jq -e 'all(.[] | select(.metadata.namespace == "arc-systems"); .spec.suspend == true or + (.spec.suspend == false and .metadata.annotations["platform.devantler.tech/arc-recovery"] == "drain-only" and + .spec.values.flags.watchSingleNamespace == "arc-runners"))' "$source_scratch/releases.json" >/dev/null; then printf 'ARC acceptance: inactive source; no runtime access\n' exit 0 fi From b2c95dde507ff67185a6dc811f7c308e6869837a Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 18:34:59 +0200 Subject: [PATCH 46/62] fix(arc): compare native defaulted listener policy without relaxing rules --- scripts/tests/test-verify-arc-transport.sh | 9 +++++++-- scripts/verify-arc-transport.sh | 5 ++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/tests/test-verify-arc-transport.sh b/scripts/tests/test-verify-arc-transport.sh index 3c38067cab..92a7adf09c 100755 --- a/scripts/tests/test-verify-arc-transport.sh +++ b/scripts/tests/test-verify-arc-transport.sh @@ -12,7 +12,8 @@ export GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform GITHUB_REF= yq -o=json '.spec.rules[0]' k8s/providers/hetzner/infrastructure/cluster-policies/restrict-arc-openbao-certificate.yaml \ | jq '{spec:{rules:[.]},status:{conditions:[{type:"Ready",status:"True"}]}}' >"$scratch/policy.json" yq -o=json '.spec' k8s/bases/infrastructure/cluster-policies/best-practices/restrict-arc-openbao-listener.yaml \ - | jq '{metadata:{uid:"policy-uid"},spec:.,status:{conditions:[{type:"Ready",status:"True"}]}}' >"$scratch/listener-policy.json" + | jq '{metadata:{uid:"policy-uid"},spec:(. + {admission:true,emitWarning:false,validationFailureAction:"Audit"}), + status:{conditions:[{type:"Ready",status:"True"}]}}' >"$scratch/listener-policy.json" yq -o=json '.' k8s/providers/hetzner/infrastructure/controllers/openbao/transport/config-map-listener.yaml \ | jq '.metadata.uid="listener-uid" | .metadata.resourceVersion="123"' >"$scratch/listener.json" cat >"$scratch/bin/gh" <<'MOCK' @@ -57,6 +58,9 @@ case "$args" in else cat "$PROOF_TEST_ROOT/listener.json"; fi ;; 'get clusterpolicy restrict-arc-openbao-listener '*) if [[ "${PROOF_CASE:-}" == listener_audit ]]; then jq '.spec.rules[0].validate.failureAction="Audit"' "$PROOF_TEST_ROOT/listener-policy.json" + elif [[ "${PROOF_CASE:-}" == listener_admission_disabled ]]; then jq '.spec.admission=false' "$PROOF_TEST_ROOT/listener-policy.json" + elif [[ "${PROOF_CASE:-}" == listener_failure_open ]]; then jq '.spec.failurePolicy="Ignore"' "$PROOF_TEST_ROOT/listener-policy.json" + elif [[ "${PROOF_CASE:-}" == listener_override ]]; then jq '.spec.rules[0].validate.failureActionOverrides=[{action:"Audit",namespaces:["openbao"]}]' "$PROOF_TEST_ROOT/listener-policy.json" else cat "$PROOF_TEST_ROOT/listener-policy.json"; fi ;; 'get clusterpolicy '*) cat "$PROOF_TEST_ROOT/policy.json" ;; 'get kustomization '*) printf '{"metadata":{"generation":1},"status":{"observedGeneration":1,"lastAppliedRevision":"latest@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","conditions":[{"type":"Ready","status":"True"}]}}' ;; @@ -132,7 +136,8 @@ GITHUB_REF=refs/heads/feature run_case unreviewed_ref 1 run_case healthy_denied 0 run_case tls_error 60 for name in sealed stale_probe admit_privilege api_failure wrong_node reachable cert_error wrong_reason lost_events \ - listener_content listener_churn listener_audit listener_admit listener_api_error; do run_case "$name" 1; done + listener_content listener_churn listener_audit listener_admission_disabled listener_failure_open listener_override \ + listener_admit listener_api_error; do run_case "$name" 1; done run_case replacement 4 run_case cleanup_failure 4 printf 'PASS: %s transport orchestration cases\n' "$cases" diff --git a/scripts/verify-arc-transport.sh b/scripts/verify-arc-transport.sh index bbebb1b67b..88211a76fd 100755 --- a/scripts/verify-arc-transport.sh +++ b/scripts/verify-arc-transport.sh @@ -53,7 +53,10 @@ go build -trimpath -o "$scratch/evidence" ./scripts/verify-arc-transport >"$scra public_listener_boundary() { kc get clusterpolicy restrict-arc-openbao-listener -o json >"$scratch/listener-policy.json" - yq -o=json '.spec' k8s/bases/infrastructure/cluster-policies/best-practices/restrict-arc-openbao-listener.yaml >"$scratch/listener-spec.json" + # Include only defaults observed from this production API. Source fields + # take precedence; unknown fields and any rule or admission drift still fail. + yq -o=json '.spec' k8s/bases/infrastructure/cluster-policies/best-practices/restrict-arc-openbao-listener.yaml \ + | jq '{admission:true,emitWarning:false,validationFailureAction:"Audit"} + .' >"$scratch/listener-spec.json" jq -e --slurpfile expected "$scratch/listener-spec.json" '.spec == $expected[0] and any(.status.conditions[]; .type == "Ready" and .status == "True")' "$scratch/listener-policy.json" >/dev/null kc -n openbao get configmap arc-openbao-listener -o json >"$scratch/listener.json" From b8c647c322e524c38a39917f13a94ab8d135b432 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 18:52:35 +0200 Subject: [PATCH 47/62] fix(arc): bind complete native issuance policy before transport proof --- scripts/tests/test-verify-arc-transport.sh | 14 ++++++++++---- scripts/verify-arc-transport.sh | 7 +++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/scripts/tests/test-verify-arc-transport.sh b/scripts/tests/test-verify-arc-transport.sh index 92a7adf09c..b0ba0c6152 100755 --- a/scripts/tests/test-verify-arc-transport.sh +++ b/scripts/tests/test-verify-arc-transport.sh @@ -9,8 +9,9 @@ go build -o "$scratch/evidence" ./scripts/verify-arc-transport export PROOF_TEST_ROOT="$scratch" readonly head=1111111111111111111111111111111111111111 export GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform GITHUB_REF=refs/heads/main GITHUB_SHA="$head" GITHUB_RUN_ID=12345 GITHUB_RUN_ATTEMPT=1 -yq -o=json '.spec.rules[0]' k8s/providers/hetzner/infrastructure/cluster-policies/restrict-arc-openbao-certificate.yaml \ - | jq '{spec:{rules:[.]},status:{conditions:[{type:"Ready",status:"True"}]}}' >"$scratch/policy.json" +yq -o=json '.spec' k8s/providers/hetzner/infrastructure/cluster-policies/restrict-arc-openbao-certificate.yaml \ + | jq '{spec:(. + {admission:true,emitWarning:false,validationFailureAction:"Audit"}), + status:{conditions:[{type:"Ready",status:"True"}]}}' >"$scratch/policy.json" yq -o=json '.spec' k8s/bases/infrastructure/cluster-policies/best-practices/restrict-arc-openbao-listener.yaml \ | jq '{metadata:{uid:"policy-uid"},spec:(. + {admission:true,emitWarning:false,validationFailureAction:"Audit"}), status:{conditions:[{type:"Ready",status:"True"}]}}' >"$scratch/listener-policy.json" @@ -62,7 +63,12 @@ case "$args" in elif [[ "${PROOF_CASE:-}" == listener_failure_open ]]; then jq '.spec.failurePolicy="Ignore"' "$PROOF_TEST_ROOT/listener-policy.json" elif [[ "${PROOF_CASE:-}" == listener_override ]]; then jq '.spec.rules[0].validate.failureActionOverrides=[{action:"Audit",namespaces:["openbao"]}]' "$PROOF_TEST_ROOT/listener-policy.json" else cat "$PROOF_TEST_ROOT/listener-policy.json"; fi ;; - 'get clusterpolicy '*) cat "$PROOF_TEST_ROOT/policy.json" ;; + 'get clusterpolicy '*) + if [[ "${PROOF_CASE:-}" == issuance_admission_disabled ]]; then jq '.spec.admission=false' "$PROOF_TEST_ROOT/policy.json" + elif [[ "${PROOF_CASE:-}" == issuance_exclusion ]]; then jq '.spec.rules[0].exclude={any:[{resources:{namespaces:["arc-runners"]}}]}' "$PROOF_TEST_ROOT/policy.json" + elif [[ "${PROOF_CASE:-}" == issuance_failure_open ]]; then jq '.spec.failurePolicy="Ignore"' "$PROOF_TEST_ROOT/policy.json" + elif [[ "${PROOF_CASE:-}" == issuance_override ]]; then jq '.spec.rules[0].validate.failureActionOverrides=[{action:"Audit",namespaces:["arc-runners"]}]' "$PROOF_TEST_ROOT/policy.json" + else cat "$PROOF_TEST_ROOT/policy.json"; fi ;; 'get kustomization '*) printf '{"metadata":{"generation":1},"status":{"observedGeneration":1,"lastAppliedRevision":"latest@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","conditions":[{"type":"Ready","status":"True"}]}}' ;; 'get service '*) printf '{"spec":{"selector":{"app.kubernetes.io/name":"openbao","app.kubernetes.io/instance":"openbao","statefulset.kubernetes.io/pod-name":"openbao-2"},"ports":[{"name":"arc-tls","port":8204,"targetPort":8204,"protocol":"TCP"}]}}' ;; 'get node '*) printf '{"metadata":{"uid":"node-uid","labels":{}},"status":{"conditions":[{"type":"Ready","status":"True"}]}}' ;; @@ -137,7 +143,7 @@ run_case healthy_denied 0 run_case tls_error 60 for name in sealed stale_probe admit_privilege api_failure wrong_node reachable cert_error wrong_reason lost_events \ listener_content listener_churn listener_audit listener_admission_disabled listener_failure_open listener_override \ - listener_admit listener_api_error; do run_case "$name" 1; done + listener_admit listener_api_error issuance_admission_disabled issuance_exclusion issuance_failure_open issuance_override; do run_case "$name" 1; done run_case replacement 4 run_case cleanup_failure 4 printf 'PASS: %s transport orchestration cases\n' "$cases" diff --git a/scripts/verify-arc-transport.sh b/scripts/verify-arc-transport.sh index 88211a76fd..32943293bd 100755 --- a/scripts/verify-arc-transport.sh +++ b/scripts/verify-arc-transport.sh @@ -95,10 +95,9 @@ jq -e --arg server "https://$host:8204" '.spec.provider.vault.server == $server kc -n "$namespace" get configmap arc-openbao-ca -o json >"$scratch/ca.json" jq -er '.data["ca.crt"] | select(startswith("-----BEGIN CERTIFICATE-----"))' "$scratch/ca.json" >"$scratch/ca.pem" kc get clusterpolicy restrict-arc-openbao-certificate -o json >"$scratch/policy.json" -yq -o=json '.spec.rules[0]' k8s/providers/hetzner/infrastructure/cluster-policies/restrict-arc-openbao-certificate.yaml >"$scratch/rule.json" -jq -e --slurpfile rule "$scratch/rule.json" '(.spec.rules | length) == 1 and - .spec.rules[0].match == $rule[0].match and .spec.rules[0].preconditions == $rule[0].preconditions and - .spec.rules[0].validate.deny == $rule[0].validate.deny and .spec.rules[0].validate.failureAction == "Enforce" and +yq -o=json '.spec' k8s/providers/hetzner/infrastructure/cluster-policies/restrict-arc-openbao-certificate.yaml \ + | jq '{admission:true,emitWarning:false,validationFailureAction:"Audit"} + .' >"$scratch/issuance-spec.json" +jq -e --slurpfile expected "$scratch/issuance-spec.json" '.spec == $expected[0] and any(.status.conditions[]; .type == "Ready" and .status == "True")' "$scratch/policy.json" >/dev/null for layer in infrastructure infrastructure-controllers; do kc -n flux-system get kustomization "$layer" -o json | jq -ce 'select(.status.observedGeneration == .metadata.generation and From f18f8a4a254a03367c59be27465a17b392fd43f7 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 6 Oct 2026 23:58:58 +0200 Subject: [PATCH 48/62] fix: match ARC TLS policies to retained Cilium identities --- docs/operations/arc-runners.md | 10 +++ ...ilium-network-policy-external-secrets.yaml | 2 +- .../transport/cilium-network-policy.yaml | 2 +- .../patches/enable-arc-transport.yaml | 3 + .../arc-staging/transport_identity_test.go | 86 +++++++++++++++++++ scripts/tests/test-verify-arc-transport.sh | 7 +- scripts/verify-arc-transport.sh | 10 +++ 7 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 scripts/tests/arc-staging/transport_identity_test.go diff --git a/docs/operations/arc-runners.md b/docs/operations/arc-runners.md index 0bfb8b2cc3..ef28fd5819 100644 --- a/docs/operations/arc-runners.md +++ b/docs/operations/arc-runners.md @@ -57,6 +57,16 @@ authenticated cluster connection to reach the leader. The existing API listener, Raft storage, audit configuration and unseal hook remain in the base configuration. An additional listener file is appended through the pinned chart. +The TLS-enabled server template declares the +`platform.devantler.tech/arc-transport=tls` label. Both Cilium allow selectors +require this label alongside the server's application and instance identities; +the credential Service retains its exact ordinal selector. Cilium excludes +Kubernetes-generated ordinal labels from security identities, so those labels +cannot select a network-policy endpoint. The protected challenge binds the +native CiliumEndpoint to the current Pod UID and checks that the transport label +is retained before creating probes. Held server replicas without this label +remain outside the TLS allow selectors. + That ConfigMap contains only public settings and paths into a mounted Secret. Kubescape's generic credential-text rule matches the mandatory `tls_key_file` setting, so its disposition covers only the named ConfigMap and that control. diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy-external-secrets.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy-external-secrets.yaml index cb30736be2..7f46d0d1d8 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy-external-secrets.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy-external-secrets.yaml @@ -13,7 +13,7 @@ spec: k8s:io.kubernetes.pod.namespace: openbao app.kubernetes.io/name: openbao app.kubernetes.io/instance: openbao - statefulset.kubernetes.io/pod-name: openbao-2 + platform.devantler.tech/arc-transport: tls toPorts: - ports: - port: "8204" diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy.yaml index 81bccebd2d..124c57ae92 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/cilium-network-policy.yaml @@ -8,7 +8,7 @@ spec: matchLabels: app.kubernetes.io/name: openbao app.kubernetes.io/instance: openbao - statefulset.kubernetes.io/pod-name: openbao-2 + platform.devantler.tech/arc-transport: tls ingress: - fromEndpoints: - matchLabels: diff --git a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/patches/enable-arc-transport.yaml b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/patches/enable-arc-transport.yaml index b08ada28b5..9316ebfe8b 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/patches/enable-arc-transport.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/openbao/transport/patches/enable-arc-transport.yaml @@ -30,6 +30,9 @@ spec: rollingUpdate: partition: 2 template: + metadata: + labels: + platform.devantler.tech/arc-transport: tls spec: automountServiceAccountToken: false values: diff --git a/scripts/tests/arc-staging/transport_identity_test.go b/scripts/tests/arc-staging/transport_identity_test.go new file mode 100644 index 0000000000..480416c2a2 --- /dev/null +++ b/scripts/tests/arc-staging/transport_identity_test.go @@ -0,0 +1,86 @@ +package arcstaging_test + +import ( + "testing" + + "gopkg.in/yaml.v3" +) + +// Cilium's default identity projection excludes generated ordinal labels. +// Keep the Kubernetes Service selector separate from this security identity. +func TestCredentialTLSSelectorsMatchCiliumSecurityIdentity(t *testing.T) { + patch := readYAML(t, transportPath+"patches/enable-arc-transport.yaml") + renderers := field(t, patch, "spec", "postRenderers").([]any) + patches := field(t, renderers[0], "kustomize", "patches").([]any) + var statefulSet map[string]any + if err := yaml.Unmarshal([]byte(field(t, patches[0], "patch").(string)), &statefulSet); err != nil { + t.Fatal(err) + } + identity := map[string]any{ + "app.kubernetes.io/name": "openbao", + "app.kubernetes.io/instance": "openbao", + } + template := field(t, statefulSet, "spec", "template").(map[string]any) + if metadata, ok := template["metadata"].(map[string]any); ok { + if labels, ok := metadata["labels"].(map[string]any); ok { + for key, value := range labels { + switch key { + case "statefulset.kubernetes.io/pod-name", "apps.kubernetes.io/pod-index", "controller-revision-hash": + // These Pod labels are absent from native Cilium identities. + default: + identity[key] = value + } + } + } + } + ingress := readYAML(t, transportPath+"cilium-network-policy.yaml") + egress := readYAML(t, transportPath+"cilium-network-policy-external-secrets.yaml") + equal(t, field(t, ingress, "metadata", "namespace"), "openbao") + equal(t, field(t, egress, "metadata", "namespace"), "external-secrets") + ingressSelector := field(t, ingress, "spec", "endpointSelector", "matchLabels").(map[string]any) + rules := field(t, egress, "spec", "egress").([]any) + if len(rules) != 1 { + t.Fatal("TLS egress must retain one bounded rule") + } + destinations := field(t, rules[0], "toEndpoints").([]any) + if len(destinations) != 1 { + t.Fatal("TLS egress must retain one bounded destination") + } + egressSelector := field(t, destinations[0], "matchLabels").(map[string]any) + identity["k8s:io.kubernetes.pod.namespace"] = "openbao" + for name, selector := range map[string]map[string]any{"ingress": ingressSelector, "egress": egressSelector} { + t.Run(name, func(t *testing.T) { + if !transportIdentityMatches(selector, identity) { + t.Fatal("TLS allow rule cannot match the declared Cilium security identity") + } + for _, key := range []string{"app.kubernetes.io/name", "app.kubernetes.io/instance", "platform.devantler.tech/arc-transport"} { + without := make(map[string]any, len(identity)) + for label, value := range identity { + if label != key { + without[label] = value + } + } + if transportIdentityMatches(selector, without) { + t.Fatalf("TLS allow rule also matches a workload missing %s", key) + } + } + }) + } + otherNamespace := make(map[string]any, len(identity)) + for key, value := range identity { + otherNamespace[key] = value + } + otherNamespace["k8s:io.kubernetes.pod.namespace"] = "arc-runners" + if transportIdentityMatches(egressSelector, otherNamespace) { + t.Fatal("TLS egress must not select another namespace") + } +} + +func transportIdentityMatches(selector, identity map[string]any) bool { + for key, value := range selector { + if identity[key] != value { + return false + } + } + return len(selector) > 0 +} diff --git a/scripts/tests/test-verify-arc-transport.sh b/scripts/tests/test-verify-arc-transport.sh index b0ba0c6152..068f7b4bfb 100755 --- a/scripts/tests/test-verify-arc-transport.sh +++ b/scripts/tests/test-verify-arc-transport.sh @@ -79,6 +79,10 @@ case "$args" in 'get pods '*external-secrets*) printf '{"items":[{"metadata":{"uid":"eso-uid"},"spec":{"nodeName":"node-a"},"status":{"podIP":"10.1.1.3","conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"eso","ready":true,"restartCount":0,"containerID":"containerd://eso"}]}}]}' ;; 'get pods '*k8s-app=cilium*) printf '{"items":[{"metadata":{"name":"cilium-a","uid":"agent-uid"},"spec":{"nodeName":"node-a"},"status":{"podIP":"10.1.1.4","conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"cilium-agent","ready":true,"restartCount":0,"containerID":"containerd://agent"}]}}]}' ;; 'get pod openbao-2 '*) printf '{"metadata":{"uid":"bao-uid"},"spec":{"nodeName":"node-a","containers":[{"name":"openbao"},{"name":"arc-tls-reload"}]},"status":{"podIP":"10.1.1.2","conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"openbao","ready":true,"restartCount":0,"containerID":"containerd://bao"},{"name":"arc-tls-reload","ready":true,"restartCount":0,"containerID":"containerd://reload"}]}}' ;; + 'get ciliumendpoint openbao-2 '*) + owner=bao-uid; [[ "${PROOF_CASE:-}" != identity_stale ]] || owner=replaced-bao-uid + jq -n --arg owner "$owner" '{metadata:{ownerReferences:[{kind:"Pod",uid:$owner}]},status:{identity:{id:123,labels:["k8s:app.kubernetes.io/name=openbao","k8s:app.kubernetes.io/instance=openbao","k8s:io.kubernetes.pod.namespace=openbao","k8s:platform.devantler.tech/arc-transport=tls"]}}}' | + if [[ "${PROOF_CASE:-}" == identity_ordinal_only ]]; then jq '.status.identity.labels |= map(select(. != "k8s:platform.devantler.tech/arc-transport=tls"))'; else cat; fi ;; 'get pod cilium-a '*) printf '{"metadata":{"uid":"agent-uid"},"spec":{"nodeName":"node-a"},"status":{"podIP":"10.1.1.4","conditions":[{"type":"Ready","status":"True"}],"containerStatuses":[{"name":"cilium-agent","ready":true,"restartCount":0,"containerID":"containerd://agent"}]}}' ;; 'get pod arc-transport-'*) [[ -e "$state/created" ]] || exit 0 @@ -143,7 +147,8 @@ run_case healthy_denied 0 run_case tls_error 60 for name in sealed stale_probe admit_privilege api_failure wrong_node reachable cert_error wrong_reason lost_events \ listener_content listener_churn listener_audit listener_admission_disabled listener_failure_open listener_override \ - listener_admit listener_api_error issuance_admission_disabled issuance_exclusion issuance_failure_open issuance_override; do run_case "$name" 1; done + listener_admit listener_api_error issuance_admission_disabled issuance_exclusion issuance_failure_open issuance_override \ + identity_stale identity_ordinal_only; do run_case "$name" 1; done run_case replacement 4 run_case cleanup_failure 4 printf 'PASS: %s transport orchestration cases\n' "$cases" diff --git a/scripts/verify-arc-transport.sh b/scripts/verify-arc-transport.sh index 32943293bd..43775e8413 100755 --- a/scripts/verify-arc-transport.sh +++ b/scripts/verify-arc-transport.sh @@ -113,6 +113,16 @@ jq -e '.metadata.uid != null and .status.podIP != null and .spec.nodeName != nul any(.spec.containers[]; .name == "arc-tls-reload")' "$scratch/bao.json" >/dev/null bao_ip=$(jq -er '.status.podIP' "$scratch/bao.json") bao_node=$(jq -er '.spec.nodeName' "$scratch/bao.json") +# Pod labels are not necessarily Cilium identity labels. Bind the retained +# transport selector to the current Pod before claiming policy isolation. +kc -n openbao get ciliumendpoint openbao-2 -o json >"$scratch/bao-identity.json" +jq -e --arg uid "$(jq -er '.metadata.uid' "$scratch/bao.json")" ' + any(.metadata.ownerReferences[]; .kind == "Pod" and .uid == $uid) and + (.status.identity.id | type == "number" and . > 0) and + (.status.identity.labels as $labels | + all(["k8s:app.kubernetes.io/name=openbao", "k8s:app.kubernetes.io/instance=openbao", + "k8s:io.kubernetes.pod.namespace=openbao", "k8s:platform.devantler.tech/arc-transport=tls"][]; + . as $label | $labels | index($label) != null))' "$scratch/bao-identity.json" >/dev/null kc -n openbao get service openbao-arc -o json | jq -e '.spec.selector == {"app.kubernetes.io/name":"openbao","app.kubernetes.io/instance":"openbao","statefulset.kubernetes.io/pod-name":"openbao-2"} and .spec.ports == [{name:"arc-tls",port:8204,protocol:"TCP",targetPort:8204}]' >/dev/null kc -n external-secrets get pods -l app.kubernetes.io/name=external-secrets -o json >"$scratch/eso.json" From 297999ef61ea00586e4f2136d254e9a6a9a938ba Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 7 Oct 2026 00:12:42 +0200 Subject: [PATCH 49/62] fix: approve re-audited ARC transport identity label --- scripts/validate-eks-ci-role-policy/approved-surface.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index 708e888405..7ff6504895 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -88,7 +88,7 @@ helm.toolkit.fluxcd.io/v2|HelmRelease|observability|coroot-operator a6c5d6dfdec7 helm.toolkit.fluxcd.io/v2|HelmRelease|open-feature-operator-system|open-feature-operator da38c30c19aea0c3e099bb86cda46d89ab6301ff5898b4fad8f020812f8ad800 -helm.toolkit.fluxcd.io/v2|HelmRelease|openbao|openbao a3e829429965ed7f8c430424564624d176c1ae96be38a05db1bd38d018ebc1d4 +helm.toolkit.fluxcd.io/v2|HelmRelease|openbao|openbao 5f42ca30331162fc4ee98a0487fa3651140f0542ff49b573a99a46a343582616 helm.toolkit.fluxcd.io/v2|HelmRelease|policy-reporter|policy-reporter f92d67d39b5638fa931d19f4015ee490d6c98043e00a01319f4ef1a5fc64bc29 From 7206c395853569ae7b7e616cbf5238449a1063ea Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 7 Oct 2026 00:30:34 +0200 Subject: [PATCH 50/62] test: validate the exact ARC TLS canary identity change --- scripts/tests/test-openbao-standby-canary.sh | 45 +++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/scripts/tests/test-openbao-standby-canary.sh b/scripts/tests/test-openbao-standby-canary.sh index 561f07bfea..1ab71ee272 100755 --- a/scripts/tests/test-openbao-standby-canary.sh +++ b/scripts/tests/test-openbao-standby-canary.sh @@ -12,6 +12,22 @@ fail() { exit 1 } +checked_postrenderer_changes() { + jq -en --slurpfile before "$1" --slurpfile after "$2" ' + ($before | length) == 1 and ($after | length) == 1 and + $after[0].spec.replicas == 3 and + $after[0].spec.updateStrategy == { + type: "RollingUpdate", rollingUpdate: {partition: 2} + } and + $after[0].spec.template.spec.automountServiceAccountToken == false and + $before[0].spec.template.metadata.labels["platform.devantler.tech/arc-transport"] == null and + $after[0].spec.template.metadata.labels["platform.devantler.tech/arc-transport"] == "tls" and + ($before[0].spec | del(.updateStrategy, .template.spec.automountServiceAccountToken)) == + ($after[0].spec | del(.updateStrategy, .template.spec.automountServiceAccountToken, + .template.metadata.labels["platform.devantler.tech/arc-transport"])) + ' >/dev/null +} + resolve_replica_placeholder() { OPENBAO_REPLICA_PLACEHOLDER="\${openbao_replicas:=1}" OPENBAO_CANARY_REPLICAS="$2" yq -i ' (.server.replicas | select(. == strenv(OPENBAO_REPLICA_PLACEHOLDER))) = env(OPENBAO_CANARY_REPLICAS) | @@ -105,11 +121,30 @@ yq -e '.spec.template.spec.containers[] | select(.name == "openbao") | yq -e '.spec.template.spec.automountServiceAccountToken == false' "${statefulset}" >/dev/null || fail 'the certificate reload helper must not receive an injected API token' -before="$(yq ea -o=json -I=0 'select(.kind == "StatefulSet" and .metadata.name == "openbao") | - .spec | del(.updateStrategy, .template.spec.automountServiceAccountToken)' "${scratch}/rendered.yaml" | jq -cS .)" -after="$(yq -o=json -I=0 '.spec | del(.updateStrategy, .template.spec.automountServiceAccountToken)' "${statefulset}" | jq -cS .)" -[[ "${before}" == "${after}" ]] || - fail 'the canary post-renderer must change only partition and the checked token-injection setting' +yq ea -o=json -I=0 'select(.kind == "StatefulSet" and .metadata.name == "openbao")' \ + "${scratch}/rendered.yaml" >"${scratch}/before.json" +yq -o=json -I=0 '.' "${statefulset}" >"${scratch}/after.json" +checked_postrenderer_changes "${scratch}/before.json" "${scratch}/after.json" || + fail 'the canary post-renderer must change only partition, token injection and the exact TLS identity label' + +# Exercise rejected mutations against the actual chart and Flux-rendered Pod. +# Allowing one retained identity label must not permit other metadata or spec edits. +for mutation in \ + 'del(.spec.template.metadata.labels["platform.devantler.tech/arc-transport"])' \ + '.spec.template.metadata.labels["platform.devantler.tech/arc-transport"] = "other"' \ + '.spec.template.metadata.labels["unreviewed-label"] = "extra"' \ + '.spec.template.metadata.annotations["unreviewed-annotation"] = "extra"' \ + '.spec.template.metadata.labels["app.kubernetes.io/name"] = "other"' \ + '.spec.template.spec.serviceAccountName = "other"' \ + '.spec.template.spec.automountServiceAccountToken = true' \ + '.spec.updateStrategy.rollingUpdate.partition = 0' \ + '.spec.updateStrategy.rollingUpdate.maxUnavailable = 1' \ + '.spec.replicas = 1'; do + jq "$mutation" "${scratch}/after.json" >"${scratch}/mutated.json" + if checked_postrenderer_changes "${scratch}/before.json" "${scratch}/mutated.json"; then + fail 'the canary post-renderer accepted an unreviewed identity, metadata or Pod-spec mutation' + fi +done # Validate the actual pinned chart and post-rendered security boundary. An # unsupported Helm value must not look like it removed the sidecar's token. From 298bdfba907acd8213e3b957dd5f5dce21938ea4 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 7 Oct 2026 01:43:45 +0200 Subject: [PATCH 51/62] fix: retain Flux handoff restart through operator reconciliation Give the existing atomic restart patch a dedicated field manager so Flux Operator does not adopt and remove its annotation. Model operator manager cleanup and preserve rejection of surviving old processes. --- scripts/refresh-flux-ghcr-auth.sh | 4 ++ .../fake_kubectl_test.go | 27 ++++++++- .../rollout_convergence_test.go | 60 +++++++++++++++++++ 3 files changed, 88 insertions(+), 3 deletions(-) diff --git a/scripts/refresh-flux-ghcr-auth.sh b/scripts/refresh-flux-ghcr-auth.sh index 593cd844bb..50773f1e46 100755 --- a/scripts/refresh-flux-ghcr-auth.sh +++ b/scripts/refresh-flux-ghcr-auth.sh @@ -6214,12 +6214,16 @@ restart_flux_kustomize_controller_for_handoff() { end) + [{op: "add", path: $restart_path, value: $restart_token}] ' >"${flux_controller_restart_patch_file}" + # Flux Operator adopts kubectl-prefixed managers, removing restart annotations + # absent from its desired template. Keep this handoff's field ownership separate + # so the operator can reconcile without reverting the process-retirement rollout. if ! kubectl \ --context "${KUBE_CONTEXT}" \ --namespace flux-system \ patch deployment.apps \ "${FLUX_KUSTOMIZE_CONTROLLER_DEPLOYMENT}" \ --type=json \ + --field-manager=platform-ghcr-auth-handoff \ --patch-file="${flux_controller_restart_patch_file}" \ -o json >"${flux_controller_deployment_state_file}" \ 2>"${flux_controller_result_file}"; then diff --git a/scripts/tests/refresh-flux-ghcr-auth/fake_kubectl_test.go b/scripts/tests/refresh-flux-ghcr-auth/fake_kubectl_test.go index 4e4c69e577..8ff5b3599f 100644 --- a/scripts/tests/refresh-flux-ghcr-auth/fake_kubectl_test.go +++ b/scripts/tests/refresh-flux-ghcr-auth/fake_kubectl_test.go @@ -859,7 +859,7 @@ func fakeFluxControllerDeploymentObject() map[string]any { annotations := map[string]any{ "prometheus.io/port": "8080", } - if restartCount > 0 { + if restartCount > 0 && !markerExists("flux-controller-restart-reverted") { annotations["kubectl.kubernetes.io/restartedAt"] = markerContent( "flux-controller-restart-token", ) @@ -932,6 +932,9 @@ func fakeKubectlPatchFluxControllerDeployment(args []string, namespace, patchFil return commandFailure(57, "admission webhook denied the restart from 10.0.0.9") } setMarkerContent("flux-controller-restart-token", restartToken) + setMarkerContent("flux-controller-restart-manager", defaultString( + flagValue(args, "--field-manager"), "kubectl-patch", + )) setMarkerContent("flux-controller-restart-count", strconv.Itoa(restartCount+1)) if os.Getenv("FAKE_LOG_FLUX_CONTROLLER_RESTART") == "true" { appendEnvFile("OPERATION_LOG", "flux-controller-restart:kustomize-controller\n") @@ -967,7 +970,25 @@ func fakeKubectlRolloutFluxController(args []string, namespace string) int { if os.Getenv("FAKE_FLUX_CONTROLLER_ROLLOUT_FAIL") == "true" { return commandFailure(56, "kustomize-controller rollout did not converge") } - setMarkerContent("flux-controller-rollout-count", strconv.Itoa(restartCount)) + restartReverted := false + if os.Getenv("FAKE_FLUX_OPERATOR_CLEANS_RESTART_MANAGERS") == "true" { + manager := markerContent("flux-controller-restart-manager") + // Flux Operator v0.50.0 removes kubectl-prefixed Update managers before + // applying its template. Explicit override managers are also removed. + // The restart annotation is absent from that template, so adopting its + // manager removes the restart while the original Ready Pod survives. + restartReverted = strings.HasPrefix(manager, "kubectl") || + manager == os.Getenv("FAKE_FLUX_OPERATOR_RESTART_OVERRIDE_MANAGER") + if restartReverted { + touchMarker("flux-controller-restart-reverted") + appendEnvFile("OPERATION_LOG", "flux-operator-restart-reverted:kustomize-controller\n") + } else { + appendEnvFile("OPERATION_LOG", "flux-operator-restart-preserved:kustomize-controller\n") + } + } + if !restartReverted { + setMarkerContent("flux-controller-rollout-count", strconv.Itoa(restartCount)) + } if os.Getenv("FAKE_FLUX_OPERATOR_RECONCILES_PARENT_ON_CONTROLLER_RESTART") == "true" { // Restarting a Flux controller changes a Deployment owned by FluxInstance. // The real Flux Operator observes that change and reconciles the generated @@ -986,7 +1007,7 @@ func fakeKubectlRolloutFluxController(args []string, namespace string) int { ) appendEnvFile("OPERATION_LOG", "flux-operator-parent-reconcile:flux-system\n") } - if os.Getenv("FAKE_LOG_FLUX_CONTROLLER_RESTART") == "true" { + if os.Getenv("FAKE_LOG_FLUX_CONTROLLER_RESTART") == "true" && !restartReverted { if os.Getenv("FAKE_FLUX_CONTROLLER_OLD_POD_TERMINATING") == "true" { appendEnvFile( "OPERATION_LOG", diff --git a/scripts/tests/refresh-flux-ghcr-auth/rollout_convergence_test.go b/scripts/tests/refresh-flux-ghcr-auth/rollout_convergence_test.go index 59a837f71e..f4f2828d31 100644 --- a/scripts/tests/refresh-flux-ghcr-auth/rollout_convergence_test.go +++ b/scripts/tests/refresh-flux-ghcr-auth/rollout_convergence_test.go @@ -1464,6 +1464,66 @@ func TestStaleFluxChildReconcilingConditionAfterPauseDoesNotBlockHandoff(t *test requireLine(t, operations, "flux-policy-resume:infrastructure") } +func TestFluxControllerRestartSurvivesOperatorManagerCleanup(t *testing.T) { + t.Parallel() + f := newFixture(t) + result := f.runHelper(validConfig(), nil, map[string]string{ + "FAKE_FLUX_OPERATOR_CLEANS_RESTART_MANAGERS": "true", + "FAKE_FLUX_OPERATOR_RECONCILES_PARENT_ON_CONTROLLER_RESTART": "true", + "FAKE_LOG_FLUX_CONTROLLER_RESTART": "true", + }) + requireSuccessResult(t, result) + operations := readLines(f.operationLog) + preserved := lineIndex(t, operations, "flux-operator-restart-preserved:kustomize-controller") + terminated := lineIndex(t, operations, "flux-controller-old-processes-terminated:kustomize-controller") + policy := lineIndex(t, operations, "ivpol-policy-apply:verify-app-images") + if preserved >= terminated || terminated >= policy { + t.Fatalf("policy mutation preceded preserved restart and process retirement: %v", operations) + } + requireNoLine(t, operations, "flux-operator-restart-reverted:kustomize-controller") +} + +func TestFluxControllerOperatorCleanupStillRejectsSurvivingProcesses(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + operation string + env map[string]string + }{ + { + name: "explicit operator override reverts the restart", + operation: "flux-operator-restart-reverted:kustomize-controller", + env: map[string]string{ + "FAKE_FLUX_OPERATOR_RESTART_OVERRIDE_MANAGER": "platform-ghcr-auth-handoff", + }, + }, + { + name: "preserved restart leaves an old terminating Pod", + operation: "flux-controller-rollout-complete-with-terminating-old-pod:kustomize-controller", + env: map[string]string{ + "FAKE_FLUX_CONTROLLER_OLD_POD_TERMINATING": "true", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + f := newFixture(t) + tc.env["FAKE_FLUX_OPERATOR_CLEANS_RESTART_MANAGERS"] = "true" + tc.env["FAKE_LOG_FLUX_CONTROLLER_RESTART"] = "true" + result := f.runHelper(validConfig(), nil, tc.env) + requireFailureResult(t, result) + requireContains(t, result.stdout+result.stderr, + "Could not prove every pre-handoff kustomize-controller process was replaced") + operations := readLines(f.operationLog) + requireLine(t, operations, tc.operation) + requireNoLine(t, operations, "ivpol-policy-apply:verify-app-images") + requireNoLine(t, operations, "root-patch") + requireLine(t, operations, "flux-policy-resume:infrastructure") + requireLine(t, operations, "flux-policy-parent-resume:flux-system") + }) + } +} + func TestFluxControllerRestartTerminatesPrePauseProcessesBeforePolicyStage(t *testing.T) { t.Parallel() f := newFixture(t) From 30b39ce1f5eb705687a1c2b39f6147b852c8da37 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 7 Oct 2026 00:50:21 +0200 Subject: [PATCH 52/62] fix(ci): separate fixture readiness from cancellation timing --- scripts/tests/test-parallel-guard-cleanup.sh | 55 ++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/scripts/tests/test-parallel-guard-cleanup.sh b/scripts/tests/test-parallel-guard-cleanup.sh index 256b3691fa..ae09eaf5d3 100755 --- a/scripts/tests/test-parallel-guard-cleanup.sh +++ b/scripts/tests/test-parallel-guard-cleanup.sh @@ -34,6 +34,23 @@ wait_for_file() { return 1 } +wait_for_witness_start() { + local file="$1" _ + # Preparation uses real fixture tools and has its own bounded readiness wait. + # Cancellation is measured only after the witness starts; its bounds below + # remain unchanged. A failed producer must never look like a ready witness. + for _ in $(seq 1 750); do + if ! kill -0 "$parent" 2>/dev/null; then + printf 'fixture producer exited before witness readiness\n' >&2 + return 1 + fi + [ ! -f "$file" ] || return 0 + sleep 0.02 + done + printf 'timed out waiting for fixture witness readiness: %s\n' "$file" >&2 + return 1 +} + wait_for_parent_exit() { local _ for _ in $(seq 1 150); do @@ -147,12 +164,24 @@ assert_stopped "$state" # Marker witnesses still wait synchronously, but must also be cancellable while # their guard blocks. Use the actual selected inline-document family. state="$scratch/interrupted-witness" -mkdir -p "$state" -PARALLEL_CLEANUP_STATE="$state" CONSUMER_CONSERVATION_REGRESSION=inline-documents \ +mkdir -p "$state" "$scratch/witness-bin" +# Fixture setup precedes the cancellation measurement. Exercise actual slow +# preparation without giving the interrupted worker more time to stop. +real_yq="$(command -v yq)" +cat >"$scratch/witness-bin/yq" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +state="${PARALLEL_CLEANUP_STATE:?}" +if mkdir "$state/setup-delayed" 2>/dev/null; then sleep 5; fi +exec "${PARALLEL_CLEANUP_REAL_YQ:?}" "$@" +EOF +chmod +x "$scratch/witness-bin/yq" +PATH="$scratch/witness-bin:$PATH" PARALLEL_CLEANUP_REAL_YQ="$real_yq" \ + PARALLEL_CLEANUP_STATE="$state" CONSUMER_CONSERVATION_REGRESSION=inline-documents \ bash "$scratch/fake-repo/scripts/tests/test-guard-consumer-discovery-conservation.sh" \ >"$state/output" 2>&1 & parent=$! -wait_for_file "$state/witness-started" +wait_for_witness_start "$state/witness-started" kill -TERM "$parent" wait_for_parent_exit rc=0 @@ -161,6 +190,26 @@ parent='' [ "$rc" -eq 143 ] || { printf 'witness cancellation lost its exit code: %s\n' "$rc" >&2; exit 1; } assert_stopped "$state" +# A producer that exits before readiness must fail, rather than consuming the +# setup deadline or allowing the cancellation assertions to inspect nothing. +state="$scratch/exited-before-witness" +mkdir -p "$state" +(exit 42) & +parent=$! +if wait_for_witness_start "$state/witness-started" >"$state/output" 2>&1; then + printf 'an exited fixture producer was accepted as ready\n' >&2 + exit 1 +fi +rc=0 +wait "$parent" || rc=$? +parent='' +[ "$rc" -eq 42 ] || { printf 'fixture producer lost its failing exit code\n' >&2; exit 1; } +grep -qF 'fixture producer exited before witness readiness' "$state/output" || { + printf 'fixture exit was mistaken for a readiness timeout\n' >&2 + cat "$state/output" >&2 + exit 1 +} + # Selected regression families must join queued cases before choosing a verdict. # A high admission bound ensures no scheduling wait can accidentally hide a # missing final drain. Every synthetic guard result deliberately fails its case. From 378adc5fa2dfbcb92f7ac976483217e7d5aef41e Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 7 Oct 2026 01:58:21 +0200 Subject: [PATCH 53/62] fix(arc): retire credential staging component during activation --- .../arc-credential-transport/kustomization.yaml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 k8s/providers/hetzner/infrastructure/arc-credential-transport/kustomization.yaml diff --git a/k8s/providers/hetzner/infrastructure/arc-credential-transport/kustomization.yaml b/k8s/providers/hetzner/infrastructure/arc-credential-transport/kustomization.yaml deleted file mode 100644 index 36eef1ddfc..0000000000 --- a/k8s/providers/hetzner/infrastructure/arc-credential-transport/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Authentication and CA validation can be exercised before reading the App key -# or activating ARC. The full pool replaces this staging reference on activation. -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - ../../../../bases/infrastructure/actions-runners/credentials/ From bfece8e28133b7f641fbb83c7b39ea9f6765c226 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 7 Oct 2026 02:09:03 +0200 Subject: [PATCH 54/62] fix(ci): retain one transport proof validation after integration --- .github/workflows/ci.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b254fba0ea..36b46e7cb0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -167,12 +167,6 @@ jobs: shellcheck scripts/verify-arc-transport.sh scripts/tests/test-verify-arc-transport.sh bash scripts/tests/test-verify-arc-transport.sh - - name: 🔐 Validate ARC transport proof boundaries - run: | - go test -race ./scripts/verify-arc-transport - shellcheck scripts/verify-arc-transport.sh scripts/tests/test-verify-arc-transport.sh - bash scripts/tests/test-verify-arc-transport.sh - - name: 🛑 Validate ARC recovery metadata proof run: go test -race ./scripts/guard-arc-recovery From fec4ccb4fa26bef61f85b12e39772059d3e3d006 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 7 Oct 2026 19:31:46 +0200 Subject: [PATCH 55/62] fix: report failing ARC runtime API stages privately --- scripts/tests/test-verify-ksail-arc-runtime.sh | 14 ++++++++++++++ scripts/verify-ksail-arc-runtime.sh | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh index 1fa66cd763..bddb5d4fc7 100755 --- a/scripts/tests/test-verify-ksail-arc-runtime.sh +++ b/scripts/tests/test-verify-ksail-arc-runtime.sh @@ -173,6 +173,10 @@ case "$args" in *) cat "$ARC_TEST_ROOT/ars" ;; esac ;; *'get configmap ksail-arc-job-metrics-'*) + if [[ "$ARC_TEST_CASE" == live-api-error ]]; then + printf 'private-api-error-canary\n' >&2 + exit 42 + fi jq -cn --rawfile script "$ARC_TEST_ROOT/scripts/ksail-arc-job-metrics.sh" \ --arg scenario "$ARC_TEST_CASE" '{metadata:{uid:"metrics-uid",annotations:{"kustomize.toolkit.fluxcd.io/substitute":"disabled"}},immutable:($scenario!="mutable-metrics"), data:{"job-metrics.sh":(if $scenario=="tampered-metrics" then "unverified" else $script end)}}' ;; @@ -352,6 +356,16 @@ for name in retained-active-min retained-active-max retained-suspended retained- [[ ! -e "$scratch/runtime-access" && ! -e "$scratch/live-pod" ]] done run_case complete-proof pass +run_case live-api-error fail +grep -Fq 'ARC acceptance: FAIL at immutable-job-metrics' "$scratch/stderr" || { + printf 'FAIL: live API error did not identify the failing acceptance stage\n' >&2 + exit 1 +} +grep -Fq 'ARC acceptance: cleanup verified' "$scratch/stdout" +if grep -Fq 'private-api-error-canary' "$scratch/stdout" "$scratch/stderr"; then + printf 'FAIL: private API diagnostics reached acceptance output\n' >&2 + exit 1 +fi run_case delayed-flux pass [[ $(cat "$scratch/flux-reads") -gt 2 && -e "$scratch/registration-budget" ]] run_case delayed-registration pass diff --git a/scripts/verify-ksail-arc-runtime.sh b/scripts/verify-ksail-arc-runtime.sh index 945e4f9789..139f382aca 100755 --- a/scripts/verify-ksail-arc-runtime.sh +++ b/scripts/verify-ksail-arc-runtime.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # CI-owned, bounded runtime acceptance. No Secret or generated JIT configuration is read. -set -euo pipefail +set -Eeuo pipefail umask 077 fail() { printf 'ARC acceptance: FAIL at %s\n' "$1" >&2; exit 1; } From 7fcb92f3e0804a0bf94dabc353ef11f222a1d486 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 7 Oct 2026 19:33:20 +0200 Subject: [PATCH 56/62] chore: audit the bounded ARC activation surface --- scripts/validate-eks-ci-role-policy/approved-surface.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index b0904362b5..00bfb6aa2c 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -16,7 +16,9 @@ helm.toolkit.fluxcd.io/v2|HelmRelease|actual-budget|actual-budget adc009b64f2a9d helm.toolkit.fluxcd.io/v2|HelmRelease|arc-ksail-analysis|ksail-analysis-runners f59980a0d056bb796135803ce4cfb1a84947957e943abe9854a78eaf1249b2b0 -helm.toolkit.fluxcd.io/v2|HelmRelease|arc-systems|arc-controller 5af946415fcf05e49fb79e9a07cd82cd1da6d3c8ee33a6ebbe0483fc6e00afa1 +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-runners|platform-runners c9d0cc2de5cdc8985887d78c76a8b7522e3874b5a5d6ee46e7c79a5fa97b410e + +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-systems|arc-controller 844e32e3408c2ad6e85b10c6628de7208c447ca3f1cffe5a0da74dcf3cdc11cf helm.toolkit.fluxcd.io/v2|HelmRelease|backstage|backstage 5ff19be19e3525e641b0a7a22e2a7047e1fbda16ef9ca7b4fa731ce3a61e686a @@ -406,6 +408,8 @@ source.toolkit.fluxcd.io/v1|HelmRepository|whoami|whoami 28dc88b5a5e69a086e4f7fb source.toolkit.fluxcd.io/v1|OCIRepository|arc-ksail-analysis|ksail-analysis-runners 5f89c75739fa50d998e046bc2a388ccf9c7fb4413377510e43f9f12ae4ff4c70 +source.toolkit.fluxcd.io/v1|OCIRepository|arc-runners|platform-runners 7224602dcdde3bce2c8bdf1c6077806e3b38ee13adfcbbf544418da4fc7e0913 + source.toolkit.fluxcd.io/v1|OCIRepository|arc-systems|arc-controller 64fb325741489bbd5ece40cf2cbf6e4aef3e87e89efea97b7b0f2ec498671fa1 source.toolkit.fluxcd.io/v1|OCIRepository|ascoachingogvaner|ascoachingogvaner e7717c1d618e5d2dd1f3ab32d589ac06205d040e867f33db22ab9651bcb105bc From 403536cf3685875be325c57dd52b2793a8d4ac3b Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Fri, 9 Oct 2026 06:26:36 +0200 Subject: [PATCH 57/62] fix: refresh ARC image after verified analysis publication --- k8s/bases/infrastructure/actions-runners/helm-release.yaml | 4 ++-- scripts/tests/arc-staging/runner_group_activation_test.go | 6 +++--- scripts/validate-eks-ci-role-policy/approved-surface.txt | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/bases/infrastructure/actions-runners/helm-release.yaml b/k8s/bases/infrastructure/actions-runners/helm-release.yaml index 85863a96dd..55e034e6c8 100644 --- a/k8s/bases/infrastructure/actions-runners/helm-release.yaml +++ b/k8s/bases/infrastructure/actions-runners/helm-release.yaml @@ -64,7 +64,7 @@ spec: path: job-metrics.sh initContainers: - name: init-runner-home - image: ghcr.io/devantler-tech/ksail-analysis-runner@sha256:1ab01644fd6f67e0b1ab0b456e10b78f58be92ac67e5e06d103cd8c12bbb119b + image: ghcr.io/devantler-tech/ksail-analysis-runner@sha256:f2fc508d5e426405bdc3732f652b251e093d93e7becdec34f46b1fe6f3309183 command: ["/bin/sh", "-ec", "cp -R /home/runner/. /runner-data/"] securityContext: allowPrivilegeEscalation: false @@ -86,7 +86,7 @@ spec: mountPath: /runner-data containers: - name: runner - image: ghcr.io/devantler-tech/ksail-analysis-runner@sha256:1ab01644fd6f67e0b1ab0b456e10b78f58be92ac67e5e06d103cd8c12bbb119b + image: ghcr.io/devantler-tech/ksail-analysis-runner@sha256:f2fc508d5e426405bdc3732f652b251e093d93e7becdec34f46b1fe6f3309183 command: ["/home/runner/run.sh"] env: - name: ACTIONS_RUNNER_HOOK_JOB_COMPLETED diff --git a/scripts/tests/arc-staging/runner_group_activation_test.go b/scripts/tests/arc-staging/runner_group_activation_test.go index c7ce0027b1..fad4c688d5 100644 --- a/scripts/tests/arc-staging/runner_group_activation_test.go +++ b/scripts/tests/arc-staging/runner_group_activation_test.go @@ -12,7 +12,7 @@ import ( const arcActivationFile = "managed-resource-activation-policy-arc.yaml" const arcRunnerGroupResource = "runnergroups.actions.github.m.upbound.io" -func validateARCActivation(policy map[string]any, resources []any) error { +func validateARCRunnerGroupActivation(policy map[string]any, resources []any) error { if policy["apiVersion"] != "apiextensions.crossplane.io/v1alpha1" || policy["kind"] != "ManagedResourceActivationPolicy" { return fmt.Errorf("expected a provider activation policy") } @@ -71,7 +71,7 @@ func TestARCResourceIsExplicitlyActivated(t *testing.T) { const component = "k8s/providers/hetzner/infrastructure/crossplane/" policy := readYAML(t, component+arcActivationFile) resources := field(t, readYAML(t, component+"kustomization.yaml"), "resources").([]any) - if err := validateARCActivation(policy, resources); err != nil { + if err := validateARCRunnerGroupActivation(policy, resources); err != nil { t.Fatal(err) } } @@ -95,7 +95,7 @@ func TestARCActivationRejectsBroaderResourcesAndMissingInclusion(t *testing.T) { "metadata": map[string]any{"name": "github-arc-runners"}, "spec": map[string]any{"activate": test.activation}, } - if err := validateARCActivation(policy, test.resources); err == nil { + if err := validateARCRunnerGroupActivation(policy, test.resources); err == nil { t.Fatal("accepted an unsafe or unreconciled activation policy") } }) diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index 8786d71d5a..06edf812cc 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -16,7 +16,7 @@ helm.toolkit.fluxcd.io/v2|HelmRelease|actual-budget|actual-budget adc009b64f2a9d helm.toolkit.fluxcd.io/v2|HelmRelease|arc-ksail-analysis|ksail-analysis-runners f59980a0d056bb796135803ce4cfb1a84947957e943abe9854a78eaf1249b2b0 -helm.toolkit.fluxcd.io/v2|HelmRelease|arc-runners|platform-runners c9d0cc2de5cdc8985887d78c76a8b7522e3874b5a5d6ee46e7c79a5fa97b410e +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-runners|platform-runners 0dd16e9d373fe4e3aa26d6b0fc3e31d4d977cf623acfc0e172b8c5b99033a95f helm.toolkit.fluxcd.io/v2|HelmRelease|arc-systems|arc-controller 844e32e3408c2ad6e85b10c6628de7208c447ca3f1cffe5a0da74dcf3cdc11cf From b7cf8d34776247e8a571cfe93e032596742498a8 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Fri, 9 Oct 2026 11:32:15 +0200 Subject: [PATCH 58/62] fix: serialize ARC canary with runner admission --- docs/operations/arc-runners.md | 18 +++ .../tests/test-verify-ksail-arc-runtime.sh | 80 ++++++++++- scripts/verify-ksail-arc-runtime.sh | 135 +++++++++++++++++- 3 files changed, 229 insertions(+), 4 deletions(-) diff --git a/docs/operations/arc-runners.md b/docs/operations/arc-runners.md index cabbb63033..2922f48a86 100644 --- a/docs/operations/arc-runners.md +++ b/docs/operations/arc-runners.md @@ -245,6 +245,24 @@ it exercises admission, allowed connectivity, intercepted denials and cleanup. Verify SecretStore and ExternalSecret readiness before admitting jobs. Keep the opt-in runner name out of workflows that have not completed onboarding. +The protected canary reserves admission with the fixed `arc-runtime-admission` +ResourceQuota in the runner namespace. Its zero-pod `NotTerminating` scope blocks +new ordinary runner Pods while existing jobs finish naturally. The deadline-bound +probe is outside that scope. The verifier checks retained runner templates, proves +the real runner template is denied by this quota, and waits up to twenty minutes +for existing runners and the dedicated node to drain. It leaves the quota in place +until probe deletion and node cleanup are verified, then deletes only the +invocation-owned quota with UID and resource-version preconditions. It changes no +runner bounds, Helm reconciliation, App permission or capacity ceiling. + +A pre-existing quota, ambiguous create, changed quota identity or incomplete probe +cleanup fails closed. An incomplete cleanup retains the fence for recovery. Before +removing a retained fence, verify its invocation ownership, prove that invocation's +probe is absent and the dedicated pool has drained, and use the current UID and +resource version as deletion preconditions. Never remove another invocation's +fence to make a deployment pass. Quota behavior and its deadline scopes are defined +in the [Kubernetes resource quota documentation](https://kubernetes.io/docs/concepts/policy/resource-quotas/). + To stop admitting jobs, restore every consumer's prior runner configuration and verify the readbacks, then use a reviewed values change to set both runner bounds to zero. Let the current job finish and prove there are no busy runners diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh index bddb5d4fc7..a84c77698d 100755 --- a/scripts/tests/test-verify-ksail-arc-runtime.sh +++ b/scripts/tests/test-verify-ksail-arc-runtime.sh @@ -33,7 +33,7 @@ image="$image" yq -i '.spec.values.template.spec.containers[0].image=strenv(imag .spec.values.template.spec.initContainers[0].image=strenv(image) | .spec.suspend=false' \ "$scratch/k8s/bases/infrastructure/actions-runners/helm-release.yaml" yq -o=json '.spec.values' "$scratch/k8s/bases/infrastructure/actions-runners/helm-release.yaml" \ - | jq '{metadata:{generation:1,annotations:{"runner-scale-set-id":"1", + | jq '{metadata:{uid:"ars-uid",generation:1,annotations:{"runner-scale-set-id":"1", "actions.github.com/runner-group-name":"platform", "actions.github.com/runner-scale-set-name":"platform-linux"}}, status:{phase:"Running",observedGeneration:1},spec:.} | @@ -170,8 +170,20 @@ case "$args" in unexpected-hook) jq '.spec.template.spec.containers[0].env[0].value="/unverified"' "$ARC_TEST_ROOT/ars" ;; writable-metrics) jq '.spec.template.spec.containers[0].volumeMounts[2].readOnly=false' "$ARC_TEST_ROOT/ars" ;; wrong-metrics-configmap) jq '.spec.template.spec.volumes[2].configMap.name="unverified"' "$ARC_TEST_ROOT/ars" ;; + runner-deadline) jq '.spec.template.spec.activeDeadlineSeconds=60' "$ARC_TEST_ROOT/ars" ;; + template-churn) + if [[ -e "$ARC_TEST_ROOT/admission-fence" ]]; then jq '.spec.template.spec.activeDeadlineSeconds=60' "$ARC_TEST_ROOT/ars" + else cat "$ARC_TEST_ROOT/ars"; fi ;; + invalid-runner-identity) jq 'del(.metadata.uid)' "$ARC_TEST_ROOT/ars" ;; + drift-during-probe) + if [[ -e "$ARC_TEST_ROOT/metrics-executed" ]]; then jq '.metadata.generation=2 | .spec.template.spec.activeDeadlineSeconds=60' "$ARC_TEST_ROOT/ars" + else cat "$ARC_TEST_ROOT/ars"; fi ;; *) cat "$ARC_TEST_ROOT/ars" ;; esac ;; + *'get ephemeralrunnersets.actions.github.com '*|*'get ephemeralrunners.actions.github.com '*) + printf 'continue=\n' + if [[ "$ARC_TEST_CASE" == retained-runner-deadline ]]; then printf 'uid=retained-uid;deadline=60\n' + else printf 'uid=retained-uid;deadline=\n'; fi ;; *'get configmap ksail-arc-job-metrics-'*) if [[ "$ARC_TEST_CASE" == live-api-error ]]; then printf 'private-api-error-canary\n' >&2 @@ -180,15 +192,28 @@ case "$args" in jq -cn --rawfile script "$ARC_TEST_ROOT/scripts/ksail-arc-job-metrics.sh" \ --arg scenario "$ARC_TEST_CASE" '{metadata:{uid:"metrics-uid",annotations:{"kustomize.toolkit.fluxcd.io/substitute":"disabled"}},immutable:($scenario!="mutable-metrics"), data:{"job-metrics.sh":(if $scenario=="tampered-metrics" then "unverified" else $script end)}}' ;; + *'get resourcequotas -l '*) + printf '{"items":[]}' ;; *'get resourcequotas '*) + { if [[ "$ARC_TEST_CASE" == full-quota ]]; then printf '{"items":[{"spec":{"hard":{"pods":"1"}},"status":{"hard":{"pods":"1"},"used":{"pods":"1"}}}]}' elif [[ "$ARC_TEST_CASE" == stale-quota ]]; then printf '{"items":[{"spec":{"hard":{"pods":"1"}},"status":{"hard":{"pods":"3"},"used":{"pods":"0"}}}]}' - else printf '{"items":[]}'; fi ;; + else printf '{"items":[]}'; fi + } | jq --slurpfile fence "$ARC_TEST_ROOT/admission-fence" '.items += $fence' ;; + *'get resourcequota arc-runtime-admission '*) + [[ -e "$ARC_TEST_ROOT/admission-fence" ]] || exit 0 + cat "$ARC_TEST_ROOT/admission-fence" ;; *'get pods -l platform.devantler.tech/arc-role=runner '*|*'get pods -l platform.devantler.tech/arc-runtime-probe'*) - printf '{"items":[]}' ;; + if [[ "$args" == *'arc-role=runner'* && -e "$ARC_TEST_ROOT/live-pod" ]]; then + jq '{items:[.]}' "$ARC_TEST_ROOT/live-pod" + elif [[ "$args" == *'arc-role=runner'* && "$ARC_TEST_CASE" == busy-runner && ! -e "$ARC_TEST_ROOT/job-completed" ]]; then + touch "$ARC_TEST_ROOT/job-completed" + printf '{"items":[{"metadata":{"name":"legitimate-runner","uid":"job-uid"},"spec":{},"status":{"phase":"Running"}}]}' + else printf '{"items":[]}'; fi ;; 'get nodes -o json') + if [[ -e "$ARC_TEST_ROOT/deleted" && ! -e "$ARC_TEST_ROOT/admission-fence" ]]; then exit 97; fi if [[ "$ARC_TEST_CASE" == failed-node-cleanup && -e "$ARC_TEST_ROOT/deleted" ]]; then exit 1; fi if [[ "$ARC_TEST_CASE" == unknown-create && -e "$ARC_TEST_ROOT/live-pod" ]]; then exit 1; fi if [[ "$ARC_TEST_CASE" == replacement-race && -e "$ARC_TEST_ROOT/replacement-preserved" ]]; then exit 1; fi @@ -199,12 +224,31 @@ case "$args" in fi ;; 'create --dry-run=server -f '*) file=$4 + if [[ -e "$ARC_TEST_ROOT/admission-fence" ]] && jq -e '.spec.activeDeadlineSeconds == null' "$file" >/dev/null; then + [[ "$ARC_TEST_CASE" != fence-transport ]] || { printf 'connection refused\n' >&2; exit 1; } + printf 'pods forbidden: exceeded quota: arc-runtime-admission, requested: pods=1, limited: pods=0\n' >&2 + exit 1 + fi if jq -e '.spec.containers[0].securityContext.privileged == true or any(.spec.volumes[]; .hostPath != null)' "$file" >/dev/null; then if [[ "$ARC_TEST_CASE" == admission-transport ]]; then printf 'connection refused\n' >&2 else printf 'violates PodSecurity restricted: privileged hostPath\n' >&2; fi exit 1 fi ;; 'create -f '*) + if jq -e '.kind == "ResourceQuota"' "$3" >/dev/null; then + jq -e '.metadata.name == "arc-runtime-admission" and .metadata.namespace == "arc-runners" and + .metadata.labels["platform.devantler.tech/arc-runtime-probe"] == "arc-proof-123-1" and + .spec == {hard:{pods:"0"},scopes:["NotTerminating"]}' "$3" >/dev/null || exit 96 + [[ "$ARC_TEST_CASE" != foreign-fence ]] || exit 1 + jq '.metadata.uid="fence-uid" | .metadata.resourceVersion="1" | + .status={hard:{pods:"0"},used:{pods:"0"}}' "$3" >"$ARC_TEST_ROOT/admission-fence" + cat "$ARC_TEST_ROOT/admission-fence" + exit 0 + fi + if [[ "$ARC_TEST_CASE" == late-runner ]]; then + [[ -e "$ARC_TEST_ROOT/admission-fence" ]] || { touch "$ARC_TEST_ROOT/late-job-admitted"; exit 1; } + touch "$ARC_TEST_ROOT/late-job-blocked" + fi cp "$3" "$ARC_TEST_ROOT/desired-pod" jq --arg image "ghcr.io/devantler-tech/ksail-analysis-runner@$digest" ' .metadata.uid="owned-uid" | .spec.nodeName="autoscale-arc-runners-0123456789abcdef" | @@ -223,6 +267,14 @@ case "$args" in jq '.spec.initContainers[0].securityContext.privileged=true' "$ARC_TEST_ROOT/live-pod" else cat "$ARC_TEST_ROOT/live-pod"; fi ;; 'delete --raw='*) + if [[ "$args" == *'/resourcequotas/'* ]]; then + [[ "$ARC_TEST_CASE" != replacement-fence ]] || { touch "$ARC_TEST_ROOT/fence-replacement-preserved"; exit 1; } + jq -e '.kind == "DeleteOptions" and .preconditions == {uid:"fence-uid",resourceVersion:"1"}' "$4" >/dev/null || exit 96 + [[ ! -e "$ARC_TEST_ROOT/live-pod" ]] || exit 96 + rm "$ARC_TEST_ROOT/admission-fence" + touch "$ARC_TEST_ROOT/fence-deleted" + exit 0 + fi [[ "$ARC_TEST_CASE" != replacement-race ]] || { touch "$ARC_TEST_ROOT/replacement-preserved"; exit 1; } jq -e '.kind=="DeleteOptions" and .preconditions.uid=="owned-uid" and .gracePeriodSeconds==5' "$4" >/dev/null || exit 96 rm "$ARC_TEST_ROOT/live-pod" @@ -294,6 +346,8 @@ run_case() { rm -f "$scratch/live-pod" "$scratch/deleted" "$scratch/replacement-preserved" "$scratch/runtime-access" rm -f "$scratch/flux-reads" "$scratch/registration-reads" "$scratch/registration-budget" rm -f "$scratch/metrics-executed" + rm -f "$scratch/admission-fence" "$scratch/fence-deleted" "$scratch/job-completed" \ + "$scratch/late-job-admitted" "$scratch/late-job-blocked" "$scratch/fence-replacement-preserved" ( cd "$scratch" PATH="$scratch/bin:$PATH" GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform \ @@ -356,6 +410,26 @@ for name in retained-active-min retained-active-max retained-suspended retained- [[ ! -e "$scratch/runtime-access" && ! -e "$scratch/live-pod" ]] done run_case complete-proof pass +run_case busy-runner pass +[[ -e "$scratch/job-completed" && -e "$scratch/fence-deleted" ]] +run_case late-runner pass +[[ -e "$scratch/late-job-blocked" && ! -e "$scratch/late-job-admitted" && -e "$scratch/fence-deleted" ]] +run_case foreign-fence fail +[[ ! -e "$scratch/live-pod" && ! -e "$scratch/fence-deleted" ]] +run_case replacement-fence fail +[[ -e "$scratch/fence-replacement-preserved" && ! -e "$scratch/fence-deleted" ]] +run_case fence-transport fail +[[ ! -e "$scratch/live-pod" && -e "$scratch/fence-deleted" ]] +for name in runner-deadline retained-runner-deadline; do + run_case "$name" fail + [[ ! -e "$scratch/live-pod" && ! -e "$scratch/admission-fence" ]] +done +run_case template-churn fail +[[ ! -e "$scratch/live-pod" && -e "$scratch/fence-deleted" ]] +run_case invalid-runner-identity fail +[[ ! -e "$scratch/admission-fence" ]] +run_case drift-during-probe fail +[[ -e "$scratch/deleted" && -e "$scratch/fence-deleted" ]] run_case live-api-error fail grep -Fq 'ARC acceptance: FAIL at immutable-job-metrics' "$scratch/stderr" || { printf 'FAIL: live API error did not identify the failing acceptance stage\n' >&2 diff --git a/scripts/verify-ksail-arc-runtime.sh b/scripts/verify-ksail-arc-runtime.sh index 139f382aca..bc6b66b1f5 100755 --- a/scripts/verify-ksail-arc-runtime.sh +++ b/scripts/verify-ksail-arc-runtime.sh @@ -58,10 +58,13 @@ trap - EXIT readonly context=admin@prod namespace=arc-runners readonly ownership_label=platform.devantler.tech/arc-runtime-probe readonly probe="arc-proof-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" +readonly fence_name=arc-runtime-admission scratch=$(mktemp -d) readonly scratch created=false probe_uid='' +fence_created=false +fence_uid='' stage=initialization trap 'printf "ARC acceptance: FAIL at %s\n" "$stage" >&2' ERR @@ -71,6 +74,42 @@ wait_ready() { wait "pod/$probe" --for=condition=Ready --timeout=600s 2>/dev/null } quiet() { "$@" >/dev/null 2>"$scratch/command-error"; } +verify_admission_fence() { + kc -n "$namespace" get resourcequota "$fence_name" -o json >"$scratch/live-fence.json" + jq -e --arg name "$fence_name" --arg owner "$probe" --arg key "$ownership_label" --arg uid "$fence_uid" ' + .metadata.name == $name and .metadata.labels[$key] == $owner and .metadata.uid == $uid and + .metadata.deletionTimestamp == null and .spec == {hard:{pods:"0"},scopes:["NotTerminating"]} and + .status.hard == {pods:"0"} + ' "$scratch/live-fence.json" >/dev/null +} +verify_runner_deadlines() { + local resource path line first + # Project only metadata and the deadline: EphemeralRunner objects can carry + # generated JIT credentials, which this acceptance must never read. + for resource in ephemeralrunnersets.actions.github.com ephemeralrunners.actions.github.com; do + if [[ "$resource" == ephemeralrunnersets.actions.github.com ]]; then + path='{.spec.ephemeralRunnerSpec.spec.activeDeadlineSeconds}' + else + path='{.spec.spec.activeDeadlineSeconds}' + fi + kc -n "$namespace" get "$resource" \ + -o "jsonpath={\"continue=\"}{.metadata.continue}{\"\\n\"}{range .items[*]}{\"uid=\"}{.metadata.uid}{\";deadline=\"}$path{\"\\n\"}{end}" \ + >"$scratch/runner-deadlines" + first=true + while IFS= read -r line; do + if "$first"; then [[ "$line" == continue= ]] || return 1; first=false + else [[ "$line" =~ ^uid=[a-zA-Z0-9-]+\;deadline=$ ]] || return 1; fi + done <"$scratch/runner-deadlines" + ! "$first" || return 1 + done +} +verify_runner_source() { + verify_runner_deadlines + kc -n "$namespace" get autoscalingrunnerset.actions.github.com platform-linux -o json >"$scratch/ars-current.json" + jq -e --slurpfile before "$scratch/ars.json" '.metadata.uid == $before[0].metadata.uid and + .metadata.generation == $before[0].metadata.generation and .spec == $before[0].spec' \ + "$scratch/ars-current.json" >/dev/null +} cleanup() { local result=$? failed=0 remaining='' @@ -104,6 +143,7 @@ cleanup() { # Never delete nodes or resize a shared pool to manufacture cleanup. local deadline=$((SECONDS + 1200)) while ((SECONDS < deadline)); do + verify_admission_fence || { failed=1; break; } remaining=$(kc get nodes -o json) || { failed=1; break; } if jq -e '.items | all(.[]; .metadata.labels["platform.devantler.tech/ci-runner"] != "enabled" and (.metadata.name | startswith("autoscale-arc-runners-") | not))' <<<"$remaining" >/dev/null; then break; fi @@ -112,6 +152,31 @@ cleanup() { jq -e '.items | all(.[]; .metadata.labels["platform.devantler.tech/ci-runner"] != "enabled" and (.metadata.name | startswith("autoscale-arc-runners-") | not))' <<<"$remaining" >/dev/null || failed=1 fi + if "$fence_created" && [[ "$failed" == 0 ]]; then + # Keep admission closed until the probe and its node are gone. A replacement + # quota, a timed-out create with no UID, or an incomplete cleanup needs recovery. + local fence + if [[ -z "$fence_uid" ]]; then + failed=1 + else + fence=$(kc -n "$namespace" get resourcequota "$fence_name" --ignore-not-found -o json) || failed=1 + if ! jq -e --arg name "$fence_name" --arg owner "$probe" --arg key "$ownership_label" --arg uid "$fence_uid" ' + .metadata.name == $name and .metadata.labels[$key] == $owner and .metadata.uid == $uid and + .spec == {hard:{pods:"0"},scopes:["NotTerminating"]} and + (.metadata.resourceVersion | type == "string" and length > 0) + ' <<<"$fence" >/dev/null; then + failed=1 + else + jq '{apiVersion:"v1",kind:"DeleteOptions",preconditions:{uid:.metadata.uid, + resourceVersion:.metadata.resourceVersion},propagationPolicy:"Background"}' \ + <<<"$fence" >"$scratch/delete-fence.json" + quiet kc delete --raw="/api/v1/namespaces/$namespace/resourcequotas/$fence_name" \ + -f "$scratch/delete-fence.json" || failed=1 + fence=$(kc -n "$namespace" get resourcequota "$fence_name" --ignore-not-found -o json) || failed=1 + [[ -z "$fence" ]] || failed=1 + fi + fi + fi rm -rf "$scratch" if [[ "$failed" != 0 ]]; then printf 'ARC acceptance: FAIL cleanup\n' >&2; exit 4; fi printf 'ARC acceptance: cleanup verified\n' @@ -139,6 +204,8 @@ stage=registration-and-bounds kc -n "$namespace" get autoscalingrunnerset.actions.github.com platform-linux -o json >"$scratch/ars.json" jq -e ' .spec.template.spec.volumes[2].configMap.name as $metrics | + (.metadata.uid | type == "string" and length > 0) and + (.metadata.generation | type == "number" and . > 0) and .status.phase == "Running" and .status.observedGeneration == .metadata.generation and ((.metadata.annotations["runner-scale-set-id"] | tonumber) > 0) and .spec.githubConfigUrl == "https://github.com/devantler-tech" and @@ -147,6 +214,7 @@ jq -e ' .metadata.annotations["actions.github.com/runner-scale-set-name"] == "platform-linux" and .spec.githubConfigSecret == "arc-github-app" and .spec.runnerScaleSetName == "platform-linux" and .spec.minRunners == 0 and .spec.maxRunners == 1 and + .spec.template.spec.activeDeadlineSeconds == null and .spec.template.spec.serviceAccountName == "platform-linux-gha-rs-no-permission" and .spec.template.spec.automountServiceAccountToken == false and .spec.template.spec.securityContext.runAsUser == 1001 and @@ -207,6 +275,55 @@ if [[ "$runtime_digest" != "$pinned_digest" ]]; then <"$scratch/runtime-image.json" >"$scratch/runtime-readback" cmp -s "$scratch/runtime-digest" "$scratch/runtime-readback" || fail runtime-descriptor fi +stage=runner-admission-fence +# A scoped quota leaves every existing job running and rejects new ordinary +# runner Pods, including creations already queued by a stale ARC controller. +# The bounded probe has activeDeadlineSeconds and is outside NotTerminating. +# Source: https://kubernetes.io/docs/concepts/policy/resource-quotas/ +verify_runner_deadlines +kc -n "$namespace" get resourcequotas -l "$ownership_label" -o json \ + | jq -e '.items | length == 0' >/dev/null +jq -n --arg name "$fence_name" --arg owner "$probe" --arg ns "$namespace" --arg key "$ownership_label" ' + {apiVersion:"v1",kind:"ResourceQuota",metadata:{name:$name,namespace:$ns,labels:{($key):$owner}}, + spec:{hard:{pods:"0"},scopes:["NotTerminating"]}}' >"$scratch/fence.json" +fence_created=true +kc create -f "$scratch/fence.json" -o json >"$scratch/created-fence.json" +fence_uid=$(jq -er --arg name "$fence_name" --arg owner "$probe" --arg ns "$namespace" --arg key "$ownership_label" ' + select(.metadata.name == $name and .metadata.namespace == $ns and .metadata.labels[$key] == $owner) | + .metadata.uid | select(type == "string" and length > 0)' "$scratch/created-fence.json") +fence_deadline=$((SECONDS + 120)) +while ((SECONDS < fence_deadline)); do + kc -n "$namespace" get resourcequota "$fence_name" -o json >"$scratch/live-fence.json" + if jq -e --arg uid "$fence_uid" '.metadata.uid == $uid and .spec == {hard:{pods:"0"},scopes:["NotTerminating"]} and + .status.hard == {pods:"0"}' "$scratch/live-fence.json" >/dev/null; then break; fi + sleep 10 +done +jq -e --arg uid "$fence_uid" '.metadata.uid == $uid and .spec == {hard:{pods:"0"},scopes:["NotTerminating"]} and + .status.hard == {pods:"0"}' "$scratch/live-fence.json" >/dev/null +# Prove the actual API denies the unchanged runner template for this quota. +# A transport error, unrelated admission denial, or dry-run success proves nothing. +jq --arg name "$probe-admission" --arg ns "$namespace" ' + {apiVersion:"v1",kind:"Pod",metadata:{name:$name,namespace:$ns, + labels:.spec.template.metadata.labels},spec:.spec.template.spec}' "$scratch/ars.json" >"$scratch/runner-admission.json" +if timeout 35s kubectl --context "$context" --request-timeout=30s create --dry-run=server \ + -f "$scratch/runner-admission.json" >"$scratch/admission-out" 2>"$scratch/admission-error"; then + fail runner-admission-fence +fi +grep -Fq "exceeded quota: $fence_name," "$scratch/admission-error" || fail unproven-runner-admission-fence +stage=runner-job-drain +drain_deadline=$((SECONDS + 1200)) +while ((SECONDS < drain_deadline)); do + kc -n "$namespace" get pods -l platform.devantler.tech/arc-role=runner -o json >"$scratch/runners.json" + kc get nodes -o json >"$scratch/drain-nodes.json" + if jq -e '.items | length == 0' "$scratch/runners.json" >/dev/null && + jq -e '.items | all(.[]; .metadata.labels["platform.devantler.tech/ci-runner"] != "enabled" and + (.metadata.name | startswith("autoscale-arc-runners-") | not))' "$scratch/drain-nodes.json" >/dev/null; then break; fi + sleep 10 +done +jq -e '.items | length == 0' "$scratch/runners.json" >/dev/null +jq -e '.items | all(.[]; .metadata.labels["platform.devantler.tech/ci-runner"] != "enabled" and + (.metadata.name | startswith("autoscale-arc-runners-") | not))' "$scratch/drain-nodes.json" >/dev/null +verify_runner_source stage=idle-pool-baseline kc -n kube-system get deployment cluster-autoscaler-hetzner-cluster-autoscaler \ -o jsonpath='{range .spec.template.spec.containers[*].args[*]}{.}{"\n"}{end}' >"$scratch/autoscaler-args" @@ -228,7 +345,18 @@ jq -e --arg key platform.devantler.tech/ci-runner \ stage=restricted-admission kc -n "$namespace" get resourcequotas -o json >"$scratch/quotas.json" -quiet go run ./scripts/verify-ksail-arc-runtime verify-quota <"$scratch/quotas.json" +# Only our immutable, deadline-scoped fence excludes this probe. Every other +# quota still goes through the existing conservative budget verification. +jq -e --arg name "$fence_name" --arg owner "$probe" --arg key "$ownership_label" --arg uid "$fence_uid" ' + [.items[] | select(.metadata.name == $name)] as $fences | + ($fences | length) == 1 and $fences[0].metadata.uid == $uid and + $fences[0].metadata.labels[$key] == $owner and + $fences[0].spec == {hard:{pods:"0"},scopes:["NotTerminating"]} and + $fences[0].status.hard == {pods:"0"} +' "$scratch/quotas.json" >/dev/null +jq --arg uid "$fence_uid" '.items |= map(select(.metadata.uid != $uid))' \ + "$scratch/quotas.json" >"$scratch/probe-quotas.json" +quiet go run ./scripts/verify-ksail-arc-runtime verify-quota <"$scratch/probe-quotas.json" # Retain the actual chart-generated template; replace only process/lifecycle and run identity. jq --arg name "$probe" --arg ns "$namespace" --arg key "$ownership_label" ' {apiVersion:"v1",kind:"Pod",metadata:{name:$name,namespace:$ns, @@ -255,6 +383,7 @@ for negative in privileged hostpath; do grep -Eqi 'hostPath|host.path' "$scratch/denied-error" || fail unrelated-admission-denial fi done +verify_admission_fence created=true kc create -f "$scratch/pod.json" -o json >"$scratch/created-pod.json" probe_uid=$(jq -er --arg name "$probe" --arg ns "$namespace" --arg key "$ownership_label" \ @@ -367,5 +496,9 @@ cmp -s "$scratch/api-before" "$scratch/api-after" || fail api-target-churn stage=job-cgroup-measurement quiet kc -n "$namespace" exec "$probe" -- /etc/ksail-arc-metrics/job-metrics.sh +verify_admission_fence +verify_runner_source +kc -n "$namespace" get pods -l platform.devantler.tech/arc-role=runner -o json \ + | jq -e --arg uid "$probe_uid" '.items | length == 1 and .[0].metadata.uid == $uid' >/dev/null stage=complete printf 'PASS: ARC registration, restricted image, admission and two intercepted egress denials; manifest=%s\n' "$PLATFORM_MANIFEST_DIGEST" From b0a77dd47087179ef893affec75079d761174f68 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Fri, 9 Oct 2026 12:44:22 +0200 Subject: [PATCH 59/62] fix(arc): recover completed canary admission fences --- .github/actions/deploy-prod/action.yml | 1 + .github/workflows/ci.yaml | 3 +- docs/operations/arc-runners.md | 28 ++- scripts/ksail-arc-admission-fence.sh | 188 ++++++++++++++++ scripts/tests/arc-staging/activation_test.go | 7 +- .../tests/test-verify-ksail-arc-runtime.sh | 206 +++++++++++++++++- scripts/verify-ksail-arc-runtime.sh | 41 ++-- 7 files changed, 449 insertions(+), 25 deletions(-) create mode 100644 scripts/ksail-arc-admission-fence.sh diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index 7ce2f88726..e806012e29 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -603,6 +603,7 @@ runs: ) shell: bash env: + GH_TOKEN: ${{ github.token }} PLATFORM_MANIFEST_DIGEST: ${{ steps.publish_platform_manifest.outputs.digest }} run: | export KUBECONFIG="${HOME}/.kube/config" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1d8ef68e93..0974131b25 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -161,7 +161,7 @@ jobs: - name: 🔬 Validate bounded ARC runtime acceptance run: | go test -race ./scripts/verify-ksail-arc-runtime - shellcheck scripts/verify-ksail-arc-runtime.sh scripts/wait-for-ksail-arc-registration.sh scripts/tests/test-verify-ksail-arc-runtime.sh scripts/ksail-arc-job-metrics.sh + shellcheck scripts/verify-ksail-arc-runtime.sh scripts/ksail-arc-admission-fence.sh scripts/wait-for-ksail-arc-registration.sh scripts/tests/test-verify-ksail-arc-runtime.sh scripts/ksail-arc-job-metrics.sh bash scripts/tests/test-verify-ksail-arc-runtime.sh - name: 🔐 Validate ARC transport proof boundaries @@ -509,6 +509,7 @@ jobs: - 'scripts/tests/test-setup-supply-chain-tools.sh' - '.github/actions/deploy-prod/action.yml' - 'scripts/verify-ksail-arc-runtime.sh' + - 'scripts/ksail-arc-admission-fence.sh' - 'scripts/wait-for-ksail-arc-registration.sh' - 'scripts/verify-ksail-arc-runtime/**' - 'scripts/tests/test-verify-ksail-arc-runtime.sh' diff --git a/docs/operations/arc-runners.md b/docs/operations/arc-runners.md index 2922f48a86..dd80197205 100644 --- a/docs/operations/arc-runners.md +++ b/docs/operations/arc-runners.md @@ -255,12 +255,28 @@ until probe deletion and node cleanup are verified, then deletes only the invocation-owned quota with UID and resource-version preconditions. It changes no runner bounds, Helm reconciliation, App permission or capacity ceiling. -A pre-existing quota, ambiguous create, changed quota identity or incomplete probe -cleanup fails closed. An incomplete cleanup retains the fence for recovery. Before -removing a retained fence, verify its invocation ownership, prove that invocation's -probe is absent and the dedicated pool has drained, and use the current UID and -resource version as deletion preconditions. Never remove another invocation's -fence to make a deployment pass. Quota behavior and its deadline scopes are defined +A later protected deployment can recover a quota left by a completed prior +workflow attempt. The quota records the fixed repository and workflow, run and +attempt, producing job, run-head and checkout revisions, manifest digest and +successfully created probe UID. Recovery joins the exact attempt and its complete +job list using the existing read-only Actions permission. It retains admission +while deleting only that recorded, standalone probe with a UID precondition, +proving absence and natural node drain, then deleting the unchanged quota with +UID and resource-version preconditions. It proves quota absence before creating +the new invocation's fence. + +Active or unknown writers, incomplete job lists, missing provenance, ambiguous +creation, replacement objects and incomplete cleanup remain HOLD. Same-attempt +Heal Prod cannot recover a fence while its attempt is still running. The first +activation therefore also depends on the inactive-main recovery support in +[#4638](https://github.com/devantler-tech/platform/issues/4638) and +[#4641](https://github.com/devantler-tech/platform/pull/4641); candidate-only +acceptance does not establish a recoverable rollout. A legacy fence without the +recorded proof needs explicit manual recovery: prove its producing job and +attempt have stopped, verify invocation ownership and probe absence, prove the +dedicated pool has drained, and use the current UID and resource version as +deletion preconditions. Never release a live or unknown invocation's fence to +make a deployment pass. Quota behavior and its deadline scopes are defined in the [Kubernetes resource quota documentation](https://kubernetes.io/docs/concepts/policy/resource-quotas/). To stop admitting jobs, restore every consumer's prior runner configuration diff --git a/scripts/ksail-arc-admission-fence.sh b/scripts/ksail-arc-admission-fence.sh new file mode 100644 index 0000000000..a288849cdb --- /dev/null +++ b/scripts/ksail-arc-admission-fence.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +# Helpers for the protected canary. Loaded only after deployment identity checks. +# API diagnostics and job metadata stay in the caller's private scratch directory. +# The canary supplies these checked variables and kc/quiet/deadline helpers. +# shellcheck disable=SC2154 + +readonly arc_origin_key=platform.devantler.tech/arc-runtime-origin +readonly arc_probe_uid_key=platform.devantler.tech/arc-runtime-probe-uid + +arc_actions_read() { + [[ -n "${GH_TOKEN:-}" ]] || return 1 + timeout 60s gh api "$1" --hostname github.com --method GET \ + -H 'Accept: application/vnd.github+json' -H 'X-GitHub-Api-Version: 2022-11-28' \ + "${@:3}" >"$2" 2>"$scratch/actions-error" || return 1 + jq -e -s 'length == 1' "$2" >/dev/null +} + +arc_attempt_identity() { + local origin=$1 terminal=$2 run attempt workflow head job job_name + run=$(jq -er '.run' "$origin") + attempt=$(jq -er '.attempt' "$origin") + workflow=$(jq -er '.workflow' "$origin") + head=$(jq -er '.head' "$origin") + job=$(jq -er '.job' "$origin") + job_name=$(jq -er '.job_name' "$origin") + arc_actions_read "repos/devantler-tech/platform/actions/runs/$run/attempts/$attempt" "$scratch/attempt.json" + jq -e --arg run "$run" --arg attempt "$attempt" --arg workflow "$workflow" --arg head "$head" --arg terminal "$terminal" ' + (.id | tostring) == $run and (.run_attempt | tostring) == $attempt and .head_sha == $head and + .path == $workflow and .repository.full_name == "devantler-tech/platform" and + .head_repository.full_name == "devantler-tech/platform" and + ((.path == ".github/workflows/ci.yaml" and .event == "merge_group") or + (.path == ".github/workflows/cd.yaml" and .event == "workflow_dispatch")) and + (if $terminal == "true" then .status == "completed" and .conclusion != null + else .status == "in_progress" end) + ' "$scratch/attempt.json" >/dev/null + arc_actions_read "repos/devantler-tech/platform/actions/runs/$run/attempts/$attempt/jobs?per_page=100" \ + "$scratch/attempt-jobs.json" --paginate --slurp + # Pagination must be complete and consistent, with no duplicate job identity. + jq -e --arg run "$run" --arg head "$head" --arg job "$job" --arg name "$job_name" --arg terminal "$terminal" ' + length > 0 and length <= 25 and + (.[0].total_count as $total | all(.[]; .total_count == $total) and + ([.[].jobs[]] | length) == $total) and + ([.[].jobs[].id] | length == (unique | length)) and + ([.[].jobs[] | select((.id | tostring) == $job and .name == $name and + (.run_id | tostring) == $run and .head_sha == $head and + (if $terminal == "true" then .status == "completed" and .conclusion != null and + (.completed_at | type == "string" and length > 0) + else .status == "in_progress" end))] | length) == 1 + ' "$scratch/attempt-jobs.json" >/dev/null +} + +arc_current_origin() { + local workflow job_name source job + [[ "${GITHUB_SHA:-}" =~ ^[0-9a-f]{40}$ && + "$GITHUB_RUN_ID" =~ ^[1-9][0-9]{0,14}$ ]] || return 1 + case "${GITHUB_WORKFLOW_REF:-}:${GITHUB_JOB:-}" in + devantler-tech/platform/.github/workflows/ci.yaml@*:deploy-prod) + workflow=.github/workflows/ci.yaml; job_name='🚀 Deploy to Prod' ;; + devantler-tech/platform/.github/workflows/cd.yaml@*:deploy-prod) + workflow=.github/workflows/cd.yaml; job_name='🚀 Deploy to Production' ;; + devantler-tech/platform/.github/workflows/ci.yaml@*:heal-prod-on-failure) + workflow=.github/workflows/ci.yaml + job_name='🩹 Heal Prod (restore main after an unsuccessful or evicted merge-group deploy)' ;; + *) return 1 ;; + esac + source=$(git rev-parse HEAD) + [[ "$source" =~ ^[0-9a-f]{40}$ ]] || return 1 + [[ "$source" == "$GITHUB_SHA" || "$GITHUB_JOB" == heal-prod-on-failure ]] || return 1 + arc_actions_read "repos/devantler-tech/platform/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100" \ + "$scratch/current-jobs.json" --paginate --slurp + job=$(jq -er --arg name "$job_name" --arg run "$GITHUB_RUN_ID" --arg head "$GITHUB_SHA" ' + [.[].jobs[] | select(.name == $name and (.run_id | tostring) == $run and + .head_sha == $head and .status == "in_progress")] | + select(length == 1) | .[0].id | tostring | select(test("^[1-9][0-9]{0,14}$")) + ' "$scratch/current-jobs.json") + jq -n --arg workflow "$workflow" --arg run "$GITHUB_RUN_ID" --arg attempt "$GITHUB_RUN_ATTEMPT" \ + --arg head "$GITHUB_SHA" --arg source "$source" --arg manifest "$PLATFORM_MANIFEST_DIGEST" \ + --arg job "$job" --arg job_name "$job_name" '{repository:"devantler-tech/platform", + workflow:$workflow,run:$run,attempt:$attempt,head:$head,source:$source, + manifest:$manifest,job:$job,job_name:$job_name}' >"$scratch/current-origin.json" + arc_attempt_identity "$scratch/current-origin.json" false +} + +arc_retained_fence_matches() { + kc -n "$namespace" get resourcequota "$fence_name" -o json >"$scratch/retained-fence-current.json" || return 1 + [[ -s "$scratch/retained-fence-current.json" ]] || return 1 + jq -e --slurpfile before "$scratch/retained-fence.json" ' + .metadata.uid == $before[0].metadata.uid and + .metadata.labels == $before[0].metadata.labels and + .metadata.annotations == $before[0].metadata.annotations and + .metadata.deletionTimestamp == null and (.metadata.ownerReferences // [] | length) == 0 and + .spec == {hard:{pods:"0"},scopes:["NotTerminating"]} and .status.hard == {pods:"0"} + ' "$scratch/retained-fence-current.json" >/dev/null +} + +arc_recover_admission_fence() { + local owner run attempt recorded_uid current deadline remaining + kc -n "$namespace" get resourcequota "$fence_name" --ignore-not-found -o json >"$scratch/retained-fence.json" + [[ -s "$scratch/retained-fence.json" ]] || return 0 + jq -e --arg name "$fence_name" --arg ns "$namespace" ' + .metadata.name == $name and .metadata.namespace == $ns and + (.metadata.uid | type == "string" and length > 0) and + (.metadata.resourceVersion | type == "string" and length > 0) and + .metadata.deletionTimestamp == null and (.metadata.ownerReferences // [] | length) == 0 and + .spec == {hard:{pods:"0"},scopes:["NotTerminating"]} and .status.hard == {pods:"0"} + ' "$scratch/retained-fence.json" >/dev/null + jq -er --arg key "$arc_origin_key" '.metadata.annotations[$key] | fromjson' \ + "$scratch/retained-fence.json" >"$scratch/prior-origin.json" + jq -e 'type == "object" and .repository == "devantler-tech/platform" and + (.run | test("^[1-9][0-9]{0,14}$")) and (.attempt | test("^[1-9][0-9]{0,2}$")) and + (.job | test("^[1-9][0-9]{0,14}$")) and + (.head | test("^[0-9a-f]{40}$")) and (.source | test("^[0-9a-f]{40}$")) and + (.manifest | test("^sha256:[0-9a-f]{64}$")) and + ((.workflow == ".github/workflows/ci.yaml" and .job_name == "🚀 Deploy to Prod" and .source == .head) or + (.workflow == ".github/workflows/cd.yaml" and .job_name == "🚀 Deploy to Production" and .source == .head) or + (.workflow == ".github/workflows/ci.yaml" and + .job_name == "🩹 Heal Prod (restore main after an unsuccessful or evicted merge-group deploy)")) + ' "$scratch/prior-origin.json" >/dev/null + run=$(jq -r '.run' "$scratch/prior-origin.json") + attempt=$(jq -r '.attempt' "$scratch/prior-origin.json") + owner="arc-proof-$run-$attempt" + [[ "$owner" != "$probe" ]] || return 1 + jq -e --arg key "$ownership_label" --arg owner "$owner" '.metadata.labels[$key] == $owner' \ + "$scratch/retained-fence.json" >/dev/null + # Whole-attempt death is deliberately required. Same-run Heal Prod remains + # HOLD; its recovery support must be staged on inactive main before activation. + arc_attempt_identity "$scratch/prior-origin.json" true + verify_runner_deadlines + arc_retained_fence_matches + current=$(kc -n "$namespace" get pod "$owner" --ignore-not-found -o json) + if [[ -n "$current" ]]; then + recorded_uid=$(jq -er --arg key "$arc_probe_uid_key" \ + '.metadata.annotations[$key] | select(type == "string" and length > 0)' "$scratch/retained-fence.json") + jq -e --arg name "$owner" --arg ns "$namespace" --arg key "$ownership_label" --arg uid "$recorded_uid" ' + .metadata.name == $name and .metadata.namespace == $ns and .metadata.labels[$key] == $name and + .metadata.uid == $uid and (.metadata.ownerReferences // [] | length) == 0 and + .spec.activeDeadlineSeconds == 1200 and .spec.restartPolicy == "Never" + ' <<<"$current" >/dev/null + jq -n --arg uid "$recorded_uid" '{apiVersion:"v1",kind:"DeleteOptions",preconditions:{uid:$uid}, + gracePeriodSeconds:5,propagationPolicy:"Background"}' >"$scratch/prior-delete.json" + arc_retained_fence_matches + quiet kc delete --raw="/api/v1/namespaces/$namespace/pods/$owner" -f "$scratch/prior-delete.json" + quiet kc -n "$namespace" wait "pod/$owner" --for=delete --timeout=25s + fi + current=$(kc -n "$namespace" get pod "$owner" --ignore-not-found -o json) + [[ -z "$current" ]] + kc -n "$namespace" get pods -l "$ownership_label=$owner" -o json >"$scratch/prior-probes.json" + jq -e '(.metadata.continue // "") == "" and (.items | length == 0)' "$scratch/prior-probes.json" >/dev/null + deadline=$((SECONDS + 1200)) + while ((SECONDS < deadline)); do + arc_retained_fence_matches + remaining=$(kc get nodes -o json) + if jq -e '(.metadata.continue // "") == "" and (.items | all(.[]; + .metadata.labels["platform.devantler.tech/ci-runner"] != "enabled" and + (.metadata.name | startswith("autoscale-arc-runners-") | not)))' <<<"$remaining" >/dev/null; then break; fi + sleep 10 + done + jq -e '(.metadata.continue // "") == "" and (.items | all(.[]; + .metadata.labels["platform.devantler.tech/ci-runner"] != "enabled" and + (.metadata.name | startswith("autoscale-arc-runners-") | not)))' <<<"$remaining" >/dev/null + arc_attempt_identity "$scratch/prior-origin.json" true + verify_runner_deadlines + arc_retained_fence_matches + current=$(kc -n "$namespace" get pod "$owner" --ignore-not-found -o json) + [[ -z "$current" ]] + kc -n "$namespace" get pods -l "$ownership_label=$owner" -o json >"$scratch/prior-probes.json" + jq -e '(.metadata.continue // "") == "" and (.items | length == 0)' "$scratch/prior-probes.json" >/dev/null + jq '{apiVersion:"v1",kind:"DeleteOptions",preconditions:{uid:.metadata.uid, + resourceVersion:.metadata.resourceVersion},propagationPolicy:"Background"}' \ + "$scratch/retained-fence-current.json" >"$scratch/prior-delete-fence.json" + quiet kc delete --raw="/api/v1/namespaces/$namespace/resourcequotas/$fence_name" -f "$scratch/prior-delete-fence.json" + current=$(kc -n "$namespace" get resourcequota "$fence_name" --ignore-not-found -o json) + [[ -z "$current" ]] + printf 'ARC acceptance: abandoned admission fence recovered\n' +} + +arc_record_probe_uid() { + verify_admission_fence + jq -n --arg uid "$fence_uid" --arg version "$(jq -r '.metadata.resourceVersion' "$scratch/live-fence.json")" \ + --arg probe_uid "$probe_uid" '[{op:"test",path:"/metadata/uid",value:$uid}, + {op:"test",path:"/metadata/resourceVersion",value:$version}, + {op:"add",path:"/metadata/annotations/platform.devantler.tech~1arc-runtime-probe-uid",value:$probe_uid}]' \ + >"$scratch/probe-uid-patch.json" + quiet kc -n "$namespace" patch resourcequota "$fence_name" --type=json --patch-file "$scratch/probe-uid-patch.json" + verify_admission_fence + jq -e --arg key "$arc_probe_uid_key" --arg uid "$probe_uid" '.metadata.annotations[$key] == $uid' \ + "$scratch/live-fence.json" >/dev/null +} diff --git a/scripts/tests/arc-staging/activation_test.go b/scripts/tests/arc-staging/activation_test.go index 02f2c2c3d2..6bcc50e3d6 100644 --- a/scripts/tests/arc-staging/activation_test.go +++ b/scripts/tests/arc-staging/activation_test.go @@ -45,7 +45,7 @@ func activationFixture(active bool) fstest.MapFS { runtimeAction: "runs:\n steps:\n - name: Verify the deployed KSail ARC canary\n" + " if: >-\n !cancelled() && steps.wait_flux_revision.outcome == 'success' && " + "( steps.cluster_update.outcome == 'skipped' || steps.wait_prod_api_stability.outcome == 'success' )\n" + - " shell: bash\n env:\n PLATFORM_MANIFEST_DIGEST: ${{ steps.publish_platform_manifest.outputs.digest }}\n" + + " shell: bash\n env:\n GH_TOKEN: ${{ github.token }}\n PLATFORM_MANIFEST_DIGEST: ${{ steps.publish_platform_manifest.outputs.digest }}\n" + " run: |\n export KUBECONFIG=\"${HOME}/.kube/config\"\n bash scripts/verify-ksail-arc-runtime.sh --if-active\n", } { files[name] = &fstest.MapFile{Data: []byte(data)} @@ -76,6 +76,8 @@ func TestActivationEnvelopeRejectsUnboundedOrUnverifiedStates(t *testing.T) { {"skipped canary", runtimeAction, "!cancelled() &&", "false &&"}, {"waived canary", runtimeAction, " shell: bash", " continue-on-error: true\n shell: bash"}, {"wrong deployment digest", runtimeAction, "steps.publish_platform_manifest.outputs.digest", "github.sha"}, + {"missing Actions read token", runtimeAction, " GH_TOKEN: ${{ github.token }}\n", ""}, + {"wrong Actions read token", runtimeAction, "GH_TOKEN: ${{ github.token }}", "GH_TOKEN: ${{ secrets.GHCR_TOKEN }}"}, {"unexpected proof environment", runtimeAction, " env:", " env:\n PATH: /unverified"}, {"missing suspension declaration", runnerRelease, "suspend: false", "unrelated: false"}, } @@ -235,7 +237,8 @@ func validateARCRuntimeHook(files fs.FS) error { run, runOK := step["run"].(string) env, envOK := step["env"].(map[string]any) if waived || !ok || strings.Join(strings.Fields(guard), " ") != condition || !runOK || strings.TrimSpace(run) != command || - step["shell"] != "bash" || !envOK || len(env) != 1 || env["PLATFORM_MANIFEST_DIGEST"] != "${{ steps.publish_platform_manifest.outputs.digest }}" { + step["shell"] != "bash" || !envOK || len(env) != 2 || env["GH_TOKEN"] != "${{ github.token }}" || + env["PLATFORM_MANIFEST_DIGEST"] != "${{ steps.publish_platform_manifest.outputs.digest }}" { return fmt.Errorf("ARC runtime proof is skipped, waived, or bound to the wrong deployment") } } diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh index a84c77698d..1db409cb5b 100755 --- a/scripts/tests/test-verify-ksail-arc-runtime.sh +++ b/scripts/tests/test-verify-ksail-arc-runtime.sh @@ -8,6 +8,7 @@ mkdir -p "$scratch/bin" "$scratch/scripts" "$scratch/k8s/providers/hetzner/infra "$scratch/k8s/bases/infrastructure/actions-runners" \ "$scratch/k8s/bases/infrastructure/controllers/actions-runner-controller" cp "$root/scripts/verify-ksail-arc-runtime.sh" "$scratch/scripts/" +cp "$root/scripts/ksail-arc-admission-fence.sh" "$scratch/scripts/" if [[ -e "$root/scripts/wait-for-ksail-arc-registration.sh" ]]; then cp "$root/scripts/wait-for-ksail-arc-registration.sh" "$scratch/scripts/" fi @@ -65,6 +66,50 @@ cat >"$scratch/bin/go" <<'SH' shift 2 exec "$ARC_TEST_ROOT/verifier" "$@" SH +cat >"$scratch/bin/git" <<'SH' +#!/usr/bin/env bash +[[ "$*" == 'rev-parse HEAD' ]] || exit 97 +printf '%040d\n' 1 +SH +cat >"$scratch/bin/gh" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +[[ "$1" == api && "$*" == *'--hostname github.com --method GET'* ]] || exit 97 +case "$2" in + 'repos/devantler-tech/platform/actions/runs/123/attempts/1') + [[ "$ARC_TEST_CASE" != orphan-current-empty-api ]] || exit 0 + printf '{"id":123,"run_attempt":1,"head_sha":"%040d","path":".github/workflows/ci.yaml","event":"merge_group","status":"in_progress","repository":{"full_name":"devantler-tech/platform"},"head_repository":{"full_name":"devantler-tech/platform"}}' 1 ;; + 'repos/devantler-tech/platform/actions/runs/122/attempts/1') + [[ "$ARC_TEST_CASE" != orphan-api-unknown ]] || exit 42 + if [[ "$ARC_TEST_CASE" == orphan-final-api-unknown && -e "$ARC_TEST_ROOT/prior-probe-deleted" ]]; then exit 42; fi + status=completed + [[ "$ARC_TEST_CASE" != orphan-active ]] || status=in_progress + jq -cn --arg status "$status" --arg scenario "$ARC_TEST_CASE" '{id:122,run_attempt:1,head_sha:"0000000000000000000000000000000000000001", + path:".github/workflows/ci.yaml",event:"merge_group",status:$status,conclusion:"cancelled", + repository:{full_name:"devantler-tech/platform"},head_repository:{full_name:"devantler-tech/platform"}} | + if $scenario=="orphan-run-mismatch" then .run_attempt=2 + elif $scenario=="orphan-wrong-api-head" then .head_sha="0000000000000000000000000000000000000002" + elif $scenario=="orphan-wrong-api-workflow" then .path=".github/workflows/unrelated.yaml" + elif $scenario=="orphan-wrong-api-repository" then .repository.full_name="devantler-tech/ksail" + else . end' ;; + 'repos/devantler-tech/platform/actions/runs/123/attempts/1/jobs?per_page=100') + [[ "$ARC_TEST_CASE" != orphan-current-job-list-empty ]] || exit 0 + printf '[{"total_count":1,"jobs":[{"id":456,"run_id":123,"head_sha":"%040d","name":"🚀 Deploy to Prod","status":"in_progress"}]}]' 1 ;; + 'repos/devantler-tech/platform/actions/runs/122/attempts/1/jobs?per_page=100') + count=1 + [[ "$ARC_TEST_CASE" != orphan-incomplete-jobs ]] || count=2 + jq -cn --argjson count "$count" --arg scenario "$ARC_TEST_CASE" '{total_count:$count,jobs:[{id:455,run_id:122, + head_sha:"0000000000000000000000000000000000000001",name:"🚀 Deploy to Prod",status:"completed", + conclusion:"cancelled",completed_at:"2026-10-05T22:00:00Z"}]} | + if $scenario=="orphan-duplicate-job" then .total_count=2 | .jobs += .jobs + elif $scenario=="orphan-job-active" then .jobs[0].status="in_progress" | .jobs[0].completed_at=null + elif $scenario=="orphan-wrong-job" then .jobs[0].id=999 + else . end | if $scenario=="orphan-paginated" then + [.total_count=2, {total_count:2,jobs:[{id:999,run_id:122,name:"Unrelated job",status:"completed"}]}] + else [.] end' ;; + *) exit 97 ;; +esac +SH cat >"$scratch/bin/cosign" <<'SH' #!/usr/bin/env bash [[ "$*" == *"--certificate-identity https://github.com/devantler-tech/ksail/.github/workflows/publish-ksail-analysis-runner.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com ghcr.io/devantler-tech/ksail-analysis-runner@$digest" ]] || exit 92 @@ -204,8 +249,29 @@ case "$args" in } | jq --slurpfile fence "$ARC_TEST_ROOT/admission-fence" '.items += $fence' ;; *'get resourcequota arc-runtime-admission '*) [[ -e "$ARC_TEST_ROOT/admission-fence" ]] || exit 0 + if [[ "$ARC_TEST_CASE" == own-fence-empty && -e "$ARC_TEST_ROOT/fence-mutated" ]]; then exit 0; fi + if [[ "$ARC_TEST_CASE" == orphan-quota-churn ]]; then + if [[ -e "$ARC_TEST_ROOT/prior-fence-read" ]]; then + jq '.metadata.uid="replacement-fence-uid"' "$ARC_TEST_ROOT/admission-fence" >"$ARC_TEST_ROOT/replaced-fence" + mv "$ARC_TEST_ROOT/replaced-fence" "$ARC_TEST_ROOT/admission-fence" + fi + touch "$ARC_TEST_ROOT/prior-fence-read" + fi cat "$ARC_TEST_ROOT/admission-fence" ;; + *'patch resourcequota arc-runtime-admission --type=json --patch-file '*) + patch=${!#} + jq -e '.[0] == {op:"test",path:"/metadata/uid",value:"fence-uid"} and + .[1] == {op:"test",path:"/metadata/resourceVersion",value:"1"} and + .[2] == {op:"add",path:"/metadata/annotations/platform.devantler.tech~1arc-runtime-probe-uid",value:"owned-uid"}' \ + "$patch" >/dev/null || exit 96 + jq '.metadata.annotations["platform.devantler.tech/arc-runtime-probe-uid"]="owned-uid" | + .metadata.resourceVersion="2"' "$ARC_TEST_ROOT/admission-fence" >"$ARC_TEST_ROOT/patched-fence" + mv "$ARC_TEST_ROOT/patched-fence" "$ARC_TEST_ROOT/admission-fence" ;; *'get pods -l platform.devantler.tech/arc-role=runner '*|*'get pods -l platform.devantler.tech/arc-runtime-probe'*) + if [[ "$ARC_TEST_CASE" == orphan-incomplete-probes && "$args" == *'arc-runtime-probe='* ]]; then + printf '{"metadata":{"continue":"next"},"items":[]}' + exit 0 + fi if [[ "$args" == *'arc-role=runner'* && -e "$ARC_TEST_ROOT/live-pod" ]]; then jq '{items:[.]}' "$ARC_TEST_ROOT/live-pod" elif [[ "$args" == *'arc-role=runner'* && "$ARC_TEST_CASE" == busy-runner && ! -e "$ARC_TEST_ROOT/job-completed" ]]; then @@ -213,6 +279,7 @@ case "$args" in printf '{"items":[{"metadata":{"name":"legitimate-runner","uid":"job-uid"},"spec":{},"status":{"phase":"Running"}}]}' else printf '{"items":[]}'; fi ;; 'get nodes -o json') + if [[ "$ARC_TEST_CASE" == orphan-node-api && -e "$ARC_TEST_ROOT/prior-probe-deleted" ]]; then exit 42; fi if [[ -e "$ARC_TEST_ROOT/deleted" && ! -e "$ARC_TEST_ROOT/admission-fence" ]]; then exit 97; fi if [[ "$ARC_TEST_CASE" == failed-node-cleanup && -e "$ARC_TEST_ROOT/deleted" ]]; then exit 1; fi if [[ "$ARC_TEST_CASE" == unknown-create && -e "$ARC_TEST_ROOT/live-pod" ]]; then exit 1; fi @@ -233,9 +300,20 @@ case "$args" in if [[ "$ARC_TEST_CASE" == admission-transport ]]; then printf 'connection refused\n' >&2 else printf 'violates PodSecurity restricted: privileged hostPath\n' >&2; fi exit 1 - fi ;; + fi + case "$ARC_TEST_CASE" in + own-fence-origin-churn|own-fence-controller-churn|own-fence-deleting|own-fence-empty) + jq --arg scenario "$ARC_TEST_CASE" ' + if $scenario=="own-fence-origin-churn" then .metadata.annotations["platform.devantler.tech/arc-runtime-origin"]="changed" + elif $scenario=="own-fence-controller-churn" then .metadata.ownerReferences=[{uid:"controller-uid"}] + elif $scenario=="own-fence-deleting" then .metadata.deletionTimestamp="2026-10-05T22:00:00Z" + else . end' "$ARC_TEST_ROOT/admission-fence" >"$ARC_TEST_ROOT/mutated-fence" + mv "$ARC_TEST_ROOT/mutated-fence" "$ARC_TEST_ROOT/admission-fence" + touch "$ARC_TEST_ROOT/fence-mutated" ;; + esac ;; 'create -f '*) if jq -e '.kind == "ResourceQuota"' "$3" >/dev/null; then + [[ ! -e "$ARC_TEST_ROOT/admission-fence" ]] || exit 1 jq -e '.metadata.name == "arc-runtime-admission" and .metadata.namespace == "arc-runners" and .metadata.labels["platform.devantler.tech/arc-runtime-probe"] == "arc-proof-123-1" and .spec == {hard:{pods:"0"},scopes:["NotTerminating"]}' "$3" >/dev/null || exit 96 @@ -268,13 +346,28 @@ case "$args" in else cat "$ARC_TEST_ROOT/live-pod"; fi ;; 'delete --raw='*) if [[ "$args" == *'/resourcequotas/'* ]]; then + if jq -e '.preconditions.uid == "prior-fence-uid"' "$4" >/dev/null; then + [[ "$ARC_TEST_CASE" != orphan-replacement ]] || { touch "$ARC_TEST_ROOT/prior-replacement-preserved"; exit 1; } + jq -e '.preconditions == {uid:"prior-fence-uid",resourceVersion:"1"}' "$4" >/dev/null || exit 96 + [[ ! -e "$ARC_TEST_ROOT/live-pod" ]] || exit 96 + rm "$ARC_TEST_ROOT/admission-fence" + touch "$ARC_TEST_ROOT/prior-fence-deleted" + exit 0 + fi [[ "$ARC_TEST_CASE" != replacement-fence ]] || { touch "$ARC_TEST_ROOT/fence-replacement-preserved"; exit 1; } - jq -e '.kind == "DeleteOptions" and .preconditions == {uid:"fence-uid",resourceVersion:"1"}' "$4" >/dev/null || exit 96 + jq -e '.kind == "DeleteOptions" and .preconditions.uid == "fence-uid" and + (.preconditions.resourceVersion == "1" or .preconditions.resourceVersion == "2")' "$4" >/dev/null || exit 96 [[ ! -e "$ARC_TEST_ROOT/live-pod" ]] || exit 96 rm "$ARC_TEST_ROOT/admission-fence" touch "$ARC_TEST_ROOT/fence-deleted" exit 0 fi + if [[ "$args" == *'/pods/arc-proof-122-1'* ]]; then + jq -e '.preconditions.uid == "prior-probe-uid" and .gracePeriodSeconds == 5' "$4" >/dev/null || exit 96 + rm "$ARC_TEST_ROOT/live-pod" + touch "$ARC_TEST_ROOT/prior-probe-deleted" + exit 0 + fi [[ "$ARC_TEST_CASE" != replacement-race ]] || { touch "$ARC_TEST_ROOT/replacement-preserved"; exit 1; } jq -e '.kind=="DeleteOptions" and .preconditions.uid=="owned-uid" and .gracePeriodSeconds==5' "$4" >/dev/null || exit 96 rm "$ARC_TEST_ROOT/live-pod" @@ -341,17 +434,82 @@ esac SH chmod +x "$scratch/bin/"* +expected_refusal() { + case "$1" in + orphan-current-empty-api|orphan-current-job-list-empty|orphan-unauthorized-current) printf 'runner-admission-fence' ;; + orphan-*) printf 'runner-fence-recovery' ;; + own-fence-*) printf 'restricted-admission' ;; + retained-active-min|retained-active-max|retained-suspended|retained-missing-bound|retained-wrong-scale-set) + printf 'retained-source-state' ;; + foreign-fence|retained-runner-deadline) printf 'runner-admission-fence' ;; + replacement-fence|replacement-race|failed-node-cleanup) printf 'cleanup' ;; + fence-transport) printf 'unproven-runner-admission-fence' ;; + template-churn) printf 'runner-job-drain' ;; + invalid-runner-identity|unexpected-hook|writable-metrics|wrong-metrics-configmap|runner-deadline) printf 'registration-and-bounds' ;; + drift-during-probe|failed-metrics-hook) printf 'job-cgroup-measurement' ;; + live-api-error|mutable-metrics|tampered-metrics) printf 'immutable-job-metrics' ;; + registration-timeout|stale-flux|no-registration) printf 'registration-convergence' ;; + rendered-hidden-activation) printf 'partial-activation' ;; + source-render-failure) printf 'source-render' ;; + rendered-suspended|rendered-wrong-image) printf 'rendered-source-state' ;; + invalid-node-suffix-*|wrong-node-pool|full-quota|stale-quota|mutated-init|unknown-create) printf 'restricted-admission' ;; + bad-signature|tampered-image) printf 'immutable-image' ;; + wider-ceiling) printf 'autoscaler-boundary' ;; + admission-transport) printf 'unproven-admission-denial' ;; + insufficient-memory) printf 'allocatable-headroom' ;; + unhealthy-target) printf 'unhealthy-internal-control' ;; + curl-transport) printf 'unproven-curl-timeout' ;; + allowed-egress) printf 'network-isolation' ;; + forwarded-flow|wrong-source|observer-loss) printf 'intercepted-egress-denials' ;; + observer-diagnostics) printf 'observer-diagnostics' ;; + *) printf 'FAIL: no expected refusal for %s\n' "$1" >&2; return 1 ;; + esac +} + run_case() { - local name=$1 expected=$2 code=0 + local name=$1 expected=$2 code=0 producer_job=deploy-prod + [[ "$name" != orphan-unauthorized-current ]] || producer_job=unrelated rm -f "$scratch/live-pod" "$scratch/deleted" "$scratch/replacement-preserved" "$scratch/runtime-access" rm -f "$scratch/flux-reads" "$scratch/registration-reads" "$scratch/registration-budget" rm -f "$scratch/metrics-executed" rm -f "$scratch/admission-fence" "$scratch/fence-deleted" "$scratch/job-completed" \ "$scratch/late-job-admitted" "$scratch/late-job-blocked" "$scratch/fence-replacement-preserved" + rm -f "$scratch/prior-fence-deleted" "$scratch/prior-probe-deleted" "$scratch/prior-replacement-preserved" + rm -f "$scratch/prior-fence-read" "$scratch/fence-mutated" + if [[ "$name" == orphan-* ]]; then + jq -cn --arg manifest "$digest" --arg key platform.devantler.tech/arc-runtime-origin \ + --arg scenario "$name" '{metadata:{name:"arc-runtime-admission",namespace:"arc-runners",uid:"prior-fence-uid",resourceVersion:"1", + labels:{"platform.devantler.tech/arc-runtime-probe":"arc-proof-122-1"}, + annotations:{($key):({repository:"devantler-tech/platform",workflow:".github/workflows/ci.yaml",run:"122",attempt:"1", + head:"0000000000000000000000000000000000000001",source:"0000000000000000000000000000000000000001", + manifest:$manifest,job:"455",job_name:"🚀 Deploy to Prod"}|tojson), + "platform.devantler.tech/arc-runtime-probe-uid":"prior-probe-uid"}}, + spec:{hard:{pods:"0"},scopes:["NotTerminating"]},status:{hard:{pods:"0"},used:{pods:"0"}}} | + if $scenario=="orphan-missing-provenance" then del(.metadata.annotations[$key]) + elif $scenario=="orphan-missing-probe-uid" then del(.metadata.annotations["platform.devantler.tech/arc-runtime-probe-uid"]) + elif $scenario=="orphan-current-attempt" then + .metadata.labels["platform.devantler.tech/arc-runtime-probe"]="arc-proof-123-1" | + .metadata.annotations[$key] |= (fromjson | .run="123" | .job="456" | tojson) + elif $scenario=="orphan-foreign-provenance" then + .metadata.annotations[$key] |= (fromjson | .repository="devantler-tech/ksail" | tojson) + elif $scenario=="orphan-wrong-quota" then .spec.scopes=["Terminating"] + else . end' >"$scratch/admission-fence" + if [[ "$name" != orphan-empty ]]; then + jq -cn --arg scenario "$name" '{metadata:{name:"arc-proof-122-1",namespace:"arc-runners", + uid:(if $scenario=="orphan-wrong-probe" then "replacement-probe-uid" else "prior-probe-uid" end), + labels:{"platform.devantler.tech/arc-runtime-probe":"arc-proof-122-1"}}, + spec:{activeDeadlineSeconds:1200,restartPolicy:"Never"}} | + if $scenario=="orphan-controlled-probe" then .metadata.ownerReferences=[{uid:"controller-uid"}] + elif $scenario=="orphan-wrong-deadline" then .spec.activeDeadlineSeconds=60 + else . end' >"$scratch/live-pod" + fi + fi ( cd "$scratch" PATH="$scratch/bin:$PATH" GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform \ GITHUB_RUN_ID=123 GITHUB_RUN_ATTEMPT=1 PLATFORM_MANIFEST_DIGEST="$digest" ARC_TEST_CASE="$name" \ + GITHUB_SHA=0000000000000000000000000000000000000001 GITHUB_JOB="$producer_job" \ + GITHUB_WORKFLOW_REF=devantler-tech/platform/.github/workflows/ci.yaml@refs/heads/main GH_TOKEN=fixture-only \ bash scripts/verify-ksail-arc-runtime.sh --if-active ) >"$scratch/stdout" 2>"$scratch/stderr" || code=$? if [[ "$expected" == pass ]]; then @@ -359,6 +517,19 @@ run_case() { grep -q '^PASS: ARC registration' "$scratch/stdout" || exit 1 else [[ "$code" != 0 ]] || { printf 'FAIL: accepted %s\n' "$name" >&2; exit 1; } + [[ "$code" -lt 91 || "$code" -gt 98 ]] || { + printf 'FAIL: %s failed with an unexpected fixture/transport exit %s\n' "$name" "$code" >&2 + exit 1 + } + local refusal marker + refusal=$(expected_refusal "$name") || exit 1 + marker="ARC acceptance: FAIL at $refusal" + [[ "$refusal" != cleanup ]] || marker='ARC acceptance: FAIL cleanup' + grep -Fxq "$marker" "$scratch/stderr" || { + printf 'FAIL: %s refused for an unrelated reason (expected %s, exit %s)\n' "$name" "$refusal" "$code" >&2 + cat "$scratch/stderr" >&2 + exit 1 + } case "$name" in replacement-race) [[ -e "$scratch/replacement-preserved" && ! -e "$scratch/deleted" && "$code" == 4 ]] || exit 1 ;; unknown-create) [[ -e "$scratch/live-pod" && ! -e "$scratch/deleted" && "$code" == 4 ]] || exit 1 ;; @@ -410,6 +581,28 @@ for name in retained-active-min retained-active-max retained-suspended retained- [[ ! -e "$scratch/runtime-access" && ! -e "$scratch/live-pod" ]] done run_case complete-proof pass +run_case orphan-completed pass +[[ -e "$scratch/prior-fence-deleted" && -e "$scratch/prior-probe-deleted" ]] +run_case orphan-empty pass +[[ -e "$scratch/prior-fence-deleted" && ! -e "$scratch/prior-probe-deleted" ]] +run_case orphan-paginated pass +[[ -e "$scratch/prior-fence-deleted" && -e "$scratch/prior-probe-deleted" ]] +for name in orphan-active orphan-api-unknown orphan-incomplete-jobs orphan-missing-provenance orphan-wrong-probe \ + orphan-replacement orphan-run-mismatch orphan-wrong-api-head orphan-wrong-api-workflow orphan-wrong-api-repository \ + orphan-duplicate-job orphan-job-active orphan-wrong-job orphan-quota-churn orphan-incomplete-probes orphan-node-api \ + orphan-missing-probe-uid orphan-current-attempt orphan-foreign-provenance orphan-wrong-quota orphan-controlled-probe orphan-wrong-deadline \ + orphan-current-empty-api orphan-current-job-list-empty orphan-unauthorized-current orphan-final-api-unknown; do + run_case "$name" fail + [[ ! -e "$scratch/prior-fence-deleted" ]] + case "$name" in + orphan-replacement|orphan-incomplete-probes|orphan-node-api|orphan-final-api-unknown) [[ -e "$scratch/prior-probe-deleted" ]] ;; + *) [[ ! -e "$scratch/prior-probe-deleted" ]] ;; + esac +done +for name in own-fence-origin-churn own-fence-controller-churn own-fence-deleting own-fence-empty; do + run_case "$name" fail + [[ ! -e "$scratch/live-pod" && ! -e "$scratch/fence-deleted" && -e "$scratch/admission-fence" ]] +done run_case busy-runner pass [[ -e "$scratch/job-completed" && -e "$scratch/fence-deleted" ]] run_case late-runner pass @@ -479,4 +672,11 @@ for name in no-registration unexpected-hook writable-metrics wrong-metrics-confi forwarded-flow wrong-source observer-loss observer-diagnostics failed-node-cleanup; do run_case "$name" fail done +# Prove that an unrelated API failure cannot stand in for an egress refusal. +sed 's/== live-api-error/== allowed-egress/' "$scratch/kubectl-original" >"$scratch/bin/kubectl" +if (run_case allowed-egress fail) >"$scratch/wrong-cause-out" 2>"$scratch/wrong-cause-error"; then + printf 'FAIL: an unrelated API error satisfied the egress negative control\n' >&2 + exit 1 +fi +cp "$scratch/kubectl-original" "$scratch/bin/kubectl" printf 'PASS: inactive and unauthorized sources never reach runtime access\n' diff --git a/scripts/verify-ksail-arc-runtime.sh b/scripts/verify-ksail-arc-runtime.sh index bc6b66b1f5..7d91e2a98d 100755 --- a/scripts/verify-ksail-arc-runtime.sh +++ b/scripts/verify-ksail-arc-runtime.sh @@ -73,11 +73,22 @@ wait_ready() { timeout 630s kubectl --context "$context" --request-timeout=610s -n "$namespace" \ wait "pod/$probe" --for=condition=Ready --timeout=600s 2>/dev/null } -quiet() { "$@" >/dev/null 2>"$scratch/command-error"; } +quiet() { + local code=0 + "$@" >/dev/null 2>"$scratch/command-error" || code=$? + if [[ "$code" != 0 ]]; then printf 'ARC acceptance: FAIL at %s\n' "$stage" >&2; fi + return "$code" +} +# shellcheck source=scripts/ksail-arc-admission-fence.sh +source scripts/ksail-arc-admission-fence.sh verify_admission_fence() { - kc -n "$namespace" get resourcequota "$fence_name" -o json >"$scratch/live-fence.json" - jq -e --arg name "$fence_name" --arg owner "$probe" --arg key "$ownership_label" --arg uid "$fence_uid" ' + kc -n "$namespace" get resourcequota "$fence_name" -o json >"$scratch/live-fence.json" || return 1 + [[ -s "$scratch/live-fence.json" ]] || return 1 + jq -e --arg name "$fence_name" --arg ns "$namespace" --arg owner "$probe" --arg key "$ownership_label" --arg uid "$fence_uid" \ + --arg origin_key "$arc_origin_key" --slurpfile origin "$scratch/current-origin.json" ' .metadata.name == $name and .metadata.labels[$key] == $owner and .metadata.uid == $uid and + .metadata.namespace == $ns and (.metadata.ownerReferences // [] | length) == 0 and + .metadata.annotations[$origin_key] == ($origin[0]|tojson) and .metadata.deletionTimestamp == null and .spec == {hard:{pods:"0"},scopes:["NotTerminating"]} and .status.hard == {pods:"0"} ' "$scratch/live-fence.json" >/dev/null @@ -159,17 +170,14 @@ cleanup() { if [[ -z "$fence_uid" ]]; then failed=1 else - fence=$(kc -n "$namespace" get resourcequota "$fence_name" --ignore-not-found -o json) || failed=1 - if ! jq -e --arg name "$fence_name" --arg owner "$probe" --arg key "$ownership_label" --arg uid "$fence_uid" ' - .metadata.name == $name and .metadata.labels[$key] == $owner and .metadata.uid == $uid and - .spec == {hard:{pods:"0"},scopes:["NotTerminating"]} and - (.metadata.resourceVersion | type == "string" and length > 0) - ' <<<"$fence" >/dev/null; then + if ! verify_admission_fence || ! jq -e ' + .metadata.resourceVersion | type == "string" and length > 0 + ' "$scratch/live-fence.json" >/dev/null; then failed=1 else jq '{apiVersion:"v1",kind:"DeleteOptions",preconditions:{uid:.metadata.uid, resourceVersion:.metadata.resourceVersion},propagationPolicy:"Background"}' \ - <<<"$fence" >"$scratch/delete-fence.json" + "$scratch/live-fence.json" >"$scratch/delete-fence.json" quiet kc delete --raw="/api/v1/namespaces/$namespace/resourcequotas/$fence_name" \ -f "$scratch/delete-fence.json" || failed=1 fence=$(kc -n "$namespace" get resourcequota "$fence_name" --ignore-not-found -o json) || failed=1 @@ -281,10 +289,16 @@ stage=runner-admission-fence # The bounded probe has activeDeadlineSeconds and is outside NotTerminating. # Source: https://kubernetes.io/docs/concepts/policy/resource-quotas/ verify_runner_deadlines +arc_current_origin +stage=runner-fence-recovery +arc_recover_admission_fence +stage=runner-admission-fence kc -n "$namespace" get resourcequotas -l "$ownership_label" -o json \ | jq -e '.items | length == 0' >/dev/null -jq -n --arg name "$fence_name" --arg owner "$probe" --arg ns "$namespace" --arg key "$ownership_label" ' - {apiVersion:"v1",kind:"ResourceQuota",metadata:{name:$name,namespace:$ns,labels:{($key):$owner}}, +jq -n --arg name "$fence_name" --arg owner "$probe" --arg ns "$namespace" --arg key "$ownership_label" \ + --arg origin_key "$arc_origin_key" --slurpfile origin "$scratch/current-origin.json" ' + {apiVersion:"v1",kind:"ResourceQuota",metadata:{name:$name,namespace:$ns,labels:{($key):$owner}, + annotations:{($origin_key):($origin[0]|tojson)}}, spec:{hard:{pods:"0"},scopes:["NotTerminating"]}}' >"$scratch/fence.json" fence_created=true kc create -f "$scratch/fence.json" -o json >"$scratch/created-fence.json" @@ -389,6 +403,7 @@ kc create -f "$scratch/pod.json" -o json >"$scratch/created-pod.json" probe_uid=$(jq -er --arg name "$probe" --arg ns "$namespace" --arg key "$ownership_label" \ 'select(.metadata.name == $name and .metadata.namespace == $ns and .metadata.labels[$key] == $name) | .metadata.uid | select(type == "string" and length > 0)' "$scratch/created-pod.json") +arc_record_probe_uid quiet wait_ready kc -n "$namespace" get pod "$probe" -o json >"$scratch/live-pod.json" jq -e --arg uid "$probe_uid" '.metadata.uid == $uid' "$scratch/live-pod.json" >/dev/null @@ -494,7 +509,7 @@ kc -n default get endpoints kubernetes -o json | jq -c '.subsets' >"$scratch/api jq -c '.subsets' "$scratch/api.json" >"$scratch/api-before" cmp -s "$scratch/api-before" "$scratch/api-after" || fail api-target-churn -stage=job-cgroup-measurement +stage='job-cgroup-measurement' quiet kc -n "$namespace" exec "$probe" -- /etc/ksail-arc-metrics/job-metrics.sh verify_admission_fence verify_runner_source From 1fac82c1f3472d9563fafecc0429e69452f4ffd9 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Fri, 9 Oct 2026 14:08:32 +0200 Subject: [PATCH 60/62] fix(arc): preserve bounded metrics and safe convergence diagnostics --- .../config-map-job-metrics.yaml | 24 +++++++-- .../runtime_test.go | 51 +++++++++++++++++++ scripts/ksail-arc-job-metrics.sh | 22 ++++++-- .../tests/test-verify-ksail-arc-runtime.sh | 18 ++++++- .../test-wait-for-ksail-arc-registration.sh | 29 +++++++++-- scripts/verify-ksail-arc-runtime.sh | 13 +++++ scripts/wait-for-ksail-arc-registration.sh | 9 ++++ 7 files changed, 153 insertions(+), 13 deletions(-) diff --git a/k8s/bases/infrastructure/actions-runners/config-map-job-metrics.yaml b/k8s/bases/infrastructure/actions-runners/config-map-job-metrics.yaml index 4ae45b7a5d..5c8cafe3d3 100644 --- a/k8s/bases/infrastructure/actions-runners/config-map-job-metrics.yaml +++ b/k8s/bases/infrastructure/actions-runners/config-map-job-metrics.yaml @@ -10,14 +10,25 @@ data: return 1 } + arc_metrics_read() { + local root=$1 name=$2 + if [[ "$root" == /sys/fs/cgroup ]]; then + timeout 5s cat "$root/$name" 2>/dev/null + else + # Only regular test-owned fixture files use the portable read path. + [[ -f "$root/$name" && ! -L "$root/$name" ]] || return 1 + cat "$root/$name" 2>/dev/null + fi + } + # The production entrypoint uses only the container's kernel cgroup. A supplied # fixture directory is used when this function is sourced by offline tests. arc_job_metrics() { local root=$1 limit peak events key value extra - local low='' high='' max='' oom='' oom_kill='' oom_group_kill='' - limit=$(timeout 5s cat "$root/memory.max" 2>/dev/null) || { arc_metrics_fail read-limit; return 1; } - peak=$(timeout 5s cat "$root/memory.peak" 2>/dev/null) || { arc_metrics_fail read-peak; return 1; } - events=$(timeout 5s cat "$root/memory.events" 2>/dev/null) || { arc_metrics_fail read-events; return 1; } + local low='' high='' max='' oom='' oom_kill='' oom_group_kill='' sock_throttled='' + limit=$(arc_metrics_read "$root" memory.max) || { arc_metrics_fail read-limit; return 1; } + peak=$(arc_metrics_read "$root" memory.peak) || { arc_metrics_fail read-peak; return 1; } + events=$(arc_metrics_read "$root" memory.events) || { arc_metrics_fail read-events; return 1; } [[ "$limit" == 15032385536 ]] || { arc_metrics_fail limit; return 1; } [[ "$peak" =~ ^(0|[1-9][0-9]{0,17})$ && "$peak" -gt 0 && "$peak" -le "$limit" ]] || { arc_metrics_fail peak; return 1; @@ -28,6 +39,9 @@ data: low|high|max|oom|oom_kill|oom_group_kill) [[ -z "${!key}" ]] || { arc_metrics_fail duplicate-event; return 1; } printf -v "$key" '%s' "$value" ;; + sock_throttled) + [[ -z "$sock_throttled" ]] || { arc_metrics_fail duplicate-event; return 1; } + sock_throttled=$value ;; *) arc_metrics_fail unknown-event; return 1 ;; esac done <<<"$events" @@ -50,5 +64,5 @@ metadata: kustomize.toolkit.fluxcd.io/substitute: disabled labels: app.kubernetes.io/name: ksail-arc-job-metrics - name: ksail-arc-job-metrics-d6e7291092 + name: ksail-arc-job-metrics-6f02a2616f namespace: arc-runners diff --git a/scripts/generate-ksail-arc-job-metrics/runtime_test.go b/scripts/generate-ksail-arc-job-metrics/runtime_test.go index a72513aa93..057aebd4f7 100644 --- a/scripts/generate-ksail-arc-job-metrics/runtime_test.go +++ b/scripts/generate-ksail-arc-job-metrics/runtime_test.go @@ -26,6 +26,11 @@ func TestWholeJobCgroupMeasurement(t *testing.T) { ok bool }{ {"complete", "15032385536", "9876543210", "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\noom_group_kill 0\n", true}, + {"socket throttled zero", "15032385536", "9876543210", "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0\n", true}, + {"socket throttled nonzero", "15032385536", "9876543210", "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 2\n", true}, + {"duplicate socket counter", "15032385536", "10", "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0\nsock_throttled 0\n", false}, + {"malformed socket counter", "15032385536", "10", "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled nope\n", false}, + {"socket counter cannot replace required event", "15032385536", "10", "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\nsock_throttled 0\n", false}, {"missing limit", "", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, {"unbounded", "max", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, {"wrong cgroup", "17179869184", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, @@ -94,3 +99,49 @@ func TestWholeJobCgroupMeasurement(t *testing.T) { }) } } + +func TestMetricsFixturesArePortableAndProductionReadsRemainBounded(t *testing.T) { + root := t.TempDir() + for name, value := range map[string]string{ + "memory.max": "15032385536\n", "memory.peak": "9876543210\n", + "memory.events": "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\noom_group_kill 0\n", + } { + if err := os.WriteFile(filepath.Join(root, name), []byte(value), 0600); err != nil { + t.Fatal(err) + } + } + bin := t.TempDir() + log := filepath.Join(bin, "timeout-invocations") + if err := os.WriteFile(filepath.Join(bin, "timeout"), []byte("#!/bin/bash\nprintf '%s\\n' \"$*\" >> \"$TIMEOUT_LOG\"\nexit 124\n"), 0700); err != nil { + t.Fatal(err) + } + script, err := filepath.Abs(filepath.Join(repoRoot, metricsScript)) + if err != nil { + t.Fatal(err) + } + for _, production := range []bool{false, true} { + cgroup := root + if production { + cgroup = "/sys/fs/cgroup" + } + cmd := exec.Command("bash", "-c", `source "$1"; arc_job_metrics "$2"`, "metrics-test", script, cgroup) + cmd.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH"), "TIMEOUT_LOG="+log) + output, err := cmd.CombinedOutput() + if production { + if err == nil || strings.Contains(string(output), "KSail ARC whole-job cgroup:") { + t.Fatalf("unbounded production receipt: %s", output) + } + calls, readErr := os.ReadFile(log) + if readErr != nil || string(calls) != "5s cat /sys/fs/cgroup/memory.max\n" { + t.Fatalf("production timeout not used: %q, %v", calls, readErr) + } + } else { + if err != nil || !strings.Contains(string(output), "KSail ARC whole-job cgroup:") { + t.Fatalf("fixture required GNU timeout: %s", output) + } + if _, err := os.Stat(log); !os.IsNotExist(err) { + t.Fatalf("fixture invoked production timeout: %v", err) + } + } + } +} diff --git a/scripts/ksail-arc-job-metrics.sh b/scripts/ksail-arc-job-metrics.sh index f158d1121e..2aa0c64921 100755 --- a/scripts/ksail-arc-job-metrics.sh +++ b/scripts/ksail-arc-job-metrics.sh @@ -6,14 +6,25 @@ arc_metrics_fail() { return 1 } +arc_metrics_read() { + local root=$1 name=$2 + if [[ "$root" == /sys/fs/cgroup ]]; then + timeout 5s cat "$root/$name" 2>/dev/null + else + # Only regular test-owned fixture files use the portable read path. + [[ -f "$root/$name" && ! -L "$root/$name" ]] || return 1 + cat "$root/$name" 2>/dev/null + fi +} + # The production entrypoint uses only the container's kernel cgroup. A supplied # fixture directory is used when this function is sourced by offline tests. arc_job_metrics() { local root=$1 limit peak events key value extra - local low='' high='' max='' oom='' oom_kill='' oom_group_kill='' - limit=$(timeout 5s cat "$root/memory.max" 2>/dev/null) || { arc_metrics_fail read-limit; return 1; } - peak=$(timeout 5s cat "$root/memory.peak" 2>/dev/null) || { arc_metrics_fail read-peak; return 1; } - events=$(timeout 5s cat "$root/memory.events" 2>/dev/null) || { arc_metrics_fail read-events; return 1; } + local low='' high='' max='' oom='' oom_kill='' oom_group_kill='' sock_throttled='' + limit=$(arc_metrics_read "$root" memory.max) || { arc_metrics_fail read-limit; return 1; } + peak=$(arc_metrics_read "$root" memory.peak) || { arc_metrics_fail read-peak; return 1; } + events=$(arc_metrics_read "$root" memory.events) || { arc_metrics_fail read-events; return 1; } [[ "$limit" == 15032385536 ]] || { arc_metrics_fail limit; return 1; } [[ "$peak" =~ ^(0|[1-9][0-9]{0,17})$ && "$peak" -gt 0 && "$peak" -le "$limit" ]] || { arc_metrics_fail peak; return 1; @@ -24,6 +35,9 @@ arc_job_metrics() { low|high|max|oom|oom_kill|oom_group_kill) [[ -z "${!key}" ]] || { arc_metrics_fail duplicate-event; return 1; } printf -v "$key" '%s' "$value" ;; + sock_throttled) + [[ -z "$sock_throttled" ]] || { arc_metrics_fail duplicate-event; return 1; } + sock_throttled=$value ;; *) arc_metrics_fail unknown-event; return 1 ;; esac done <<<"$events" diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh index 1db409cb5b..638633ffae 100755 --- a/scripts/tests/test-verify-ksail-arc-runtime.sh +++ b/scripts/tests/test-verify-ksail-arc-runtime.sh @@ -47,6 +47,12 @@ SH cat >"$scratch/bin/timeout" <<'SH' #!/usr/bin/env bash export ARC_TEST_TIMEOUT=$1 +if [[ "$1" == 660s && "$ARC_TEST_CASE" == registration-diagnostic-redaction ]]; then + printf 'ARC registration pending: runner-scale-set\n' >&2 + printf 'ARC registration pending: private-registration-error-canary\n' >&2 + printf 'private-registration-error-canary\n' >&2 + exit 124 +fi if [[ "$1" == 660s ]]; then touch "$ARC_TEST_ROOT/registration-budget"; fi shift exec "$@" @@ -448,7 +454,7 @@ expected_refusal() { invalid-runner-identity|unexpected-hook|writable-metrics|wrong-metrics-configmap|runner-deadline) printf 'registration-and-bounds' ;; drift-during-probe|failed-metrics-hook) printf 'job-cgroup-measurement' ;; live-api-error|mutable-metrics|tampered-metrics) printf 'immutable-job-metrics' ;; - registration-timeout|stale-flux|no-registration) printf 'registration-convergence' ;; + registration-timeout|stale-flux|no-registration|registration-diagnostic-redaction) printf 'registration-convergence' ;; rendered-hidden-activation) printf 'partial-activation' ;; source-render-failure) printf 'source-render' ;; rendered-suspended|rendered-wrong-image) printf 'rendered-source-state' ;; @@ -640,7 +646,17 @@ run_case delayed-registration pass for name in registration-timeout stale-flux; do run_case "$name" fail [[ -e "$scratch/registration-budget" && ! -e "$scratch/live-pod" && ! -e "$scratch/deleted" ]] + if [[ "$name" == registration-timeout ]]; then label=runner-scale-set + else label=flux-infrastructure; fi + grep -Fxq "ARC registration pending: $label" "$scratch/stderr" || { + printf 'FAIL: safe registration diagnostic was suppressed\n' >&2; exit 1; + } done +run_case registration-diagnostic-redaction fail +grep -Fxq 'ARC registration pending: runner-scale-set' "$scratch/stderr" || exit 1 +if grep -Fq 'private-registration-error-canary' "$scratch/stdout" "$scratch/stderr"; then + printf 'FAIL: raw or unrecognized registration diagnostic escaped\n' >&2; exit 1 +fi cp "$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" "$scratch/controller-aggregate" cp "$scratch/k8s/providers/hetzner/infrastructure/kustomization.yaml" "$scratch/runner-aggregate" sed 's/^resources:/bases:/' "$scratch/controller-aggregate" >"$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" diff --git a/scripts/tests/test-wait-for-ksail-arc-registration.sh b/scripts/tests/test-wait-for-ksail-arc-registration.sh index a8f24b1473..85133542a6 100755 --- a/scripts/tests/test-wait-for-ksail-arc-registration.sh +++ b/scripts/tests/test-wait-for-ksail-arc-registration.sh @@ -19,7 +19,7 @@ set -euo pipefail [[ "$*" == *'--context admin@prod'* ]] || exit 92 case "$*" in *'get providerconfigs.github.m.upbound.io --all-namespaces '*) - [[ "$GROUP_CASE" != unreadable-provider ]] || exit 1 + [[ "$GROUP_CASE" != unreadable-provider ]] || { printf 'private-registration-error-canary\n' >&2; exit 1; } jq -cn --arg scenario "$GROUP_CASE" '{items:[{ apiVersion:"github.m.upbound.io/v1beta1",kind:"ProviderConfig", metadata:{name:"arc-runtime-platform-app",namespace:"arc-runners"}, @@ -33,6 +33,11 @@ case "$*" in *'get providerconfigs.github.upbound.io '*) jq -cn --arg scenario "$GROUP_CASE" '{items:(if $scenario=="legacy-provider-collision" then [{metadata:{name:"arc-runtime-platform-app"}}] else [] end)}' ;; *'get kustomization '*) + if [[ ( "$GROUP_CASE" == unready-infrastructure && "$*" == *'get kustomization infrastructure '* ) || + ( "$GROUP_CASE" == unready-apps && "$*" == *'get kustomization apps '* ) ]]; then + printf '{"metadata":{"generation":2},"status":{"observedGeneration":1,"conditions":[]}}' + exit 0 + fi jq -cn --arg revision "latest@$GROUP_DIGEST" '{metadata:{generation:1},status:{observedGeneration:1,lastAppliedRevision:$revision,conditions:[{type:"Ready",status:"True"}]}}' ;; *'get runnergroups.actions.github.m.upbound.io platform '*) [[ "$GROUP_CASE" != unreadable-group ]] || exit 1 @@ -70,12 +75,30 @@ chmod 700 "$scratch/bin/"* for scenario in normal all-repositories extra-repository default-group inherited-group wrong-remote-org \ stale-observation missing-observation unbound-id unreadable-group wrong-registered-group \ unrestricted-workflows extra-workflow wrong-workflow-ref unrestricted-desired \ - duplicate-provider wrong-provider-secret cluster-provider-collision legacy-provider-collision unreadable-provider; do + duplicate-provider wrong-provider-secret cluster-provider-collision legacy-provider-collision unreadable-provider \ + unready-infrastructure unready-apps; do if PATH="$scratch/bin:$PATH" GROUP_DIGEST="sha256:$(printf 'a%.0s' {1..64})" GROUP_CASE="$scenario" \ GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform \ bash "$root/scripts/wait-for-ksail-arc-registration.sh" "sha256:$(printf 'a%.0s' {1..64})" \ >"$scratch/out" 2>"$scratch/err"; then [[ "$scenario" == normal ]] || { echo "accepted $scenario"; exit 1; } - else [[ "$scenario" != normal ]] || { cat "$scratch/err"; exit 1; }; fi + else + [[ "$scenario" != normal ]] || { cat "$scratch/err"; exit 1; } + case "$scenario" in + duplicate-provider|wrong-provider-secret|unreadable-provider) label=provider-config ;; + cluster-provider-collision) label=provider-collision-cluster ;; + legacy-provider-collision) label=provider-collision-legacy ;; + wrong-registered-group) label=runner-scale-set ;; + unready-infrastructure) label=flux-infrastructure ;; + unready-apps) label=flux-apps ;; + *) label=runner-group ;; + esac + grep -Fxq "ARC registration pending: $label" "$scratch/err" || { + printf 'FAIL: missing safe diagnostic for %s\n' "$scenario" >&2; exit 1; + } + fi + if grep -Fq 'private-registration-error-canary' "$scratch/out" "$scratch/err"; then + printf 'FAIL: raw registration error escaped\n' >&2; exit 1 + fi done printf 'PASS: organization registration requires an observed KSail-only group\n' diff --git a/scripts/verify-ksail-arc-runtime.sh b/scripts/verify-ksail-arc-runtime.sh index 7d91e2a98d..d96f8c89d3 100755 --- a/scripts/verify-ksail-arc-runtime.sh +++ b/scripts/verify-ksail-arc-runtime.sh @@ -199,6 +199,19 @@ quiet bash scripts/wait-for-platform-flux-revision.sh "$PLATFORM_MANIFEST_DIGEST # revision/generation join before taking the immutable acceptance snapshots. stage=registration-convergence if ! quiet timeout 660s bash scripts/wait-for-ksail-arc-registration.sh "$PLATFORM_MANIFEST_DIGEST"; then + # Preserve fixed check labels even when the outer timeout stops the waiter. + # Captured command diagnostics may contain private cluster values. + while IFS= read -r line; do + case "$line" in + 'ARC registration pending: flux-infrastructure'|\ + 'ARC registration pending: flux-apps'|\ + 'ARC registration pending: provider-config'|\ + 'ARC registration pending: provider-collision-cluster'|\ + 'ARC registration pending: provider-collision-legacy'|\ + 'ARC registration pending: runner-group'|\ + 'ARC registration pending: runner-scale-set') printf '%s\n' "$line" >&2 ;; + esac + done <"$scratch/command-error" fail registration-convergence fi for layer in infrastructure apps; do diff --git a/scripts/wait-for-ksail-arc-registration.sh b/scripts/wait-for-ksail-arc-registration.sh index 511651c82b..e45facd265 100755 --- a/scripts/wait-for-ksail-arc-registration.sh +++ b/scripts/wait-for-ksail-arc-registration.sh @@ -6,6 +6,7 @@ set -euo pipefail readonly revision="latest@$1" readonly deadline=$((SECONDS + 600)) kc() { timeout 35s kubectl --context admin@prod --request-timeout=30s "$@" 2>/dev/null; } +pending() { printf 'ARC registration pending: %s\n' "$1" >&2; } while ((SECONDS < deadline)); do ready=true @@ -17,6 +18,7 @@ while ((SECONDS < deadline)); do any(.status.conditions[]; .type == "Ready" and .status == "True") ' <<<"$state" >/dev/null 2>&1; then ready=false + pending "flux-$layer" fi done # The pinned provider caches by reference name, across namespaces and kinds. @@ -33,12 +35,17 @@ while ((SECONDS < deadline)); do "namespace":"arc-runners","name":"arc-github-app","key":"provider-credentials"}} ' <<<"$state" >/dev/null 2>&1; then ready=false + pending provider-config fi for resource in clusterproviderconfigs.github.m.upbound.io providerconfigs.github.upbound.io; do if ! state=$(kc get "$resource" -o json) || ! jq -e '[.items[] | select(.metadata.name == "arc-runtime-platform-app")] | length == 0' \ <<<"$state" >/dev/null 2>&1; then ready=false + case "$resource" in + clusterproviderconfigs.github.m.upbound.io) pending provider-collision-cluster ;; + providerconfigs.github.upbound.io) pending provider-collision-legacy ;; + esac fi done # The provider observation comes from the existing organization's App. @@ -71,6 +78,7 @@ while ((SECONDS < deadline)); do any(.status.conditions[]; .type == "Synced" and .status == "True" and .observedGeneration == $generation) ' <<<"$state" >/dev/null 2>&1; then ready=false + pending runner-group fi if ! state=$(kc -n arc-runners get autoscalingrunnerset.actions.github.com platform-linux -o json) || ! jq -e ' @@ -82,6 +90,7 @@ while ((SECONDS < deadline)); do .metadata.annotations["actions.github.com/runner-scale-set-name"] == "platform-linux" ' <<<"$state" >/dev/null 2>&1; then ready=false + pending runner-scale-set fi if "$ready"; then printf 'ARC layers and registration converged at the deployed revision\n' From 1451f9cf5bc17569b45effa821658e8bf5896ab8 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Fri, 9 Oct 2026 14:47:13 +0200 Subject: [PATCH 61/62] fix(arc): renew audited metrics volume reference approval --- .../approved-surface-history.md | 28 +++++++++++++++++++ .../approved-surface.txt | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/scripts/validate-eks-ci-role-policy/approved-surface-history.md b/scripts/validate-eks-ci-role-policy/approved-surface-history.md index 2e5631b777..b25557873b 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface-history.md +++ b/scripts/validate-eks-ci-role-policy/approved-surface-history.md @@ -3322,3 +3322,31 @@ devantler-tech/ksail could list usage but not the limit. These values were computed locally with kubectl v1.37.1 / Kustomize v5.8.1, which is not the approved renderer; the required CI job re-renders with the approved kubectl v1.36.2 / Kustomize v5.8.1 and is the authority. + +## 2026-10-09 — ARC metrics source repair retains its authorization boundary + +The checksum-verified kubectl v1.36.2 / Kustomize v5.8.1 renderer reproduces +exactly one changed authorization entry for #4544 at `d82bcd49`, compared with +its preceding reviewed source at `b0a77dd4`: HelmRelease +arc-runners/platform-runners. Its fingerprint moves from +`0dd16e9d373fe4e3aa26d6b0fc3e31d4d977cf623acfc0e172b8c5b99033a95f` to +`52f89508024a6a9d9547f85a1f292b6fa05819f679fed5f5e3ee3972cd6eb388`. + +All five production layers contain 686 rendered documents before and after. +The generated metrics ConfigMap changes name from +`ksail-arc-job-metrics-d6e7291092` to `ksail-arc-job-metrics-6f02a2616f`; the +HelmRelease's corresponding ConfigMap volume reference is its only scalar +change. Every other complete rendered document is equal. Integrating main at +`ce0be361` adds no rendered change. The metrics script repair makes regular +offline fixture reads portable and accepts the optional kernel counter while +retaining bounded production reads and every mandatory memory/OOM counter. +No IAM/RBAC grant, App permission, runner identity, image, resource bound or +security setting changes. + +The old entry fails with precisely this mismatch. Renewing only this entry +passes the complete authorization package (74 tests and 253 subtests, no +skips) and its CLI. Negative controls retain rejection of expanded grants, +missing, duplicated or newly added authorization resources, changed Flux +handoff and unauthorized AWS identity bindings. Local validation uses cached +Go 1.26.9; required native CI uses the repository's Go 1.26.6 pin and remains +the authority. This approval establishes no runtime or activation clearance. diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index 06edf812cc..38c18b6643 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -16,7 +16,7 @@ helm.toolkit.fluxcd.io/v2|HelmRelease|actual-budget|actual-budget adc009b64f2a9d helm.toolkit.fluxcd.io/v2|HelmRelease|arc-ksail-analysis|ksail-analysis-runners f59980a0d056bb796135803ce4cfb1a84947957e943abe9854a78eaf1249b2b0 -helm.toolkit.fluxcd.io/v2|HelmRelease|arc-runners|platform-runners 0dd16e9d373fe4e3aa26d6b0fc3e31d4d977cf623acfc0e172b8c5b99033a95f +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-runners|platform-runners 52f89508024a6a9d9547f85a1f292b6fa05819f679fed5f5e3ee3972cd6eb388 helm.toolkit.fluxcd.io/v2|HelmRelease|arc-systems|arc-controller 844e32e3408c2ad6e85b10c6628de7208c447ca3f1cffe5a0da74dcf3cdc11cf From 78551d20ce719c3fea6f3c40f84d1866f716156c Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Fri, 9 Oct 2026 15:52:36 +0200 Subject: [PATCH 62/62] fix(arc): retain rejected over-budget memory measurements --- .../config-map-job-metrics.yaml | 7 +++-- .../hetzner/infrastructure/kustomization.yaml | 2 +- .../runtime_test.go | 28 ++++++++++++++++++- scripts/ksail-arc-job-metrics.sh | 5 +++- scripts/tests/arc-staging/job_metrics_test.go | 1 - .../approved-surface-history.md | 25 +++++++++++++++++ .../approved-surface.txt | 2 +- .../verify-ksail-arc-runtime/budget_test.go | 25 +++++++++-------- 8 files changed, 77 insertions(+), 18 deletions(-) diff --git a/k8s/bases/infrastructure/actions-runners/config-map-job-metrics.yaml b/k8s/bases/infrastructure/actions-runners/config-map-job-metrics.yaml index 5c8cafe3d3..696d72a660 100644 --- a/k8s/bases/infrastructure/actions-runners/config-map-job-metrics.yaml +++ b/k8s/bases/infrastructure/actions-runners/config-map-job-metrics.yaml @@ -30,7 +30,7 @@ data: peak=$(arc_metrics_read "$root" memory.peak) || { arc_metrics_fail read-peak; return 1; } events=$(arc_metrics_read "$root" memory.events) || { arc_metrics_fail read-events; return 1; } [[ "$limit" == 15032385536 ]] || { arc_metrics_fail limit; return 1; } - [[ "$peak" =~ ^(0|[1-9][0-9]{0,17})$ && "$peak" -gt 0 && "$peak" -le "$limit" ]] || { + [[ "$peak" =~ ^(0|[1-9][0-9]{0,17})$ && "$peak" -gt 0 ]] || { arc_metrics_fail peak; return 1; } while read -r key value extra; do @@ -51,6 +51,9 @@ data: [[ "$oom" == 0 && "$oom_kill" == 0 && "$oom_group_kill" == 0 ]] || { arc_metrics_fail oom; return 1; } printf 'KSail ARC whole-job cgroup: {"schemaVersion":1,"memoryMaxBytes":%s,"memoryPeakBytes":%s,"limitEvents":%s,"oomEvents":%s,"oomKills":%s,"oomGroupKills":%s}\n' \ "$limit" "$peak" "$max" "$oom" "$oom_kill" "$oom_group_kill" + # The kernel may temporarily exceed memory.max. Retain that valid measurement, + # while preserving the existing runtime acceptance ceiling as a separate refusal. + [[ "$peak" -le "$limit" ]] || { arc_metrics_fail budget; return 1; } } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then @@ -64,5 +67,5 @@ metadata: kustomize.toolkit.fluxcd.io/substitute: disabled labels: app.kubernetes.io/name: ksail-arc-job-metrics - name: ksail-arc-job-metrics-6f02a2616f + name: ksail-arc-job-metrics-2fbcf0b1d7 namespace: arc-runners diff --git a/k8s/providers/hetzner/infrastructure/kustomization.yaml b/k8s/providers/hetzner/infrastructure/kustomization.yaml index 4859ef03ec..c3db65cb6d 100644 --- a/k8s/providers/hetzner/infrastructure/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/kustomization.yaml @@ -3,7 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../../../bases/infrastructure/ - # Prod-only: one bounded KSail analysis runner, authenticated by the shared App. + # Prod-only: the organization runner pool (max one runner), authenticated by the shared App. - ../../../bases/infrastructure/actions-runners/ # Re-adopt the retained release without activating a runner. Suspension does # not drain an installed scale set; runtime acceptance remains a separate gate. diff --git a/scripts/generate-ksail-arc-job-metrics/runtime_test.go b/scripts/generate-ksail-arc-job-metrics/runtime_test.go index 057aebd4f7..d7762f7338 100644 --- a/scripts/generate-ksail-arc-job-metrics/runtime_test.go +++ b/scripts/generate-ksail-arc-job-metrics/runtime_test.go @@ -36,7 +36,6 @@ func TestWholeJobCgroupMeasurement(t *testing.T) { {"wrong cgroup", "17179869184", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, {"missing peak", "15032385536", "", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, {"zero peak", "15032385536", "0", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, - {"exceeded", "15032385536", "15032385537", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, {"malformed peak", "15032385536", "1\n2", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, {"noncanonical peak", "15032385536", "001", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, {"overflow", "15032385536", "99999999999999999999999", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, @@ -49,6 +48,8 @@ func TestWholeJobCgroupMeasurement(t *testing.T) { {"OOM", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom 1\noom_kill 0\noom_group_kill 0\n", false}, {"OOM kill", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 1\noom_group_kill 0\n", false}, {"OOM group kill", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 1\n", false}, + {"over budget with OOM", "15032385536", "15032385537", "low 0\nhigh 0\nmax 3\noom 1\noom_kill 0\noom_group_kill 0\n", false}, + {"over budget with invalid events", "15032385536", "15032385537", "low 0\nhigh 0\nmax 3\noom nope\noom_kill 0\noom_group_kill 0\n", false}, } { t.Run(tc.name, func(t *testing.T) { root := t.TempDir() @@ -100,6 +101,31 @@ func TestWholeJobCgroupMeasurement(t *testing.T) { } } +func TestOverBudgetPeakRetainsMeasurementAndRejectsAcceptance(t *testing.T) { + root := t.TempDir() + for name, value := range map[string]string{ + "memory.max": "15032385536\n", "memory.peak": "15032385537\n", + "memory.events": "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\noom_group_kill 0\n", + } { + if err := os.WriteFile(filepath.Join(root, name), []byte(value), 0600); err != nil { + t.Fatal(err) + } + } + script, err := filepath.Abs(filepath.Join(repoRoot, metricsScript)) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("bash", "-c", `source "$1"; arc_job_metrics "$2"`, "metrics-test", script, root) + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatal("over-budget measurement accepted") + } + const receipt = `KSail ARC whole-job cgroup: {"schemaVersion":1,"memoryMaxBytes":15032385536,"memoryPeakBytes":15032385537,"limitEvents":3,"oomEvents":0,"oomKills":0,"oomGroupKills":0}` + if string(output) != receipt+"\n::error::KSail ARC whole-job cgroup measurement failed: budget\n" { + t.Fatalf("lost measurement or budget refusal: %s", output) + } +} + func TestMetricsFixturesArePortableAndProductionReadsRemainBounded(t *testing.T) { root := t.TempDir() for name, value := range map[string]string{ diff --git a/scripts/ksail-arc-job-metrics.sh b/scripts/ksail-arc-job-metrics.sh index 2aa0c64921..4a09f7d6e8 100755 --- a/scripts/ksail-arc-job-metrics.sh +++ b/scripts/ksail-arc-job-metrics.sh @@ -26,7 +26,7 @@ arc_job_metrics() { peak=$(arc_metrics_read "$root" memory.peak) || { arc_metrics_fail read-peak; return 1; } events=$(arc_metrics_read "$root" memory.events) || { arc_metrics_fail read-events; return 1; } [[ "$limit" == 15032385536 ]] || { arc_metrics_fail limit; return 1; } - [[ "$peak" =~ ^(0|[1-9][0-9]{0,17})$ && "$peak" -gt 0 && "$peak" -le "$limit" ]] || { + [[ "$peak" =~ ^(0|[1-9][0-9]{0,17})$ && "$peak" -gt 0 ]] || { arc_metrics_fail peak; return 1; } while read -r key value extra; do @@ -47,6 +47,9 @@ arc_job_metrics() { [[ "$oom" == 0 && "$oom_kill" == 0 && "$oom_group_kill" == 0 ]] || { arc_metrics_fail oom; return 1; } printf 'KSail ARC whole-job cgroup: {"schemaVersion":1,"memoryMaxBytes":%s,"memoryPeakBytes":%s,"limitEvents":%s,"oomEvents":%s,"oomKills":%s,"oomGroupKills":%s}\n' \ "$limit" "$peak" "$max" "$oom" "$oom_kill" "$oom_group_kill" + # The kernel may temporarily exceed memory.max. Retain that valid measurement, + # while preserving the existing runtime acceptance ceiling as a separate refusal. + [[ "$peak" -le "$limit" ]] || { arc_metrics_fail budget; return 1; } } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then diff --git a/scripts/tests/arc-staging/job_metrics_test.go b/scripts/tests/arc-staging/job_metrics_test.go index ebb9c07243..3e1804eb9d 100644 --- a/scripts/tests/arc-staging/job_metrics_test.go +++ b/scripts/tests/arc-staging/job_metrics_test.go @@ -3,7 +3,6 @@ package arcstaging_test import "testing" const metricsComponent = "k8s/bases/infrastructure/actions-runners" -const metricsScript = "scripts/ksail-arc-job-metrics.sh" func TestRunnerRetainsWholeJobCgroupMeasurement(t *testing.T) { config := readYAML(t, metricsComponent+"/config-map-job-metrics.yaml") diff --git a/scripts/validate-eks-ci-role-policy/approved-surface-history.md b/scripts/validate-eks-ci-role-policy/approved-surface-history.md index b25557873b..8a9b7e6821 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface-history.md +++ b/scripts/validate-eks-ci-role-policy/approved-surface-history.md @@ -3350,3 +3350,28 @@ missing, duplicated or newly added authorization resources, changed Flux handoff and unauthorized AWS identity bindings. Local validation uses cached Go 1.26.9; required native CI uses the repository's Go 1.26.6 pin and remains the authority. This approval establishes no runtime or activation clearance. + +## 2026-10-09 — ARC over-budget measurements remain visible without accepting them + +The approved kubectl v1.36.2 / Kustomize v5.8.1 renderer compares all five +production layers against #4544 at `1451f9cf`. Both renders contain 686 +documents, with no duplicate identities. Only the generated metrics ConfigMap +and its HelmRelease volume reference change. The ConfigMap name moves from +`ksail-arc-job-metrics-6f02a2616f` to `ksail-arc-job-metrics-2fbcf0b1d7`. +Removing that single reference scalar makes the complete HelmRelease equal; +every other complete rendered document is equal. + +The script now emits a canonical measurement for a transient peak above the +kernel limit, then returns the existing budget failure with a nonzero exit. +The 14 GiB limit, positive numeric peak, complete event counters and zero-OOM +requirements remain enforced. Over-budget measurements with invalid events or +OOM events still emit no receipt. Runtime acceptance continues to reject an +over-budget peak. No IAM/RBAC grant, App permission, runner identity, image, +resource bound or security setting changes. + +The validator reproduces only the pool entry's fingerprint change from +`52f89508024a6a9d9547f85a1f292b6fa05819f679fed5f5e3ee3972cd6eb388` to +`442c4bc77b77c3a927b1c479592d6960d535627050bf408ce55e794aa3ec505a`. +Renew only this entry; all authorization checks and negative controls remain +enabled. Local validation uses cached Go 1.26.9; native CI's repository pin +remains authoritative. This audit grants no runtime or activation clearance. diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index 38c18b6643..121cbdf50e 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -16,7 +16,7 @@ helm.toolkit.fluxcd.io/v2|HelmRelease|actual-budget|actual-budget adc009b64f2a9d helm.toolkit.fluxcd.io/v2|HelmRelease|arc-ksail-analysis|ksail-analysis-runners f59980a0d056bb796135803ce4cfb1a84947957e943abe9854a78eaf1249b2b0 -helm.toolkit.fluxcd.io/v2|HelmRelease|arc-runners|platform-runners 52f89508024a6a9d9547f85a1f292b6fa05819f679fed5f5e3ee3972cd6eb388 +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-runners|platform-runners 442c4bc77b77c3a927b1c479592d6960d535627050bf408ce55e794aa3ec505a helm.toolkit.fluxcd.io/v2|HelmRelease|arc-systems|arc-controller 844e32e3408c2ad6e85b10c6628de7208c447ca3f1cffe5a0da74dcf3cdc11cf diff --git a/scripts/verify-ksail-arc-runtime/budget_test.go b/scripts/verify-ksail-arc-runtime/budget_test.go index 67e7d6a994..e94fba5e64 100644 --- a/scripts/verify-ksail-arc-runtime/budget_test.go +++ b/scripts/verify-ksail-arc-runtime/budget_test.go @@ -35,17 +35,20 @@ func TestQuotaRequiresCompleteHeadroom(t *testing.T) { if err := verifyQuota([]byte(`{"items":[]}`)); err != nil { t.Fatal(err) } - for _, bad := range []string{ - strings.ReplaceAll(good, `"pods":"3"`, `"pods":"1"`), - strings.ReplaceAll(good, `"limits.memory":"16Gi"`, `"limits.memory":"14Gi"`), - strings.ReplaceAll(good, `"requests.memory":"1Gi",`, ``), - `{"items":[{"spec":{"hard":{"pods":"1"}},"status":{}}]}`, - `{"items":[{"spec":{"hard":{"pods":"1"}},"status":{"hard":{"pods":"5"},"used":{"pods":"1"}}}]}`, - `{"items":[{"spec":{"hard":{"pods":"1"},"scopes":["Terminating"]},"status":{"hard":{"pods":"1"},"used":{"pods":"0"}}}]}`, - `{}`, `{"items":[{"status":{"hard":{"pods":"unknown"},"used":{"pods":"0"}}}]}`, + for _, tc := range []struct{ name, input, want string }{ + {"pods", strings.ReplaceAll(good, `"pods":"3"`, `"pods":"1"`), "insufficient namespace quota"}, + {"memory", strings.ReplaceAll(good, `"limits.memory":"16Gi"`, `"limits.memory":"14Gi"`), "insufficient namespace quota"}, + {"used accounting", strings.ReplaceAll(good, `"requests.memory":"1Gi",`, ``), "incomplete quota accounting"}, + {"limit accounting", `{"items":[{"spec":{"hard":{"pods":"1"}},"status":{}}]}`, "missing quota limit accounting"}, + {"stale accounting", `{"items":[{"spec":{"hard":{"pods":"1"}},"status":{"hard":{"pods":"5"},"used":{"pods":"1"}}}]}`, "stale quota limit accounting"}, + {"scoped", `{"items":[{"spec":{"hard":{"pods":"1"},"scopes":["Terminating"]},"status":{"hard":{"pods":"1"},"used":{"pods":"0"}}}]}`, "unresolved scoped quota"}, + {"missing read", `{}`, "invalid quota read"}, + {"unknown quantity", `{"items":[{"spec":{"hard":{"pods":"unknown"}},"status":{"hard":{"pods":"unknown"},"used":{"pods":"0"}}}]}`, "unknown resource quantity"}, } { - if verifyQuota([]byte(bad)) == nil { - t.Fatal("accepted incomplete or insufficient quota") - } + t.Run(tc.name, func(t *testing.T) { + if err := verifyQuota([]byte(tc.input)); err == nil || err.Error() != tc.want { + t.Fatalf("got %v, want %q", err, tc.want) + } + }) } }