diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index c0a7f9dd8b..e806012e29 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -593,6 +593,22 @@ runs: shell: bash run: ./scripts/verify-gateway-routes-applied.sh + - name: Verify the deployed KSail ARC canary + if: >- + !cancelled() && + steps.wait_flux_revision.outcome == 'success' && + ( + steps.cluster_update.outcome == 'skipped' || + steps.wait_prod_api_stability.outcome == 'success' + ) + shell: bash + env: + GH_TOKEN: ${{ github.token }} + PLATFORM_MANIFEST_DIGEST: ${{ steps.publish_platform_manifest.outputs.digest }} + run: | + export KUBECONFIG="${HOME}/.kube/config" + bash scripts/verify-ksail-arc-runtime.sh --if-active + - name: Verify the deployed data product UI release # The protected deployment must prove current Apps/chart/Helm identities, # all current Pods and public endpoints before its candidate can merge. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index db6a555183..0974131b25 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -149,8 +149,20 @@ jobs: - name: ๐Ÿ”Ž Validate finite runtime-log evidence run: go test ./scripts/analyze-cri-logs - - name: ๐Ÿ›‘ Validate inactive ARC staging boundaries - run: go test ./scripts/tests/arc-staging + - name: ๐Ÿ›‘ Validate bounded ARC activation boundaries + run: | + go test ./scripts/tests/arc-staging + shellcheck scripts/tests/test-wait-for-ksail-arc-registration.sh + bash scripts/tests/test-wait-for-ksail-arc-registration.sh + + - name: ๐Ÿงพ Validate published ARC metrics bytes + run: go test ./scripts/generate-ksail-arc-job-metrics + + - name: ๐Ÿ”ฌ Validate bounded ARC runtime acceptance + run: | + go test -race ./scripts/verify-ksail-arc-runtime + shellcheck scripts/verify-ksail-arc-runtime.sh scripts/ksail-arc-admission-fence.sh scripts/wait-for-ksail-arc-registration.sh scripts/tests/test-verify-ksail-arc-runtime.sh scripts/ksail-arc-job-metrics.sh + bash scripts/tests/test-verify-ksail-arc-runtime.sh - name: ๐Ÿ” Validate ARC transport proof boundaries run: | @@ -496,6 +508,11 @@ jobs: - '.github/scripts/setup-supply-chain-tools.sh' - 'scripts/tests/test-setup-supply-chain-tools.sh' - '.github/actions/deploy-prod/action.yml' + - 'scripts/verify-ksail-arc-runtime.sh' + - 'scripts/ksail-arc-admission-fence.sh' + - 'scripts/wait-for-ksail-arc-registration.sh' + - 'scripts/verify-ksail-arc-runtime/**' + - 'scripts/tests/test-verify-ksail-arc-runtime.sh' - '.github/workflows/dr-rebuild.yaml' - '.github/workflows/validate-image-verifier-liveness.yaml' - 'scripts/tests/test-openbao-oidc-role.sh' diff --git a/docs/operations/arc-runners.md b/docs/operations/arc-runners.md index ef28fd5819..dd80197205 100644 --- a/docs/operations/arc-runners.md +++ b/docs/operations/arc-runners.md @@ -1,23 +1,15 @@ # Organization Linux runners -The organization Linux pool is prepared but inactive and remains outside the -deployment aggregate and its HelmRelease is suspended. Production reconciles the -scoped controller and the former analysis release in Flux inventory to preserve -ownership of resources left by failed activation. The former release has explicit -zero minimum and maximum runner bounds. Suspension stops Helm reconciliation, -does not drain an installed scale set, and prevents a failed release from recovering -its readiness status. Native Flux and Helm health checks stay enabled. The protected -deploy proves complete, empty metadata lists for organization runner sets, releases, -credential-sync resources, retained runner children, both runner namespaces' pods -and labeled listeners in the controller namespace before publication and after -reconciliation. The retained namespace may contain only the exact chart-declared -AutoscalingRunnerSet; Helm keeps that declaration with both runner bounds at zero. -After reconciliation the guard also proves the installed controller is fully -rolled out and excludes that retained namespace. Native empty `items: null` is -accepted only with a complete current-revision list; missing or paginated items -cannot prove absence. It requests no App credentials, JIT configuration or full -Pod responses. Registration, execution and cleanup still -need the separate proofs below before KSail #7131 can close. +The production definition includes the ARC controller and bounded organization +Linux pool. The protected deployment must prove the existing runtime App, +registration and isolated canary before activation can merge. Initial access +admits only KSail's main-branch delivery preflight. Managed analysis remains on its +existing route until actual runner delivery and full-source calibration pass; +declaring the pool does not resolve KSail #7131. The former analysis release +reconciles with explicit zero minimum and maximum runner bounds and remains +protected in Flux inventory. The scoped controller excludes its namespace. +Suspension does not drain an installed scale set and prevents failed Helm +readiness from recovering. Native Flux, Helm and orphan checks stay enabled. The controller chart and runner-set chart use the same immutable 0.15.0 artifacts. The runner image is digest-pinned. The controller manages runner sets only in the @@ -25,8 +17,8 @@ The runner image is digest-pinned. The controller manages runner sets only in th with `devantler-tech` in the dedicated `platform` runner group and exposes the `platform-linux` scale-set name. Repository access is opt-in through the group; no existing workflow or managed analysis setting is changed. Other repositories -can opt in without another App or a repository-specific pool. KSail is a proposed -first consumer, not the scope of the capability. +can opt in through reviewed access changes without another App or another pool. +KSail is the first selected consumer. The pool has zero idle runners and a maximum of one job runner **across all opted-in repositories**, not one runner per repository. There is no container mode, @@ -39,8 +31,9 @@ checksum-checked Go and Node archives provide compiler headers without job-time root access. The publisher builds and exercises the image on pull requests, then publishes, attests and signs only on a push to KSail main. Kyverno and Talos accept that workflow identity only for the exact analysis image repository. -The pool does not select that image until a verified published digest and the -consumer's runtime proof are available. +The pool selects the published KSail-owned image by immutable digest. The +protected canary verifies its signature, toolchain and resource limits before +any managed-analysis route changes. The toolchain smoke test starts the copied runner and compiles a Go program against GTK and WebKit as UID/GID 1001, with a read-only root filesystem and no @@ -108,7 +101,7 @@ absence readback are mandatory. This workflow reads no App key and requests no reader token. Its counts-only result is transport evidence, not stored-key, runner-registration or managed-analysis evidence. -Activation requires a separate reviewed change and all of these proofs: +Activation and subsequent consumer admission require all of these proofs: 1. Reuse the production platform App used for GitHub sign-in, identified by `github_app_client_id` in the production bootstrap configuration. This is @@ -139,8 +132,14 @@ Activation requires a separate reviewed change and all of these proofs: Reuse does not narrow the shared App's authority: the runner group controls job access, not what the App credential can do. Never mount the private key in a job runner. -2. An authorized organization operator creates and verifies the `platform` - runner group **before** enabling the pool. Use **Selected repositories**, not +2. The namespaced `arc-runtime-platform-app` ProviderConfig manages the + `platform` group using the existing `arc-github-app` Secret in `arc-runners`. + Its additional JSON credential key preserves ARC's three original keys; + the private key never moves to the GitHub-management namespace. The provider + name is distinct because the pinned provider caches configurations by name; + the registration gate rejects a same-name configuration in another namespace + or provider kind. + Verify the observed group **before** delivering a job. Use **Selected repositories**, not all repositories, and select only explicitly approved consumers. Repository access alone does not make arbitrary code trusted. Before admitting a public repository, prove the group's selected-workflow/ref restrictions admit only @@ -150,6 +149,11 @@ Activation requires a separate reviewed change and all of these proofs: repository excluded. Do not weaken organization-wide runner restrictions or disable GitHub-hosted runners. A failed or unauthorized group read is unknown, not permission to use the unrestricted default group. + Initial selection is exactly KSail and + `devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main`. + The native dynamic Code Quality workflow is not admitted by this initial + selection. Its identity, supported restrictions and intercepted fork-job + denial require separate API and runtime proof before routing analysis. 3. Use the declared `autoscale-arc-runners` CX53 pool for isolated organization runner capacity. It has a minimum of zero and maximum of one node, sharing the unchanged cluster ceiling of nine nodes and account ceiling of ten. The @@ -196,13 +200,13 @@ Activation requires a separate reviewed change and all of these proofs: removed before declaring that bug fixed. Those bug-specific acceptance criteria do not route or onboard other repositories automatically. -For staged validation, run `go test ./scripts/tests/arc-staging` and build both -component directories directly with `kubectl kustomize`. Local trees exclude ARC; -production retains the reconciled controller and drained protected legacy analysis resources -without including the organization pool. The unconditional CI guard runs on -pull requests and merge groups. A deliberate activation revises that guard in -the same reviewed change, alongside its evidence; deleting the guard alone is -not activation proof. +For configuration validation, run `go test ./scripts/tests/arc-staging` and build +both component directories directly with `kubectl kustomize`. Local trees exclude +ARC. The unconditional guard permits activation only through the two named +production aggregates, with immutable images and the mandatory protected +runtime canary. It runs on pull requests and merge groups; deleting it is not activation +proof. Credential staging is absorbed by the full pool component at activation, +so the reader and encrypted store have exactly one reconciled declaration. ## Repository opt-in @@ -228,16 +232,52 @@ finish. It does not require retiring the shared platform App. ## Rollout and recovery -After the gates above are approved, verify the scoped controller is healthy and -retire its `platform.devantler.tech/arc-recovery: drain-only` marker in the reviewed -activation change. The recovery guard intentionally refuses a credentialed pool -while that marker remains. That component creates both -namespaces before the chart installs its namespace-scoped RBAC. The controller's -network policy also covers the listener, which ARC creates in that namespace. -Only then reference the pool in -the infrastructure layer, which creates its dedicated secret store and reader -identity. Verify SecretStore and ExternalSecret readiness before admitting jobs. Keep -the opt-in runner name out of workflows that have not completed onboarding. +The controller layer creates both namespaces before namespace-scoped RBAC +reconciles. Its network policy also covers the listener. The infrastructure +layer declares the pool, dedicated credential reader and native runner group. +Verify the stored App identity and encrypted credential boundary before +activating this layer. The reviewed activation explicitly retires the controller's +`platform.devantler.tech/arc-recovery: drain-only` marker: that metadata guard +refuses a credentialed pool while the marker remains. Its implementation and +regression tests remain present. The protected canary joins the deployed Flux revision, +current provider/group observations, ARC registration and exact image before +it exercises admission, allowed connectivity, intercepted denials and cleanup. +Verify SecretStore and ExternalSecret readiness before admitting jobs. Keep the +opt-in runner name out of workflows that have not completed onboarding. + +The protected canary reserves admission with the fixed `arc-runtime-admission` +ResourceQuota in the runner namespace. Its zero-pod `NotTerminating` scope blocks +new ordinary runner Pods while existing jobs finish naturally. The deadline-bound +probe is outside that scope. The verifier checks retained runner templates, proves +the real runner template is denied by this quota, and waits up to twenty minutes +for existing runners and the dedicated node to drain. It leaves the quota in place +until probe deletion and node cleanup are verified, then deletes only the +invocation-owned quota with UID and resource-version preconditions. It changes no +runner bounds, Helm reconciliation, App permission or capacity ceiling. + +A later protected deployment can recover a quota left by a completed prior +workflow attempt. The quota records the fixed repository and workflow, run and +attempt, producing job, run-head and checkout revisions, manifest digest and +successfully created probe UID. Recovery joins the exact attempt and its complete +job list using the existing read-only Actions permission. It retains admission +while deleting only that recorded, standalone probe with a UID precondition, +proving absence and natural node drain, then deleting the unchanged quota with +UID and resource-version preconditions. It proves quota absence before creating +the new invocation's fence. + +Active or unknown writers, incomplete job lists, missing provenance, ambiguous +creation, replacement objects and incomplete cleanup remain HOLD. Same-attempt +Heal Prod cannot recover a fence while its attempt is still running. The first +activation therefore also depends on the inactive-main recovery support in +[#4638](https://github.com/devantler-tech/platform/issues/4638) and +[#4641](https://github.com/devantler-tech/platform/pull/4641); candidate-only +acceptance does not establish a recoverable rollout. A legacy fence without the +recorded proof needs explicit manual recovery: prove its producing job and +attempt have stopped, verify invocation ownership and probe absence, prove the +dedicated pool has drained, and use the current UID and resource version as +deletion preconditions. Never release a live or unknown invocation's fence to +make a deployment pass. Quota behavior and its deadline scopes are defined +in the [Kubernetes resource quota documentation](https://kubernetes.io/docs/concepts/policy/resource-quotas/). To stop admitting jobs, restore every consumer's prior runner configuration and verify the readbacks, then use a reviewed values change to set both runner diff --git a/k8s/bases/infrastructure/actions-runners/config-map-job-metrics.yaml b/k8s/bases/infrastructure/actions-runners/config-map-job-metrics.yaml new file mode 100644 index 0000000000..696d72a660 --- /dev/null +++ b/k8s/bases/infrastructure/actions-runners/config-map-job-metrics.yaml @@ -0,0 +1,71 @@ +# Generated by go run ./scripts/generate-ksail-arc-job-metrics -write; do not edit. +apiVersion: v1 +data: + job-metrics.sh: | + #!/usr/bin/env bash + set -euo pipefail + + arc_metrics_fail() { + printf '::error::KSail ARC whole-job cgroup measurement failed: %s\n' "$1" >&2 + return 1 + } + + arc_metrics_read() { + local root=$1 name=$2 + if [[ "$root" == /sys/fs/cgroup ]]; then + timeout 5s cat "$root/$name" 2>/dev/null + else + # Only regular test-owned fixture files use the portable read path. + [[ -f "$root/$name" && ! -L "$root/$name" ]] || return 1 + cat "$root/$name" 2>/dev/null + fi + } + + # The production entrypoint uses only the container's kernel cgroup. A supplied + # fixture directory is used when this function is sourced by offline tests. + arc_job_metrics() { + local root=$1 limit peak events key value extra + local low='' high='' max='' oom='' oom_kill='' oom_group_kill='' sock_throttled='' + limit=$(arc_metrics_read "$root" memory.max) || { arc_metrics_fail read-limit; return 1; } + peak=$(arc_metrics_read "$root" memory.peak) || { arc_metrics_fail read-peak; return 1; } + events=$(arc_metrics_read "$root" memory.events) || { arc_metrics_fail read-events; return 1; } + [[ "$limit" == 15032385536 ]] || { arc_metrics_fail limit; return 1; } + [[ "$peak" =~ ^(0|[1-9][0-9]{0,17})$ && "$peak" -gt 0 ]] || { + arc_metrics_fail peak; return 1; + } + while read -r key value extra; do + [[ -z "$extra" && "$value" =~ ^(0|[1-9][0-9]{0,17})$ ]] || { arc_metrics_fail event-format; return 1; } + case "$key" in + low|high|max|oom|oom_kill|oom_group_kill) + [[ -z "${!key}" ]] || { arc_metrics_fail duplicate-event; return 1; } + printf -v "$key" '%s' "$value" ;; + sock_throttled) + [[ -z "$sock_throttled" ]] || { arc_metrics_fail duplicate-event; return 1; } + sock_throttled=$value ;; + *) arc_metrics_fail unknown-event; return 1 ;; + esac + done <<<"$events" + [[ -n "$low" && -n "$high" && -n "$max" && -n "$oom" && -n "$oom_kill" && -n "$oom_group_kill" ]] || { + arc_metrics_fail missing-event; return 1; + } + [[ "$oom" == 0 && "$oom_kill" == 0 && "$oom_group_kill" == 0 ]] || { arc_metrics_fail oom; return 1; } + printf 'KSail ARC whole-job cgroup: {"schemaVersion":1,"memoryMaxBytes":%s,"memoryPeakBytes":%s,"limitEvents":%s,"oomEvents":%s,"oomKills":%s,"oomGroupKills":%s}\n' \ + "$limit" "$peak" "$max" "$oom" "$oom_kill" "$oom_group_kill" + # The kernel may temporarily exceed memory.max. Retain that valid measurement, + # while preserving the existing runtime acceptance ceiling as a separate refusal. + [[ "$peak" -le "$limit" ]] || { arc_metrics_fail budget; return 1; } + } + + if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + [[ $# == 0 ]] || { arc_metrics_fail arguments; exit 1; } + arc_job_metrics /sys/fs/cgroup + fi +immutable: true +kind: ConfigMap +metadata: + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled + labels: + app.kubernetes.io/name: ksail-arc-job-metrics + name: ksail-arc-job-metrics-2fbcf0b1d7 + namespace: arc-runners diff --git a/k8s/bases/infrastructure/actions-runners/external-secret.yaml b/k8s/bases/infrastructure/actions-runners/external-secret.yaml index c3250e3fc8..a043c16d2f 100644 --- a/k8s/bases/infrastructure/actions-runners/external-secret.yaml +++ b/k8s/bases/infrastructure/actions-runners/external-secret.yaml @@ -11,6 +11,13 @@ spec: target: name: arc-github-app creationPolicy: Owner + template: + engineVersion: v2 + mergePolicy: Merge + data: + # JSON encoding preserves the existing PEM and the original ARC keys. + provider-credentials: >- + {"owner":"devantler-tech","app_auth":[{"id":{{ .github_app_id | toJson }},"installation_id":{{ .github_app_installation_id | toJson }},"pem_file":{{ .github_app_private_key | toJson }}}]} data: - secretKey: github_app_id remoteRef: diff --git a/k8s/bases/infrastructure/actions-runners/helm-release.yaml b/k8s/bases/infrastructure/actions-runners/helm-release.yaml index 1fbd9e44de..55e034e6c8 100644 --- a/k8s/bases/infrastructure/actions-runners/helm-release.yaml +++ b/k8s/bases/infrastructure/actions-runners/helm-release.yaml @@ -1,4 +1,4 @@ -# Inactive, uncalibrated organization Linux pool. Do not route a workflow here until +# Active, uncalibrated organization Linux pool. Do not route a workflow here until # the registration, isolation, toolchain and capacity gates in the runbook pass. apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease @@ -8,14 +8,14 @@ metadata: annotations: kustomize.toolkit.fluxcd.io/prune: disabled spec: - suspend: true + suspend: false interval: 10m timeout: 10m chartRef: kind: OCIRepository name: platform-runners values: - # Use the controller's built-in scaler defaults during preparation. + # Use the controller's built-in scaler defaults for the bounded analysis pool. # ARC 0.15's optional tuning object is absent from the public CRD catalog. # null is explicitly supported by the pinned chart's values schema. listenerConfig: null @@ -55,9 +55,16 @@ spec: - name: runner-tmp emptyDir: sizeLimit: 2Gi + - name: runner-metrics + configMap: + name: ksail-arc-job-metrics + defaultMode: 365 # 0555: executable, read-only job hook. + items: + - key: job-metrics.sh + path: job-metrics.sh initContainers: - name: init-runner-home - image: ghcr.io/actions/actions-runner@sha256:e5496277be5d09bc968b3d64911b74e219ac4a3f2edce956a3ecf9271bea1ef4 + image: ghcr.io/devantler-tech/ksail-analysis-runner@sha256:f2fc508d5e426405bdc3732f652b251e093d93e7becdec34f46b1fe6f3309183 command: ["/bin/sh", "-ec", "cp -R /home/runner/. /runner-data/"] securityContext: allowPrivilegeEscalation: false @@ -79,8 +86,11 @@ spec: mountPath: /runner-data containers: - name: runner - image: ghcr.io/actions/actions-runner@sha256:e5496277be5d09bc968b3d64911b74e219ac4a3f2edce956a3ecf9271bea1ef4 + image: ghcr.io/devantler-tech/ksail-analysis-runner@sha256:f2fc508d5e426405bdc3732f652b251e093d93e7becdec34f46b1fe6f3309183 command: ["/home/runner/run.sh"] + env: + - name: ACTIONS_RUNNER_HOOK_JOB_COMPLETED + value: /etc/ksail-arc-metrics/job-metrics.sh securityContext: allowPrivilegeEscalation: false privileged: false @@ -92,6 +102,9 @@ spec: mountPath: /home/runner - name: runner-tmp mountPath: /tmp + - name: runner-metrics + mountPath: /etc/ksail-arc-metrics + readOnly: true resources: requests: cpu: "3" diff --git a/k8s/bases/infrastructure/actions-runners/kustomization.yaml b/k8s/bases/infrastructure/actions-runners/kustomization.yaml index a9ae04bfdc..551aa744b8 100644 --- a/k8s/bases/infrastructure/actions-runners/kustomization.yaml +++ b/k8s/bases/infrastructure/actions-runners/kustomization.yaml @@ -1,9 +1,25 @@ # Opt-in organization runner pool; activation is a separate reviewed change. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +replacements: + - source: + kind: ConfigMap + version: v1 + namespace: arc-runners + fieldPath: metadata.name + targets: + - select: + kind: HelmRelease + name: platform-runners + namespace: arc-runners + fieldPaths: + - spec.values.template.spec.volumes.[name=runner-metrics].configMap.name resources: + - config-map-job-metrics.yaml - oci-repository.yaml - helm-release.yaml - credentials/ - external-secret.yaml + - provider-config.yaml + - runner-group.yaml - cilium-network-policy-runner.yaml diff --git a/k8s/bases/infrastructure/actions-runners/provider-config.yaml b/k8s/bases/infrastructure/actions-runners/provider-config.yaml new file mode 100644 index 0000000000..c096eedd0e --- /dev/null +++ b/k8s/bases/infrastructure/actions-runners/provider-config.yaml @@ -0,0 +1,14 @@ +# A distinct cache key keeps the runtime App separate from the management App. +# The namespaced provider reads the existing ARC Secret in this namespace. +apiVersion: github.m.upbound.io/v1beta1 +kind: ProviderConfig +metadata: + name: arc-runtime-platform-app + namespace: arc-runners +spec: + credentials: + source: Secret + secretRef: + namespace: arc-runners + name: arc-github-app + key: provider-credentials diff --git a/k8s/bases/infrastructure/actions-runners/runner-group.yaml b/k8s/bases/infrastructure/actions-runners/runner-group.yaml new file mode 100644 index 0000000000..14e0702ca1 --- /dev/null +++ b/k8s/bases/infrastructure/actions-runners/runner-group.yaml @@ -0,0 +1,20 @@ +# Initial access is restricted to the main-only KSail delivery preflight. +# Dynamic managed-analysis admission requires its own verified selection. +apiVersion: actions.github.m.upbound.io/v1alpha1 +kind: RunnerGroup +metadata: + name: platform + namespace: arc-runners +spec: + managementPolicies: [Observe, Create, Update] + providerConfigRef: + name: arc-runtime-platform-app + kind: ProviderConfig + forProvider: + name: platform + visibility: selected + allowsPublicRepositories: true + selectedRepositoryIds: [737584922] + restrictedToWorkflows: true + selectedWorkflows: + - devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main diff --git a/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml b/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml index 061fe2609a..756fa800c2 100644 --- a/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml +++ b/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml @@ -1,5 +1,5 @@ -# Production reconciles this scoped release in Flux inventory. The organization -# pool stays unreferenced; activation is separate. See docs/operations/arc-runners.md. +# Active controller for the bounded organization pool. Protected deployment must +# prove registration and isolation before any consumer routes a job here. apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -7,7 +7,6 @@ metadata: namespace: arc-systems annotations: kustomize.toolkit.fluxcd.io/prune: disabled - platform.devantler.tech/arc-recovery: drain-only labels: helm.toolkit.fluxcd.io/crds: enabled spec: diff --git a/k8s/providers/hetzner/infrastructure/arc-credential-transport/kustomization.yaml b/k8s/providers/hetzner/infrastructure/arc-credential-transport/kustomization.yaml deleted file mode 100644 index 36eef1ddfc..0000000000 --- a/k8s/providers/hetzner/infrastructure/arc-credential-transport/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Authentication and CA validation can be exercised before reading the App key -# or activating ARC. The full pool replaces this staging reference on activation. -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - ../../../../bases/infrastructure/actions-runners/credentials/ diff --git a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml index 6e397ab07d..591a068459 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml @@ -8,9 +8,7 @@ resources: # local/CI only): no VirtualMachines/DataVolumes exist on Hetzner and # virt-handler crash-looped (1000+ restarts) with zero VMs. - ../../../../bases/infrastructure/controllers/ - # Retain the protected ARC namespaces and release in Flux inventory while - # registration is repaired. Flux reconciles the controller with its watch - # scope limited to arc-runners; the organization pool stays outside aggregates. + # Prod-only: the namespace-scoped organization runner controller. - ../../../../bases/infrastructure/controllers/actions-runner-controller/ # Prod-only: a minimal Alertmanager scoped to Kubescape runtime-detection # alerts. The Headlamp Kubescape plugin's "Runtime Detection > Alerts" tab diff --git a/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml b/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml index a2e16226fe..1dc563425d 100644 --- a/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml +++ b/k8s/providers/hetzner/infrastructure/flux-notifications/alert.yaml @@ -87,6 +87,9 @@ spec: - kind: HelmRelease name: "*" namespace: actual-budget + - kind: HelmRelease + name: "*" + namespace: arc-runners - kind: HelmRelease name: "*" namespace: arc-ksail-analysis @@ -183,5 +186,11 @@ spec: - kind: HelmRelease name: "*" namespace: whoami + - kind: OCIRepository + name: "*" + namespace: arc-runners + - kind: OCIRepository + name: "*" + namespace: arc-systems eventMetadata: summary: "Flux reconciliation error โ€” prod platform" diff --git a/k8s/providers/hetzner/infrastructure/kustomization.yaml b/k8s/providers/hetzner/infrastructure/kustomization.yaml index e16d747501..c3db65cb6d 100644 --- a/k8s/providers/hetzner/infrastructure/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/kustomization.yaml @@ -3,10 +3,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../../../bases/infrastructure/ + # Prod-only: the organization runner pool (max one runner), authenticated by the shared App. + - ../../../bases/infrastructure/actions-runners/ # Re-adopt the retained release without activating a runner. Suspension does # not drain an installed scale set; runtime acceptance remains a separate gate. - retained-ksail-analysis/ - - arc-credential-transport/ # Issuance restrictions must be active before credentials or runners are used. # This layer follows installation of the Kyverno admission controller. - cluster-policies/restrict-arc-openbao-certificate.yaml diff --git a/scripts/generate-ksail-arc-job-metrics/main.go b/scripts/generate-ksail-arc-job-metrics/main.go new file mode 100644 index 0000000000..aecf2d804c --- /dev/null +++ b/scripts/generate-ksail-arc-job-metrics/main.go @@ -0,0 +1,68 @@ +// Generate the public metrics hook as a manifest the KSail publisher includes. +package main + +import ( + "crypto/sha256" + "flag" + "fmt" + "os" + + "sigs.k8s.io/yaml" +) + +const ( + scriptPath = "scripts/ksail-arc-job-metrics.sh" + manifestPath = "k8s/bases/infrastructure/actions-runners/config-map-job-metrics.yaml" +) + +func generate(script []byte) ([]byte, error) { + if len(script) == 0 || len(script) > 64*1024 { + return nil, fmt.Errorf("metrics hook must contain 1โ€“65536 bytes") + } + sum := sha256.Sum256(script) + manifest := map[string]any{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]any{ + "name": fmt.Sprintf("ksail-arc-job-metrics-%x", sum[:5]), + "namespace": "arc-runners", + "labels": map[string]string{"app.kubernetes.io/name": "ksail-arc-job-metrics"}, + "annotations": map[string]string{"kustomize.toolkit.fluxcd.io/substitute": "disabled"}, + }, + "immutable": true, + "data": map[string]string{"job-metrics.sh": string(script)}, + } + data, err := yaml.Marshal(manifest) + if err != nil { + return nil, err + } + return append([]byte("# Generated by go run ./scripts/generate-ksail-arc-job-metrics -write; do not edit.\n"), data...), nil +} + +func run() error { + write := flag.Bool("write", false, "update the generated manifest") + flag.Parse() + if flag.NArg() != 0 { + return fmt.Errorf("unexpected positional arguments") + } + script, err := os.ReadFile(scriptPath) + if err != nil { + return err + } + data, err := generate(script) + if err != nil { + return err + } + if *write { + return os.WriteFile(manifestPath, data, 0644) + } + _, err = os.Stdout.Write(data) + return err +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/scripts/generate-ksail-arc-job-metrics/main_test.go b/scripts/generate-ksail-arc-job-metrics/main_test.go new file mode 100644 index 0000000000..c7b9803a85 --- /dev/null +++ b/scripts/generate-ksail-arc-job-metrics/main_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "testing" +) + +func TestPublishedMetricsManifestIsCurrent(t *testing.T) { + script, err := os.ReadFile(filepath.Join("../..", scriptPath)) + if err != nil { + t.Fatal(err) + } + expected, err := generate(script) + if err != nil { + t.Fatal(err) + } + published, err := os.ReadFile(filepath.Join("../..", manifestPath)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(published, expected) { + t.Fatal("published hook differs from its source; run go run ./scripts/generate-ksail-arc-job-metrics -write") + } +} + +func TestMetricsManifestRejectsEmptyOrOversizedSource(t *testing.T) { + for _, size := range []int{0, 65537} { + if _, err := generate(make([]byte, size)); err == nil { + t.Fatalf("accepted %d bytes", size) + } + } +} diff --git a/scripts/generate-ksail-arc-job-metrics/publication_test.go b/scripts/generate-ksail-arc-job-metrics/publication_test.go new file mode 100644 index 0000000000..b25e9cd68d --- /dev/null +++ b/scripts/generate-ksail-arc-job-metrics/publication_test.go @@ -0,0 +1,109 @@ +package main + +import ( + "bytes" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +const metricsComponent = "k8s/bases/infrastructure/actions-runners" + +func field(t *testing.T, value any, path ...string) any { + t.Helper() + for _, key := range path { + mapping, ok := value.(map[string]any) + if !ok { + t.Fatalf("%s: expected a mapping, got %T", strings.Join(path, "."), value) + } + value, ok = mapping[key] + if !ok { + t.Fatalf("missing %s", strings.Join(path, ".")) + } + } + return value +} + +func equal(t *testing.T, value, want any) { + t.Helper() + if value != want { + t.Fatalf("got %#v, want %#v", value, want) + } +} + +func TestMetricsHookSurvivesPublishedArtifactRender(t *testing.T) { + // The real publisher includes manifest files, not arbitrary checkout files. + // Render that artifact boundary so a checkout-only dependency cannot pass. + component := filepath.Join(repoRoot, metricsComponent) + published := t.TempDir() + err := filepath.WalkDir(component, func(path string, file fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if file.IsDir() { + return nil + } + switch strings.ToLower(filepath.Ext(path)) { + case ".yaml", ".yml", ".json": + data, err := os.ReadFile(path) + if err != nil { + return err + } + relative, err := filepath.Rel(component, path) + if err != nil { + return err + } + destination := filepath.Join(published, relative) + if err := os.MkdirAll(filepath.Dir(destination), 0700); err != nil { + return err + } + return os.WriteFile(destination, data, 0600) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + rendered, err := exec.Command("kubectl", "kustomize", published).CombinedOutput() + if err != nil { + t.Fatalf("published metrics dependencies cannot render: %v\n%s", err, rendered) + } + script, err := os.ReadFile(filepath.Join(repoRoot, metricsScript)) + if err != nil { + t.Fatal(err) + } + decoder := yaml.NewDecoder(bytes.NewReader(rendered)) + var config, release map[string]any + for { + var object map[string]any + if err := decoder.Decode(&object); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + if object["kind"] == "ConfigMap" { + if config != nil { + t.Fatal("ambiguous metrics ConfigMap") + } + config = object + } + if object["kind"] == "HelmRelease" { + release = object + } + } + if config == nil || release == nil { + t.Fatal("published artifact lacks the metrics ConfigMap or runner release") + } + equal(t, field(t, config, "immutable"), true) + equal(t, field(t, config, "metadata", "annotations", "kustomize.toolkit.fluxcd.io/substitute"), "disabled") + equal(t, field(t, config, "data", "job-metrics.sh"), string(script)) + volumes := field(t, release, "spec", "values", "template", "spec", "volumes").([]any) + equal(t, field(t, volumes[2], "configMap", "name"), field(t, config, "metadata", "name")) + equal(t, field(t, volumes[2], "configMap", "defaultMode"), 365) +} diff --git a/scripts/generate-ksail-arc-job-metrics/runtime_test.go b/scripts/generate-ksail-arc-job-metrics/runtime_test.go new file mode 100644 index 0000000000..d7762f7338 --- /dev/null +++ b/scripts/generate-ksail-arc-job-metrics/runtime_test.go @@ -0,0 +1,173 @@ +package main + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +const repoRoot = "../.." +const metricsScript = "scripts/ksail-arc-job-metrics.sh" + +func TestJobMetricsEntrypointRejectsAnAlternateCgroup(t *testing.T) { + script := filepath.Join(repoRoot, metricsScript) + output, err := exec.Command("bash", script, t.TempDir()).CombinedOutput() + if err == nil || strings.Contains(string(output), "KSail ARC whole-job cgroup:") { + t.Fatalf("accepted a substituted production cgroup: %s", output) + } +} + +func TestWholeJobCgroupMeasurement(t *testing.T) { + for _, tc := range []struct { + name, limit, peak, events string + ok bool + }{ + {"complete", "15032385536", "9876543210", "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\noom_group_kill 0\n", true}, + {"socket throttled zero", "15032385536", "9876543210", "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0\n", true}, + {"socket throttled nonzero", "15032385536", "9876543210", "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 2\n", true}, + {"duplicate socket counter", "15032385536", "10", "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled 0\nsock_throttled 0\n", false}, + {"malformed socket counter", "15032385536", "10", "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\noom_group_kill 0\nsock_throttled nope\n", false}, + {"socket counter cannot replace required event", "15032385536", "10", "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\nsock_throttled 0\n", false}, + {"missing limit", "", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"unbounded", "max", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"wrong cgroup", "17179869184", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"missing peak", "15032385536", "", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"zero peak", "15032385536", "0", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"malformed peak", "15032385536", "1\n2", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"noncanonical peak", "15032385536", "001", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"overflow", "15032385536", "99999999999999999999999", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"missing events", "15032385536", "10", "", false}, + {"missing OOM", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom_kill 0\noom_group_kill 0\n", false}, + {"duplicate event", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom 0\noom 0\noom_kill 0\noom_group_kill 0\n", false}, + {"unknown event", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\nsecret 0\n", false}, + {"malformed event", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom nope\noom_kill 0\noom_group_kill 0\n", false}, + {"extra event data", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom 0 extra\noom_kill 0\noom_group_kill 0\n", false}, + {"OOM", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom 1\noom_kill 0\noom_group_kill 0\n", false}, + {"OOM kill", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 1\noom_group_kill 0\n", false}, + {"OOM group kill", "15032385536", "10", "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 1\n", false}, + {"over budget with OOM", "15032385536", "15032385537", "low 0\nhigh 0\nmax 3\noom 1\noom_kill 0\noom_group_kill 0\n", false}, + {"over budget with invalid events", "15032385536", "15032385537", "low 0\nhigh 0\nmax 3\noom nope\noom_kill 0\noom_group_kill 0\n", false}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + for name, value := range map[string]string{"memory.max": tc.limit, "memory.peak": tc.peak, "memory.events": tc.events} { + if value != "" { + if err := os.WriteFile(filepath.Join(root, name), []byte(strings.TrimSuffix(value, "\n")+"\n"), 0600); err != nil { + t.Fatal(err) + } + } + } + script, err := filepath.Abs(filepath.Join(repoRoot, metricsScript)) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("bash", "-c", `source "$1"; arc_job_metrics "$2"`, "metrics-test", script, root) + cmd.Env = append(os.Environ(), "ACTIONS_RUNNER_INPUT_JITCONFIG=must-not-be-read", "SECRET_CANARY=must-not-be-read") + output, err := cmd.CombinedOutput() + if (err == nil) != tc.ok { + t.Fatalf("success=%t, want %t: %s", err == nil, tc.ok, output) + } + if strings.Contains(string(output), "must-not-be-read") { + t.Fatal("measurement exposed environment data") + } + if !tc.ok { + if strings.Contains(string(output), "KSail ARC whole-job cgroup:") { + t.Fatal("failed measurement emitted a success receipt") + } + return + } + var got struct { + SchemaVersion int `json:"schemaVersion"` + MemoryMaxBytes uint64 `json:"memoryMaxBytes"` + MemoryPeakBytes uint64 `json:"memoryPeakBytes"` + LimitEvents uint64 `json:"limitEvents"` + OOMEvents uint64 `json:"oomEvents"` + OOMKills uint64 `json:"oomKills"` + OOMGroupKills uint64 `json:"oomGroupKills"` + } + decoder := json.NewDecoder(strings.NewReader(strings.TrimPrefix(string(output), "KSail ARC whole-job cgroup: "))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&got); err != nil { + t.Fatal(err) + } + if got.SchemaVersion != 1 || got.MemoryMaxBytes != 15032385536 || got.MemoryPeakBytes != 9876543210 || + got.LimitEvents != 3 || got.OOMEvents != 0 || got.OOMKills != 0 || got.OOMGroupKills != 0 { + t.Fatalf("wrong measurement: %+v", got) + } + }) + } +} + +func TestOverBudgetPeakRetainsMeasurementAndRejectsAcceptance(t *testing.T) { + root := t.TempDir() + for name, value := range map[string]string{ + "memory.max": "15032385536\n", "memory.peak": "15032385537\n", + "memory.events": "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\noom_group_kill 0\n", + } { + if err := os.WriteFile(filepath.Join(root, name), []byte(value), 0600); err != nil { + t.Fatal(err) + } + } + script, err := filepath.Abs(filepath.Join(repoRoot, metricsScript)) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("bash", "-c", `source "$1"; arc_job_metrics "$2"`, "metrics-test", script, root) + output, err := cmd.CombinedOutput() + if err == nil { + t.Fatal("over-budget measurement accepted") + } + const receipt = `KSail ARC whole-job cgroup: {"schemaVersion":1,"memoryMaxBytes":15032385536,"memoryPeakBytes":15032385537,"limitEvents":3,"oomEvents":0,"oomKills":0,"oomGroupKills":0}` + if string(output) != receipt+"\n::error::KSail ARC whole-job cgroup measurement failed: budget\n" { + t.Fatalf("lost measurement or budget refusal: %s", output) + } +} + +func TestMetricsFixturesArePortableAndProductionReadsRemainBounded(t *testing.T) { + root := t.TempDir() + for name, value := range map[string]string{ + "memory.max": "15032385536\n", "memory.peak": "9876543210\n", + "memory.events": "low 0\nhigh 0\nmax 3\noom 0\noom_kill 0\noom_group_kill 0\n", + } { + if err := os.WriteFile(filepath.Join(root, name), []byte(value), 0600); err != nil { + t.Fatal(err) + } + } + bin := t.TempDir() + log := filepath.Join(bin, "timeout-invocations") + if err := os.WriteFile(filepath.Join(bin, "timeout"), []byte("#!/bin/bash\nprintf '%s\\n' \"$*\" >> \"$TIMEOUT_LOG\"\nexit 124\n"), 0700); err != nil { + t.Fatal(err) + } + script, err := filepath.Abs(filepath.Join(repoRoot, metricsScript)) + if err != nil { + t.Fatal(err) + } + for _, production := range []bool{false, true} { + cgroup := root + if production { + cgroup = "/sys/fs/cgroup" + } + cmd := exec.Command("bash", "-c", `source "$1"; arc_job_metrics "$2"`, "metrics-test", script, cgroup) + cmd.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH"), "TIMEOUT_LOG="+log) + output, err := cmd.CombinedOutput() + if production { + if err == nil || strings.Contains(string(output), "KSail ARC whole-job cgroup:") { + t.Fatalf("unbounded production receipt: %s", output) + } + calls, readErr := os.ReadFile(log) + if readErr != nil || string(calls) != "5s cat /sys/fs/cgroup/memory.max\n" { + t.Fatalf("production timeout not used: %q, %v", calls, readErr) + } + } else { + if err != nil || !strings.Contains(string(output), "KSail ARC whole-job cgroup:") { + t.Fatalf("fixture required GNU timeout: %s", output) + } + if _, err := os.Stat(log); !os.IsNotExist(err) { + t.Fatalf("fixture invoked production timeout: %v", err) + } + } + } +} diff --git a/scripts/ksail-arc-admission-fence.sh b/scripts/ksail-arc-admission-fence.sh new file mode 100644 index 0000000000..a288849cdb --- /dev/null +++ b/scripts/ksail-arc-admission-fence.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +# Helpers for the protected canary. Loaded only after deployment identity checks. +# API diagnostics and job metadata stay in the caller's private scratch directory. +# The canary supplies these checked variables and kc/quiet/deadline helpers. +# shellcheck disable=SC2154 + +readonly arc_origin_key=platform.devantler.tech/arc-runtime-origin +readonly arc_probe_uid_key=platform.devantler.tech/arc-runtime-probe-uid + +arc_actions_read() { + [[ -n "${GH_TOKEN:-}" ]] || return 1 + timeout 60s gh api "$1" --hostname github.com --method GET \ + -H 'Accept: application/vnd.github+json' -H 'X-GitHub-Api-Version: 2022-11-28' \ + "${@:3}" >"$2" 2>"$scratch/actions-error" || return 1 + jq -e -s 'length == 1' "$2" >/dev/null +} + +arc_attempt_identity() { + local origin=$1 terminal=$2 run attempt workflow head job job_name + run=$(jq -er '.run' "$origin") + attempt=$(jq -er '.attempt' "$origin") + workflow=$(jq -er '.workflow' "$origin") + head=$(jq -er '.head' "$origin") + job=$(jq -er '.job' "$origin") + job_name=$(jq -er '.job_name' "$origin") + arc_actions_read "repos/devantler-tech/platform/actions/runs/$run/attempts/$attempt" "$scratch/attempt.json" + jq -e --arg run "$run" --arg attempt "$attempt" --arg workflow "$workflow" --arg head "$head" --arg terminal "$terminal" ' + (.id | tostring) == $run and (.run_attempt | tostring) == $attempt and .head_sha == $head and + .path == $workflow and .repository.full_name == "devantler-tech/platform" and + .head_repository.full_name == "devantler-tech/platform" and + ((.path == ".github/workflows/ci.yaml" and .event == "merge_group") or + (.path == ".github/workflows/cd.yaml" and .event == "workflow_dispatch")) and + (if $terminal == "true" then .status == "completed" and .conclusion != null + else .status == "in_progress" end) + ' "$scratch/attempt.json" >/dev/null + arc_actions_read "repos/devantler-tech/platform/actions/runs/$run/attempts/$attempt/jobs?per_page=100" \ + "$scratch/attempt-jobs.json" --paginate --slurp + # Pagination must be complete and consistent, with no duplicate job identity. + jq -e --arg run "$run" --arg head "$head" --arg job "$job" --arg name "$job_name" --arg terminal "$terminal" ' + length > 0 and length <= 25 and + (.[0].total_count as $total | all(.[]; .total_count == $total) and + ([.[].jobs[]] | length) == $total) and + ([.[].jobs[].id] | length == (unique | length)) and + ([.[].jobs[] | select((.id | tostring) == $job and .name == $name and + (.run_id | tostring) == $run and .head_sha == $head and + (if $terminal == "true" then .status == "completed" and .conclusion != null and + (.completed_at | type == "string" and length > 0) + else .status == "in_progress" end))] | length) == 1 + ' "$scratch/attempt-jobs.json" >/dev/null +} + +arc_current_origin() { + local workflow job_name source job + [[ "${GITHUB_SHA:-}" =~ ^[0-9a-f]{40}$ && + "$GITHUB_RUN_ID" =~ ^[1-9][0-9]{0,14}$ ]] || return 1 + case "${GITHUB_WORKFLOW_REF:-}:${GITHUB_JOB:-}" in + devantler-tech/platform/.github/workflows/ci.yaml@*:deploy-prod) + workflow=.github/workflows/ci.yaml; job_name='๐Ÿš€ Deploy to Prod' ;; + devantler-tech/platform/.github/workflows/cd.yaml@*:deploy-prod) + workflow=.github/workflows/cd.yaml; job_name='๐Ÿš€ Deploy to Production' ;; + devantler-tech/platform/.github/workflows/ci.yaml@*:heal-prod-on-failure) + workflow=.github/workflows/ci.yaml + job_name='๐Ÿฉน Heal Prod (restore main after an unsuccessful or evicted merge-group deploy)' ;; + *) return 1 ;; + esac + source=$(git rev-parse HEAD) + [[ "$source" =~ ^[0-9a-f]{40}$ ]] || return 1 + [[ "$source" == "$GITHUB_SHA" || "$GITHUB_JOB" == heal-prod-on-failure ]] || return 1 + arc_actions_read "repos/devantler-tech/platform/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100" \ + "$scratch/current-jobs.json" --paginate --slurp + job=$(jq -er --arg name "$job_name" --arg run "$GITHUB_RUN_ID" --arg head "$GITHUB_SHA" ' + [.[].jobs[] | select(.name == $name and (.run_id | tostring) == $run and + .head_sha == $head and .status == "in_progress")] | + select(length == 1) | .[0].id | tostring | select(test("^[1-9][0-9]{0,14}$")) + ' "$scratch/current-jobs.json") + jq -n --arg workflow "$workflow" --arg run "$GITHUB_RUN_ID" --arg attempt "$GITHUB_RUN_ATTEMPT" \ + --arg head "$GITHUB_SHA" --arg source "$source" --arg manifest "$PLATFORM_MANIFEST_DIGEST" \ + --arg job "$job" --arg job_name "$job_name" '{repository:"devantler-tech/platform", + workflow:$workflow,run:$run,attempt:$attempt,head:$head,source:$source, + manifest:$manifest,job:$job,job_name:$job_name}' >"$scratch/current-origin.json" + arc_attempt_identity "$scratch/current-origin.json" false +} + +arc_retained_fence_matches() { + kc -n "$namespace" get resourcequota "$fence_name" -o json >"$scratch/retained-fence-current.json" || return 1 + [[ -s "$scratch/retained-fence-current.json" ]] || return 1 + jq -e --slurpfile before "$scratch/retained-fence.json" ' + .metadata.uid == $before[0].metadata.uid and + .metadata.labels == $before[0].metadata.labels and + .metadata.annotations == $before[0].metadata.annotations and + .metadata.deletionTimestamp == null and (.metadata.ownerReferences // [] | length) == 0 and + .spec == {hard:{pods:"0"},scopes:["NotTerminating"]} and .status.hard == {pods:"0"} + ' "$scratch/retained-fence-current.json" >/dev/null +} + +arc_recover_admission_fence() { + local owner run attempt recorded_uid current deadline remaining + kc -n "$namespace" get resourcequota "$fence_name" --ignore-not-found -o json >"$scratch/retained-fence.json" + [[ -s "$scratch/retained-fence.json" ]] || return 0 + jq -e --arg name "$fence_name" --arg ns "$namespace" ' + .metadata.name == $name and .metadata.namespace == $ns and + (.metadata.uid | type == "string" and length > 0) and + (.metadata.resourceVersion | type == "string" and length > 0) and + .metadata.deletionTimestamp == null and (.metadata.ownerReferences // [] | length) == 0 and + .spec == {hard:{pods:"0"},scopes:["NotTerminating"]} and .status.hard == {pods:"0"} + ' "$scratch/retained-fence.json" >/dev/null + jq -er --arg key "$arc_origin_key" '.metadata.annotations[$key] | fromjson' \ + "$scratch/retained-fence.json" >"$scratch/prior-origin.json" + jq -e 'type == "object" and .repository == "devantler-tech/platform" and + (.run | test("^[1-9][0-9]{0,14}$")) and (.attempt | test("^[1-9][0-9]{0,2}$")) and + (.job | test("^[1-9][0-9]{0,14}$")) and + (.head | test("^[0-9a-f]{40}$")) and (.source | test("^[0-9a-f]{40}$")) and + (.manifest | test("^sha256:[0-9a-f]{64}$")) and + ((.workflow == ".github/workflows/ci.yaml" and .job_name == "๐Ÿš€ Deploy to Prod" and .source == .head) or + (.workflow == ".github/workflows/cd.yaml" and .job_name == "๐Ÿš€ Deploy to Production" and .source == .head) or + (.workflow == ".github/workflows/ci.yaml" and + .job_name == "๐Ÿฉน Heal Prod (restore main after an unsuccessful or evicted merge-group deploy)")) + ' "$scratch/prior-origin.json" >/dev/null + run=$(jq -r '.run' "$scratch/prior-origin.json") + attempt=$(jq -r '.attempt' "$scratch/prior-origin.json") + owner="arc-proof-$run-$attempt" + [[ "$owner" != "$probe" ]] || return 1 + jq -e --arg key "$ownership_label" --arg owner "$owner" '.metadata.labels[$key] == $owner' \ + "$scratch/retained-fence.json" >/dev/null + # Whole-attempt death is deliberately required. Same-run Heal Prod remains + # HOLD; its recovery support must be staged on inactive main before activation. + arc_attempt_identity "$scratch/prior-origin.json" true + verify_runner_deadlines + arc_retained_fence_matches + current=$(kc -n "$namespace" get pod "$owner" --ignore-not-found -o json) + if [[ -n "$current" ]]; then + recorded_uid=$(jq -er --arg key "$arc_probe_uid_key" \ + '.metadata.annotations[$key] | select(type == "string" and length > 0)' "$scratch/retained-fence.json") + jq -e --arg name "$owner" --arg ns "$namespace" --arg key "$ownership_label" --arg uid "$recorded_uid" ' + .metadata.name == $name and .metadata.namespace == $ns and .metadata.labels[$key] == $name and + .metadata.uid == $uid and (.metadata.ownerReferences // [] | length) == 0 and + .spec.activeDeadlineSeconds == 1200 and .spec.restartPolicy == "Never" + ' <<<"$current" >/dev/null + jq -n --arg uid "$recorded_uid" '{apiVersion:"v1",kind:"DeleteOptions",preconditions:{uid:$uid}, + gracePeriodSeconds:5,propagationPolicy:"Background"}' >"$scratch/prior-delete.json" + arc_retained_fence_matches + quiet kc delete --raw="/api/v1/namespaces/$namespace/pods/$owner" -f "$scratch/prior-delete.json" + quiet kc -n "$namespace" wait "pod/$owner" --for=delete --timeout=25s + fi + current=$(kc -n "$namespace" get pod "$owner" --ignore-not-found -o json) + [[ -z "$current" ]] + kc -n "$namespace" get pods -l "$ownership_label=$owner" -o json >"$scratch/prior-probes.json" + jq -e '(.metadata.continue // "") == "" and (.items | length == 0)' "$scratch/prior-probes.json" >/dev/null + deadline=$((SECONDS + 1200)) + while ((SECONDS < deadline)); do + arc_retained_fence_matches + remaining=$(kc get nodes -o json) + if jq -e '(.metadata.continue // "") == "" and (.items | all(.[]; + .metadata.labels["platform.devantler.tech/ci-runner"] != "enabled" and + (.metadata.name | startswith("autoscale-arc-runners-") | not)))' <<<"$remaining" >/dev/null; then break; fi + sleep 10 + done + jq -e '(.metadata.continue // "") == "" and (.items | all(.[]; + .metadata.labels["platform.devantler.tech/ci-runner"] != "enabled" and + (.metadata.name | startswith("autoscale-arc-runners-") | not)))' <<<"$remaining" >/dev/null + arc_attempt_identity "$scratch/prior-origin.json" true + verify_runner_deadlines + arc_retained_fence_matches + current=$(kc -n "$namespace" get pod "$owner" --ignore-not-found -o json) + [[ -z "$current" ]] + kc -n "$namespace" get pods -l "$ownership_label=$owner" -o json >"$scratch/prior-probes.json" + jq -e '(.metadata.continue // "") == "" and (.items | length == 0)' "$scratch/prior-probes.json" >/dev/null + jq '{apiVersion:"v1",kind:"DeleteOptions",preconditions:{uid:.metadata.uid, + resourceVersion:.metadata.resourceVersion},propagationPolicy:"Background"}' \ + "$scratch/retained-fence-current.json" >"$scratch/prior-delete-fence.json" + quiet kc delete --raw="/api/v1/namespaces/$namespace/resourcequotas/$fence_name" -f "$scratch/prior-delete-fence.json" + current=$(kc -n "$namespace" get resourcequota "$fence_name" --ignore-not-found -o json) + [[ -z "$current" ]] + printf 'ARC acceptance: abandoned admission fence recovered\n' +} + +arc_record_probe_uid() { + verify_admission_fence + jq -n --arg uid "$fence_uid" --arg version "$(jq -r '.metadata.resourceVersion' "$scratch/live-fence.json")" \ + --arg probe_uid "$probe_uid" '[{op:"test",path:"/metadata/uid",value:$uid}, + {op:"test",path:"/metadata/resourceVersion",value:$version}, + {op:"add",path:"/metadata/annotations/platform.devantler.tech~1arc-runtime-probe-uid",value:$probe_uid}]' \ + >"$scratch/probe-uid-patch.json" + quiet kc -n "$namespace" patch resourcequota "$fence_name" --type=json --patch-file "$scratch/probe-uid-patch.json" + verify_admission_fence + jq -e --arg key "$arc_probe_uid_key" --arg uid "$probe_uid" '.metadata.annotations[$key] == $uid' \ + "$scratch/live-fence.json" >/dev/null +} diff --git a/scripts/ksail-arc-job-metrics.sh b/scripts/ksail-arc-job-metrics.sh new file mode 100755 index 0000000000..4a09f7d6e8 --- /dev/null +++ b/scripts/ksail-arc-job-metrics.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euo pipefail + +arc_metrics_fail() { + printf '::error::KSail ARC whole-job cgroup measurement failed: %s\n' "$1" >&2 + return 1 +} + +arc_metrics_read() { + local root=$1 name=$2 + if [[ "$root" == /sys/fs/cgroup ]]; then + timeout 5s cat "$root/$name" 2>/dev/null + else + # Only regular test-owned fixture files use the portable read path. + [[ -f "$root/$name" && ! -L "$root/$name" ]] || return 1 + cat "$root/$name" 2>/dev/null + fi +} + +# The production entrypoint uses only the container's kernel cgroup. A supplied +# fixture directory is used when this function is sourced by offline tests. +arc_job_metrics() { + local root=$1 limit peak events key value extra + local low='' high='' max='' oom='' oom_kill='' oom_group_kill='' sock_throttled='' + limit=$(arc_metrics_read "$root" memory.max) || { arc_metrics_fail read-limit; return 1; } + peak=$(arc_metrics_read "$root" memory.peak) || { arc_metrics_fail read-peak; return 1; } + events=$(arc_metrics_read "$root" memory.events) || { arc_metrics_fail read-events; return 1; } + [[ "$limit" == 15032385536 ]] || { arc_metrics_fail limit; return 1; } + [[ "$peak" =~ ^(0|[1-9][0-9]{0,17})$ && "$peak" -gt 0 ]] || { + arc_metrics_fail peak; return 1; + } + while read -r key value extra; do + [[ -z "$extra" && "$value" =~ ^(0|[1-9][0-9]{0,17})$ ]] || { arc_metrics_fail event-format; return 1; } + case "$key" in + low|high|max|oom|oom_kill|oom_group_kill) + [[ -z "${!key}" ]] || { arc_metrics_fail duplicate-event; return 1; } + printf -v "$key" '%s' "$value" ;; + sock_throttled) + [[ -z "$sock_throttled" ]] || { arc_metrics_fail duplicate-event; return 1; } + sock_throttled=$value ;; + *) arc_metrics_fail unknown-event; return 1 ;; + esac + done <<<"$events" + [[ -n "$low" && -n "$high" && -n "$max" && -n "$oom" && -n "$oom_kill" && -n "$oom_group_kill" ]] || { + arc_metrics_fail missing-event; return 1; + } + [[ "$oom" == 0 && "$oom_kill" == 0 && "$oom_group_kill" == 0 ]] || { arc_metrics_fail oom; return 1; } + printf 'KSail ARC whole-job cgroup: {"schemaVersion":1,"memoryMaxBytes":%s,"memoryPeakBytes":%s,"limitEvents":%s,"oomEvents":%s,"oomKills":%s,"oomGroupKills":%s}\n' \ + "$limit" "$peak" "$max" "$oom" "$oom_kill" "$oom_group_kill" + # The kernel may temporarily exceed memory.max. Retain that valid measurement, + # while preserving the existing runtime acceptance ceiling as a separate refusal. + [[ "$peak" -le "$limit" ]] || { arc_metrics_fail budget; return 1; } +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + [[ $# == 0 ]] || { arc_metrics_fail arguments; exit 1; } + arc_job_metrics /sys/fs/cgroup +fi diff --git a/scripts/tests/arc-staging/activation_test.go b/scripts/tests/arc-staging/activation_test.go new file mode 100644 index 0000000000..6bcc50e3d6 --- /dev/null +++ b/scripts/tests/arc-staging/activation_test.go @@ -0,0 +1,249 @@ +package arcstaging_test + +import ( + "fmt" + "io/fs" + "os" + "regexp" + "strings" + "testing" + "testing/fstest" + + "gopkg.in/yaml.v3" +) + +const controllerAggregate = "k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" +const runnerAggregate = "k8s/providers/hetzner/infrastructure/kustomization.yaml" +const controllerReference = "../../../../bases/infrastructure/controllers/actions-runner-controller/" +const runnerReference = "../../../bases/infrastructure/actions-runners/" +const controllerRelease = "k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml" +const runnerRelease = "k8s/bases/infrastructure/actions-runners/helm-release.yaml" +const runtimeAction = ".github/actions/deploy-prod/action.yml" + +func TestDeploymentAggregatesKeepARCInsideActivationEnvelope(t *testing.T) { + if err := validateARCActivation(os.DirFS(repoRoot)); err != nil { + t.Fatal(err) + } +} + +func activationFixture(active bool) fstest.MapFS { + suspended, controller, runner := "true", "[]", "[]" + image := "ghcr.io/actions/actions-runner@sha256:" + strings.Repeat("a", 64) + if active { + suspended = "false" + controller = fmt.Sprintf("[%q]", controllerReference) + runner = fmt.Sprintf("[%q]", runnerReference) + image = analysisRepository + "@sha256:" + strings.Repeat("b", 64) + } + files := fstest.MapFS{} + for name, data := range map[string]string{ + controllerAggregate: "resources: " + controller + "\n", + runnerAggregate: "resources: " + runner + "\n", + controllerRelease: "spec:\n suspend: " + suspended + "\n", + runnerRelease: "spec:\n suspend: " + suspended + "\n values:\n template:\n spec:\n" + + " containers: [{image: " + image + "}]\n initContainers: [{image: " + image + "}]\n", + runtimeAction: "runs:\n steps:\n - name: Verify the deployed KSail ARC canary\n" + + " if: >-\n !cancelled() && steps.wait_flux_revision.outcome == 'success' && " + + "( steps.cluster_update.outcome == 'skipped' || steps.wait_prod_api_stability.outcome == 'success' )\n" + + " shell: bash\n env:\n GH_TOKEN: ${{ github.token }}\n PLATFORM_MANIFEST_DIGEST: ${{ steps.publish_platform_manifest.outputs.digest }}\n" + + " run: |\n export KUBECONFIG=\"${HOME}/.kube/config\"\n bash scripts/verify-ksail-arc-runtime.sh --if-active\n", + } { + files[name] = &fstest.MapFile{Data: []byte(data)} + } + return files +} + +func TestActivationEnvelopeRejectsUnboundedOrUnverifiedStates(t *testing.T) { + for _, active := range []bool{false, true} { + if err := validateARCActivation(activationFixture(active)); err != nil { + t.Fatalf("valid active=%t: %v", active, err) + } + } + tests := []struct { + name, file, from, to string + }{ + {"controller only", runnerAggregate, runnerReference, "unrelated"}, + {"runner only", controllerAggregate, controllerReference, "unrelated"}, + {"duplicate activation", runnerAggregate, "resources:", "components: [\"" + runnerReference + "\"]\nresources:"}, + {"component activation", runnerAggregate, "resources:", "components:"}, + {"legacy base activation", runnerAggregate, "resources:", "bases:"}, + {"suspended controller", controllerRelease, "suspend: false", "suspend: true"}, + {"suspended runner", runnerRelease, "suspend: false", "suspend: true"}, + {"upstream image active", runnerRelease, analysisRepository, "ghcr.io/actions/actions-runner"}, + {"mutable image", runnerRelease, "@sha256:" + strings.Repeat("b", 64), ":latest"}, + {"zero image digest", runnerRelease, strings.Repeat("b", 64), strings.Repeat("0", 64)}, + {"missing canary", runtimeAction, "Verify the deployed KSail ARC canary", "Unrelated step"}, + {"skipped canary", runtimeAction, "!cancelled() &&", "false &&"}, + {"waived canary", runtimeAction, " shell: bash", " continue-on-error: true\n shell: bash"}, + {"wrong deployment digest", runtimeAction, "steps.publish_platform_manifest.outputs.digest", "github.sha"}, + {"missing Actions read token", runtimeAction, " GH_TOKEN: ${{ github.token }}\n", ""}, + {"wrong Actions read token", runtimeAction, "GH_TOKEN: ${{ github.token }}", "GH_TOKEN: ${{ secrets.GHCR_TOKEN }}"}, + {"unexpected proof environment", runtimeAction, " env:", " env:\n PATH: /unverified"}, + {"missing suspension declaration", runnerRelease, "suspend: false", "unrelated: false"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + files := activationFixture(true) + files[test.file].Data = []byte(strings.ReplaceAll(string(files[test.file].Data), test.from, test.to)) + if err := validateARCActivation(files); err == nil { + t.Fatal("accepted invalid activation") + } + }) + } + for _, test := range []struct{ name, path string }{ + {"other cluster", "k8s/clusters/local/kustomization.yaml"}, + {"other production layer", "k8s/providers/hetzner/apps/kustomization.yaml"}, + {"alternate yaml filename", "k8s/clusters/local/kustomization.yml"}, + {"alternate canonical filename", "k8s/clusters/local/Kustomization"}, + } { + t.Run(test.name, func(t *testing.T) { + files := activationFixture(true) + files[test.path] = &fstest.MapFile{Data: []byte("resources: [\"" + runnerReference + "\"]\n")} + if err := validateARCActivation(files); err == nil { + t.Fatal("accepted activation from another aggregate") + } + }) + } + files := activationFixture(false) + files[runnerRelease].Data = []byte(strings.ReplaceAll(string(files[runnerRelease].Data), "suspend: true", "suspend: false")) + if err := validateARCActivation(files); err == nil { + t.Fatal("accepted unsuspended inactive runner") + } +} + +func TestActivationEnvelopeRetainsOnlyASuspendedControllerWithoutAPool(t *testing.T) { + files := activationFixture(false) + files[controllerAggregate].Data = []byte(fmt.Sprintf("resources: [%q]\n", controllerReference)) + if err := validateARCActivation(files); err != nil { + t.Fatalf("suspended controller inventory must remain recoverable: %v", err) + } + files[controllerRelease].Data = []byte("spec:\n suspend: false\n") + if err := validateARCActivation(files); err == nil { + t.Fatal("accepted an active controller without the verified pool") + } +} + +func validateARCActivation(files fs.FS) error { + counts := map[string]int{controllerAggregate: 0, runnerAggregate: 0} + allowed := map[string]string{controllerAggregate: controllerReference, runnerAggregate: runnerReference} + err := fs.WalkDir(files, "k8s", func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || (entry.Name() != "kustomization.yaml" && entry.Name() != "kustomization.yml" && entry.Name() != "Kustomization") || + strings.HasPrefix(name, strings.TrimSuffix(controllerRelease, "/helm-release.yaml")+"/") || + strings.HasPrefix(name, strings.TrimSuffix(runnerRelease, "/helm-release.yaml")+"/") { + return nil + } + var aggregate struct { + Resources []string `yaml:"resources"` + Components []string `yaml:"components"` + Bases []string `yaml:"bases"` + } + if err := activationYAML(files, name, &aggregate); err != nil { + return err + } + for _, reference := range append(aggregate.Components, aggregate.Bases...) { + if strings.Contains(reference, "actions-runner-controller") || strings.Contains(reference, "actions-runners") { + return fmt.Errorf("ARC activation must be an explicit resource, not a component or legacy base: %s", name) + } + } + for _, reference := range aggregate.Resources { + if !strings.Contains(reference, "actions-runner-controller") && !strings.Contains(reference, "actions-runners") { + continue + } + if expected, ok := allowed[name]; !ok || reference != expected { + return fmt.Errorf("ARC reference outside approved aggregate: %s", name) + } + counts[name]++ + } + return nil + }) + if err != nil { + return err + } + active := counts[controllerAggregate] == 1 && counts[runnerAggregate] == 1 + retainedController := counts[controllerAggregate] == 1 && counts[runnerAggregate] == 0 + if !active && !retainedController && (counts[controllerAggregate] != 0 || counts[runnerAggregate] != 0) { + return fmt.Errorf("partial or duplicate ARC activation") + } + var image string + for _, name := range []string{controllerRelease, runnerRelease} { + var release struct { + Spec struct { + Suspend *bool `yaml:"suspend"` + Values struct { + Template struct { + Spec struct { + Containers []struct{ Image string } `yaml:"containers"` + InitContainers []struct{ Image string } `yaml:"initContainers"` + } `yaml:"spec"` + } `yaml:"template"` + } `yaml:"values"` + } `yaml:"spec"` + } + if err := activationYAML(files, name, &release); err != nil { + return err + } + if release.Spec.Suspend == nil || *release.Spec.Suspend == active { + return fmt.Errorf("ARC release suspension does not match aggregate activation: %s", name) + } + if name == runnerRelease { + containers := release.Spec.Values.Template.Spec.Containers + init := release.Spec.Values.Template.Spec.InitContainers + if len(containers) != 1 || len(init) != 1 || containers[0].Image != init[0].Image { + return fmt.Errorf("runner and init images must have one identical immutable pin") + } + image = containers[0].Image + } + } + repository := regexp.QuoteMeta(analysisRepository) + if !active { + repository += "|" + regexp.QuoteMeta("ghcr.io/actions/actions-runner") + } + if !regexp.MustCompile("^("+repository+")@sha256:[0-9a-f]{64}$").MatchString(image) || strings.HasSuffix(image, strings.Repeat("0", 64)) { + return fmt.Errorf("ARC image is outside the immutable publication envelope") + } + return validateARCRuntimeHook(files) +} + +func activationYAML(files fs.FS, name string, target any) error { + data, err := fs.ReadFile(files, name) + if err != nil { + return err + } + return yaml.Unmarshal(data, target) +} + +func validateARCRuntimeHook(files fs.FS) error { + var action struct { + Runs struct { + Steps []map[string]any `yaml:"steps"` + } `yaml:"runs"` + } + if err := activationYAML(files, runtimeAction, &action); err != nil { + return err + } + count := 0 + condition := "!cancelled() && steps.wait_flux_revision.outcome == 'success' && ( steps.cluster_update.outcome == 'skipped' || steps.wait_prod_api_stability.outcome == 'success' )" + command := "export KUBECONFIG=\"${HOME}/.kube/config\"\nbash scripts/verify-ksail-arc-runtime.sh --if-active" + for _, step := range action.Runs.Steps { + if step["name"] != "Verify the deployed KSail ARC canary" { + continue + } + count++ + _, waived := step["continue-on-error"] + guard, ok := step["if"].(string) + run, runOK := step["run"].(string) + env, envOK := step["env"].(map[string]any) + if waived || !ok || strings.Join(strings.Fields(guard), " ") != condition || !runOK || strings.TrimSpace(run) != command || + step["shell"] != "bash" || !envOK || len(env) != 2 || env["GH_TOKEN"] != "${{ github.token }}" || + env["PLATFORM_MANIFEST_DIGEST"] != "${{ steps.publish_platform_manifest.outputs.digest }}" { + return fmt.Errorf("ARC runtime proof is skipped, waived, or bound to the wrong deployment") + } + } + if count != 1 { + return fmt.Errorf("exactly one protected ARC runtime proof is required") + } + return nil +} diff --git a/scripts/tests/arc-staging/job_metrics_test.go b/scripts/tests/arc-staging/job_metrics_test.go new file mode 100644 index 0000000000..3e1804eb9d --- /dev/null +++ b/scripts/tests/arc-staging/job_metrics_test.go @@ -0,0 +1,18 @@ +package arcstaging_test + +import "testing" + +const metricsComponent = "k8s/bases/infrastructure/actions-runners" + +func TestRunnerRetainsWholeJobCgroupMeasurement(t *testing.T) { + config := readYAML(t, metricsComponent+"/config-map-job-metrics.yaml") + equal(t, field(t, config, "metadata", "namespace"), "arc-runners") + equal(t, field(t, config, "immutable"), true) + equal(t, field(t, config, "metadata", "annotations", "kustomize.toolkit.fluxcd.io/substitute"), "disabled") + release := readYAML(t, metricsComponent+"/helm-release.yaml") + runner := field(t, release, "spec", "values", "template", "spec", "containers").([]any)[0] + env := field(t, runner, "env").([]any) + equal(t, len(env), 1) + equal(t, field(t, env[0], "name"), "ACTIONS_RUNNER_HOOK_JOB_COMPLETED") + equal(t, field(t, env[0], "value"), "/etc/ksail-arc-metrics/job-metrics.sh") +} diff --git a/scripts/tests/arc-staging/runner_group_activation_test.go b/scripts/tests/arc-staging/runner_group_activation_test.go index c7ce0027b1..fad4c688d5 100644 --- a/scripts/tests/arc-staging/runner_group_activation_test.go +++ b/scripts/tests/arc-staging/runner_group_activation_test.go @@ -12,7 +12,7 @@ import ( const arcActivationFile = "managed-resource-activation-policy-arc.yaml" const arcRunnerGroupResource = "runnergroups.actions.github.m.upbound.io" -func validateARCActivation(policy map[string]any, resources []any) error { +func validateARCRunnerGroupActivation(policy map[string]any, resources []any) error { if policy["apiVersion"] != "apiextensions.crossplane.io/v1alpha1" || policy["kind"] != "ManagedResourceActivationPolicy" { return fmt.Errorf("expected a provider activation policy") } @@ -71,7 +71,7 @@ func TestARCResourceIsExplicitlyActivated(t *testing.T) { const component = "k8s/providers/hetzner/infrastructure/crossplane/" policy := readYAML(t, component+arcActivationFile) resources := field(t, readYAML(t, component+"kustomization.yaml"), "resources").([]any) - if err := validateARCActivation(policy, resources); err != nil { + if err := validateARCRunnerGroupActivation(policy, resources); err != nil { t.Fatal(err) } } @@ -95,7 +95,7 @@ func TestARCActivationRejectsBroaderResourcesAndMissingInclusion(t *testing.T) { "metadata": map[string]any{"name": "github-arc-runners"}, "spec": map[string]any{"activate": test.activation}, } - if err := validateARCActivation(policy, test.resources); err == nil { + if err := validateARCRunnerGroupActivation(policy, test.resources); err == nil { t.Fatal("accepted an unsafe or unreconciled activation policy") } }) diff --git a/scripts/tests/arc-staging/runner_group_test.go b/scripts/tests/arc-staging/runner_group_test.go new file mode 100644 index 0000000000..b1dd16dcc3 --- /dev/null +++ b/scripts/tests/arc-staging/runner_group_test.go @@ -0,0 +1,102 @@ +package arcstaging_test + +import ( + "bytes" + "encoding/json" + "reflect" + "strings" + "testing" + "text/template" +) + +const runtimeProviderName = "arc-runtime-platform-app" +const acceptedWorkflow = "devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main" + +func TestRunnerGroupUsesOnlyTheRuntimeAppInsideItsNamespace(t *testing.T) { + const component = "k8s/bases/infrastructure/actions-runners/" + config := readYAML(t, component+"provider-config.yaml") + equal(t, field(t, config, "apiVersion"), "github.m.upbound.io/v1beta1") + equal(t, field(t, config, "kind"), "ProviderConfig") + equal(t, field(t, config, "metadata", "name"), runtimeProviderName) + equal(t, field(t, config, "metadata", "namespace"), "arc-runners") + credentials := field(t, config, "spec", "credentials") + equal(t, field(t, credentials, "source"), "Secret") + equal(t, field(t, credentials, "secretRef", "namespace"), "arc-runners") + equal(t, field(t, credentials, "secretRef", "name"), "arc-github-app") + equal(t, field(t, credentials, "secretRef", "key"), "provider-credentials") + group := readYAML(t, component+"runner-group.yaml") + equal(t, field(t, group, "apiVersion"), "actions.github.m.upbound.io/v1alpha1") + equal(t, field(t, group, "kind"), "RunnerGroup") + equal(t, field(t, group, "metadata", "name"), "platform") + equal(t, field(t, group, "metadata", "namespace"), "arc-runners") + if !reflect.DeepEqual(field(t, group, "spec", "managementPolicies"), []any{"Observe", "Create", "Update"}) { + t.Fatal("runner-group ownership must exclude deletion and late initialization") + } + if !reflect.DeepEqual(field(t, group, "spec", "providerConfigRef"), map[string]any{"name": runtimeProviderName, "kind": "ProviderConfig"}) { + t.Fatal("runner group must use its distinct runtime-App provider") + } + parameters := field(t, group, "spec", "forProvider") + if !reflect.DeepEqual(parameters, map[string]any{ + "name": "platform", "visibility": "selected", "allowsPublicRepositories": true, + "restrictedToWorkflows": true, "selectedRepositoryIds": []any{737584922}, + "selectedWorkflows": []any{acceptedWorkflow}, + }) { + t.Fatal("initial access must admit only KSail's main-only delivery preflight") + } + resources := field(t, readYAML(t, component+"kustomization.yaml"), "resources").([]any) + for _, required := range []string{"provider-config.yaml", "runner-group.yaml"} { + count := 0 + for _, resource := range resources { + if resource == required { + count++ + } + } + if count != 1 { + t.Fatalf("%s must be included exactly once", required) + } + } +} + +func TestProviderCredentialTemplatePreservesARCKeysAndEscapesTheExistingPEM(t *testing.T) { + secret := readYAML(t, "k8s/bases/infrastructure/actions-runners/external-secret.yaml") + configuration := field(t, secret, "spec", "target", "template") + equal(t, field(t, configuration, "engineVersion"), "v2") + equal(t, field(t, configuration, "mergePolicy"), "Merge") + data := field(t, configuration, "data").(map[string]any) + if len(data) != 1 { + t.Fatal("template must add only the provider's JSON credential key") + } + source := field(t, data, "provider-credentials").(string) + parsed, err := template.New("credentials").Option("missingkey=error").Funcs(template.FuncMap{ + "toJson": func(value any) (string, error) { + encoded, err := json.Marshal(value) + return string(encoded), err + }, + }).Parse(source) + if err != nil { + t.Fatal(err) + } + for _, pem := range []string{"synthetic-key\nsecond-line\n", "synthetic\"key\\with\ncontrols\t"} { + inputs := map[string]string{ + "github_app_id": "123", "github_app_installation_id": "456", "github_app_private_key": pem, + } + var output bytes.Buffer + if err := parsed.Execute(&output, inputs); err != nil { + t.Fatal(err) + } + var value map[string]any + if err := json.Unmarshal(output.Bytes(), &value); err != nil { + t.Fatalf("credential JSON cannot round-trip a PEM: %v", err) + } + want := map[string]any{"owner": "devantler-tech", "app_auth": []any{ + map[string]any{"id": "123", "installation_id": "456", "pem_file": pem}, + }} + if !reflect.DeepEqual(value, want) { + t.Fatal("provider credentials changed the existing App fields") + } + } + var output bytes.Buffer + if err := parsed.Execute(&output, map[string]string{}); err == nil || !strings.Contains(err.Error(), "github_app_") { + t.Fatal("missing existing App fields must fail the credential template") + } +} diff --git a/scripts/tests/arc-staging/runtime_test.go b/scripts/tests/arc-staging/runtime_test.go index 000209f7d7..1674212bb9 100644 --- a/scripts/tests/arc-staging/runtime_test.go +++ b/scripts/tests/arc-staging/runtime_test.go @@ -52,18 +52,23 @@ func assertRunnerStorage(t *testing.T, spec any) { wantVolumes := []any{ map[string]any{"name": "runner-home", "emptyDir": map[string]any{"sizeLimit": "40Gi"}}, map[string]any{"name": "runner-tmp", "emptyDir": map[string]any{"sizeLimit": "2Gi"}}, + map[string]any{"name": "runner-metrics", "configMap": map[string]any{ + "name": "ksail-arc-job-metrics", "defaultMode": 365, + "items": []any{map[string]any{"key": "job-metrics.sh", "path": "job-metrics.sh"}}, + }}, } if !reflect.DeepEqual(field(t, spec, "volumes"), wantVolumes) { - t.Fatal("only two explicitly bounded disk emptyDirs are permitted") + t.Fatal("only two bounded disk emptyDirs and the immutable metrics hook are permitted") } runner := field(t, spec, "containers").([]any)[0] equal(t, field(t, runner, "securityContext", "readOnlyRootFilesystem"), true) wantMounts := []any{ map[string]any{"name": "runner-home", "mountPath": "/home/runner"}, map[string]any{"name": "runner-tmp", "mountPath": "/tmp"}, + map[string]any{"name": "runner-metrics", "mountPath": "/etc/ksail-arc-metrics", "readOnly": true}, } if !reflect.DeepEqual(field(t, runner, "volumeMounts"), wantMounts) { - t.Fatal("runner may mount only its disposable home and temporary storage") + t.Fatal("runner may mount only disposable storage and its read-only metrics hook") } initializers := field(t, spec, "initContainers").([]any) equal(t, len(initializers), 1) diff --git a/scripts/tests/arc-staging/staging_test.go b/scripts/tests/arc-staging/staging_test.go index 06330e75a5..a8782c7afa 100644 --- a/scripts/tests/arc-staging/staging_test.go +++ b/scripts/tests/arc-staging/staging_test.go @@ -48,14 +48,14 @@ func equal(t *testing.T, value, want any) { } } -func TestRecoveryControllerReconcilesWhileOrganizationPoolStaysSuspended(t *testing.T) { +func TestReleasesUsePinnedCharts(t *testing.T) { for _, component := range []string{ "k8s/bases/infrastructure/controllers/actions-runner-controller", "k8s/bases/infrastructure/actions-runners", } { t.Run(filepath.Base(component), func(t *testing.T) { release := readYAML(t, component+"/helm-release.yaml") - equal(t, field(t, release, "spec", "suspend"), filepath.Base(component) == "actions-runners") + equal(t, field(t, release, "spec", "suspend"), false) equal(t, field(t, release, "spec", "chartRef", "kind"), "OCIRepository") source := readYAML(t, component+"/oci-repository.yaml") equal(t, field(t, source, "spec", "ref", "tag"), "0.15.0") @@ -105,7 +105,8 @@ func TestPoolCannotCreateUnboundedOrPrivilegedRunners(t *testing.T) { runner := containers[0] equal(t, field(t, runner, "name"), "runner") image, ok := field(t, runner, "image").(string) - if !ok || !strings.HasPrefix(image, "ghcr.io/actions/actions-runner@sha256:") { + if !ok || (!strings.HasPrefix(image, "ghcr.io/actions/actions-runner@sha256:") && + !strings.HasPrefix(image, analysisRepository+"@sha256:")) { t.Fatal("runner image must be digest-pinned") } for _, name := range []string{"privileged", "allowPrivilegeEscalation"} { @@ -308,21 +309,24 @@ func TestStagingGuardRunsUnconditionallyOnPullRequestsAndMergeGroups(t *testing. } for _, step := range field(t, changes, "steps").([]any) { mapping := step.(map[string]any) - if mapping["run"] == "go test ./scripts/tests/arc-staging" { - if _, conditional := mapping["if"]; conditional { - t.Fatal("staging guard cannot be conditional") + run, _ := mapping["run"].(string) + for _, line := range strings.Split(run, "\n") { + if strings.TrimSpace(line) == "go test ./scripts/tests/arc-staging" { + if _, conditional := mapping["if"]; conditional { + t.Fatal("staging guard cannot be conditional") + } + return } - return } } t.Fatal("missing unconditional staging guard") } -func TestNoDeploymentAggregateActivatesARC(t *testing.T) { +func TestDeploymentAggregatesRestrictARCToProduction(t *testing.T) { const retainedControllerAggregate = "k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" const retainedControllerReference = "../../../../bases/infrastructure/controllers/actions-runner-controller/" controllerFound := false - credentialStageFound := false + poolFound := false err := filepath.WalkDir(filepath.Join(repoRoot, "k8s"), func(path string, entry os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr @@ -341,6 +345,13 @@ func TestNoDeploymentAggregateActivatesARC(t *testing.T) { if err := yaml.Unmarshal(data, &document); err != nil { return err } + if strings.HasSuffix(path, "k8s/providers/hetzner/infrastructure/kustomization.yaml") { + for _, reference := range document.Resources { + if reference == "arc-credential-transport/" { + t.Fatal("activation must not duplicate credential staging resources") + } + } + } for _, reference := range append(document.Resources, document.Components...) { if strings.Contains(reference, "actions-runner-controller") || strings.Contains(reference, "actions-runners") { relative, err := filepath.Rel(repoRoot, path) @@ -351,9 +362,9 @@ func TestNoDeploymentAggregateActivatesARC(t *testing.T) { controllerFound = true continue } - if relative == "k8s/providers/hetzner/infrastructure/arc-credential-transport/kustomization.yaml" && - reference == "../../../../bases/infrastructure/actions-runners/credentials/" && !credentialStageFound { - credentialStageFound = true + if relative == "k8s/providers/hetzner/infrastructure/kustomization.yaml" && + reference == "../../../bases/infrastructure/actions-runners/" && !poolFound { + poolFound = true continue } t.Errorf("%s activates ARC through %q", path, reference) @@ -365,10 +376,10 @@ func TestNoDeploymentAggregateActivatesARC(t *testing.T) { t.Fatal(err) } if !controllerFound { - t.Fatal("production must retain the scoped controller in its inventory") + t.Fatal("production must declare its bounded controller") } - if !credentialStageFound { - t.Fatal("production must stage only the dedicated credential authentication resources") + if !poolFound { + t.Fatal("production must declare its bounded pool") } } @@ -386,7 +397,10 @@ func TestRetainedAnalysisResourcesCannotActivateOrLosePruneProtection(t *testing equal(t, field(t, release, "spec", "values", "maxRunners"), 0) controller := readYAML(t, "k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml") equal(t, field(t, controller, "spec", "values", "flags", "watchSingleNamespace"), "arc-runners") - equal(t, field(t, controller, "metadata", "annotations", "platform.devantler.tech/arc-recovery"), "drain-only") + annotations := field(t, controller, "metadata", "annotations").(map[string]any) + if _, exists := annotations["platform.devantler.tech/arc-recovery"]; exists { + t.Fatal("reviewed activation must explicitly retire the drain-only marker") + } } func TestRecoveryMetadataProofSurroundsPublicationAndReconciliation(t *testing.T) { diff --git a/scripts/tests/arc-staging/transport_test.go b/scripts/tests/arc-staging/transport_test.go index 0e92ffde72..88ebe3fce1 100644 --- a/scripts/tests/arc-staging/transport_test.go +++ b/scripts/tests/arc-staging/transport_test.go @@ -146,15 +146,22 @@ func TestCredentialStoreRequiresDedicatedVerifiedTLS(t *testing.T) { equal(t, field(t, vault, "caProvider", "key"), "ca.crt") } -func TestCredentialAuthenticationStagingDoesNotReadAppKeys(t *testing.T) { - stage := readYAML(t, "k8s/providers/hetzner/infrastructure/arc-credential-transport/kustomization.yaml") - resources := field(t, stage, "resources").([]any) - if len(resources) != 1 { - t.Fatal("credential staging must contain only its authentication component") +func TestCredentialAuthenticationComponentDoesNotReadAppKeys(t *testing.T) { + pool := readYAML(t, "k8s/bases/infrastructure/actions-runners/kustomization.yaml") + count := 0 + for _, resource := range field(t, pool, "resources").([]any) { + if resource == "credentials/" { + count++ + } + if resource == "secret-store.yaml" || resource == "service-account.yaml" { + t.Fatal("pool duplicates its authentication component") + } + } + if count != 1 { + t.Fatal("pool must include its authentication component exactly once") } - equal(t, resources[0], "../../../../bases/infrastructure/actions-runners/credentials/") credentials := readYAML(t, "k8s/bases/infrastructure/actions-runners/credentials/kustomization.yaml") - resources = field(t, credentials, "resources").([]any) + resources := field(t, credentials, "resources").([]any) if len(resources) != 2 { t.Fatal("authentication staging must not include any key reader or runner") } diff --git a/scripts/tests/test-verify-ksail-arc-runtime.sh b/scripts/tests/test-verify-ksail-arc-runtime.sh new file mode 100755 index 0000000000..638633ffae --- /dev/null +++ b/scripts/tests/test-verify-ksail-arc-runtime.sh @@ -0,0 +1,698 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/bin" "$scratch/scripts" "$scratch/k8s/providers/hetzner/infrastructure/controllers" \ + "$scratch/k8s/bases/infrastructure/actions-runners" \ + "$scratch/k8s/bases/infrastructure/controllers/actions-runner-controller" +cp "$root/scripts/verify-ksail-arc-runtime.sh" "$scratch/scripts/" +cp "$root/scripts/ksail-arc-admission-fence.sh" "$scratch/scripts/" +if [[ -e "$root/scripts/wait-for-ksail-arc-registration.sh" ]]; then + cp "$root/scripts/wait-for-ksail-arc-registration.sh" "$scratch/scripts/" +fi +cp "$root/k8s/bases/infrastructure/actions-runners/helm-release.yaml" \ + "$scratch/k8s/bases/infrastructure/actions-runners/" +cp "$root/scripts/ksail-arc-job-metrics.sh" "$scratch/scripts/" +cp "$root/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml" \ + "$scratch/k8s/bases/infrastructure/controllers/actions-runner-controller/" +cp "$root/k8s/providers/hetzner/infrastructure/retained-ksail-analysis/helm-release.yaml" \ + "$scratch/retained-release.yaml" +yq -i '.spec.suspend=false' "$scratch/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml" +go build -o "$scratch/verifier" "$root/scripts/verify-ksail-arc-runtime" +export ARC_TEST_ROOT="$scratch" +printf '{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{},"layers":[]}' >"$scratch/runtime-image" +runtime_digest="sha256:$(shasum -a 256 "$scratch/runtime-image" | cut -d' ' -f1)" +export runtime_digest +jq -cn --arg child "$runtime_digest" '{schemaVersion:2,mediaType:"application/vnd.oci.image.index.v1+json", + manifests:[{digest:$child,platform:{os:"linux",architecture:"amd64"}}]}' >"$scratch/image" +digest="sha256:$(shasum -a 256 "$scratch/image" | cut -d' ' -f1)" +export digest +image="ghcr.io/devantler-tech/ksail-analysis-runner@$digest" +image="$image" yq -i '.spec.values.template.spec.containers[0].image=strenv(image) | + .spec.values.template.spec.initContainers[0].image=strenv(image) | .spec.suspend=false' \ + "$scratch/k8s/bases/infrastructure/actions-runners/helm-release.yaml" +yq -o=json '.spec.values' "$scratch/k8s/bases/infrastructure/actions-runners/helm-release.yaml" \ + | jq '{metadata:{uid:"ars-uid",generation:1,annotations:{"runner-scale-set-id":"1", + "actions.github.com/runner-group-name":"platform", + "actions.github.com/runner-scale-set-name":"platform-linux"}}, + status:{phase:"Running",observedGeneration:1},spec:.} | + .spec.template.spec.serviceAccountName="platform-linux-gha-rs-no-permission" | + .spec.template.spec.volumes[2].configMap.name="ksail-arc-job-metrics-abc123xyz4"' >"$scratch/ars" +cat >"$scratch/scripts/wait-for-platform-flux-revision.sh" <<'SH' +#!/usr/bin/env bash +[[ "$1" == "$digest" ]] +SH +cat >"$scratch/bin/timeout" <<'SH' +#!/usr/bin/env bash +export ARC_TEST_TIMEOUT=$1 +if [[ "$1" == 660s && "$ARC_TEST_CASE" == registration-diagnostic-redaction ]]; then + printf 'ARC registration pending: runner-scale-set\n' >&2 + printf 'ARC registration pending: private-registration-error-canary\n' >&2 + printf 'private-registration-error-canary\n' >&2 + exit 124 +fi +if [[ "$1" == 660s ]]; then touch "$ARC_TEST_ROOT/registration-budget"; fi +shift +exec "$@" +SH +cat >"$scratch/bin/sleep" <<'SH' +#!/usr/bin/env bash +[[ "$1" == 10 ]] || exit 98 +case "$ARC_TEST_CASE" in no-registration|registration-timeout|stale-flux) exit 124 ;; esac +SH +cat >"$scratch/bin/date" <<'SH' +#!/usr/bin/env bash +printf '2026-10-05T22:00:00.000000000Z\n' +SH +cat >"$scratch/bin/go" <<'SH' +#!/usr/bin/env bash +[[ "$1" == run && "$2" == ./scripts/verify-ksail-arc-runtime ]] || exit 91 +shift 2 +exec "$ARC_TEST_ROOT/verifier" "$@" +SH +cat >"$scratch/bin/git" <<'SH' +#!/usr/bin/env bash +[[ "$*" == 'rev-parse HEAD' ]] || exit 97 +printf '%040d\n' 1 +SH +cat >"$scratch/bin/gh" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +[[ "$1" == api && "$*" == *'--hostname github.com --method GET'* ]] || exit 97 +case "$2" in + 'repos/devantler-tech/platform/actions/runs/123/attempts/1') + [[ "$ARC_TEST_CASE" != orphan-current-empty-api ]] || exit 0 + printf '{"id":123,"run_attempt":1,"head_sha":"%040d","path":".github/workflows/ci.yaml","event":"merge_group","status":"in_progress","repository":{"full_name":"devantler-tech/platform"},"head_repository":{"full_name":"devantler-tech/platform"}}' 1 ;; + 'repos/devantler-tech/platform/actions/runs/122/attempts/1') + [[ "$ARC_TEST_CASE" != orphan-api-unknown ]] || exit 42 + if [[ "$ARC_TEST_CASE" == orphan-final-api-unknown && -e "$ARC_TEST_ROOT/prior-probe-deleted" ]]; then exit 42; fi + status=completed + [[ "$ARC_TEST_CASE" != orphan-active ]] || status=in_progress + jq -cn --arg status "$status" --arg scenario "$ARC_TEST_CASE" '{id:122,run_attempt:1,head_sha:"0000000000000000000000000000000000000001", + path:".github/workflows/ci.yaml",event:"merge_group",status:$status,conclusion:"cancelled", + repository:{full_name:"devantler-tech/platform"},head_repository:{full_name:"devantler-tech/platform"}} | + if $scenario=="orphan-run-mismatch" then .run_attempt=2 + elif $scenario=="orphan-wrong-api-head" then .head_sha="0000000000000000000000000000000000000002" + elif $scenario=="orphan-wrong-api-workflow" then .path=".github/workflows/unrelated.yaml" + elif $scenario=="orphan-wrong-api-repository" then .repository.full_name="devantler-tech/ksail" + else . end' ;; + 'repos/devantler-tech/platform/actions/runs/123/attempts/1/jobs?per_page=100') + [[ "$ARC_TEST_CASE" != orphan-current-job-list-empty ]] || exit 0 + printf '[{"total_count":1,"jobs":[{"id":456,"run_id":123,"head_sha":"%040d","name":"๐Ÿš€ Deploy to Prod","status":"in_progress"}]}]' 1 ;; + 'repos/devantler-tech/platform/actions/runs/122/attempts/1/jobs?per_page=100') + count=1 + [[ "$ARC_TEST_CASE" != orphan-incomplete-jobs ]] || count=2 + jq -cn --argjson count "$count" --arg scenario "$ARC_TEST_CASE" '{total_count:$count,jobs:[{id:455,run_id:122, + head_sha:"0000000000000000000000000000000000000001",name:"๐Ÿš€ Deploy to Prod",status:"completed", + conclusion:"cancelled",completed_at:"2026-10-05T22:00:00Z"}]} | + if $scenario=="orphan-duplicate-job" then .total_count=2 | .jobs += .jobs + elif $scenario=="orphan-job-active" then .jobs[0].status="in_progress" | .jobs[0].completed_at=null + elif $scenario=="orphan-wrong-job" then .jobs[0].id=999 + else . end | if $scenario=="orphan-paginated" then + [.total_count=2, {total_count:2,jobs:[{id:999,run_id:122,name:"Unrelated job",status:"completed"}]}] + else [.] end' ;; + *) exit 97 ;; +esac +SH +cat >"$scratch/bin/cosign" <<'SH' +#!/usr/bin/env bash +[[ "$*" == *"--certificate-identity https://github.com/devantler-tech/ksail/.github/workflows/publish-ksail-analysis-runner.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com ghcr.io/devantler-tech/ksail-analysis-runner@$digest" ]] || exit 92 +[[ "$ARC_TEST_CASE" != bad-signature ]] || exit 1 +SH +cat >"$scratch/bin/docker" <<'SH' +#!/usr/bin/env bash +if [[ "$*" == "buildx imagetools inspect ghcr.io/devantler-tech/ksail-analysis-runner@$digest --raw" ]]; then + cat "$ARC_TEST_ROOT/image" +elif [[ "$*" == "buildx imagetools inspect ghcr.io/devantler-tech/ksail-analysis-runner@$runtime_digest --raw" ]]; then + cat "$ARC_TEST_ROOT/runtime-image" +else exit 93; fi +[[ "$ARC_TEST_CASE" != tampered-image ]] || printf ' ' +SH +cat >"$scratch/bin/kubectl" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +if [[ "${1:-}" == kustomize ]]; then + [[ "${ARC_TEST_CASE:-}" != source-render-failure ]] || exit 1 + case "$2" in + k8s/providers/hetzner/infrastructure/controllers) + aggregate="$ARC_TEST_ROOT/$2/kustomization.yaml" + release="$ARC_TEST_ROOT/k8s/bases/infrastructure/controllers/actions-runner-controller/helm-release.yaml" + reference='actions-runner-controller' ;; + k8s/providers/hetzner/infrastructure) + aggregate="$ARC_TEST_ROOT/$2/kustomization.yaml" + release="$ARC_TEST_ROOT/k8s/bases/infrastructure/actions-runners/helm-release.yaml" + reference='actions-runners' ;; + *) exit 0 ;; + esac + references=$(yq -o=json '(.resources // []) + (.bases // [])' "$aggregate") + if [[ "$2" == k8s/providers/hetzner/infrastructure ]]; then + case "${ARC_TEST_CASE:-}" in + rendered-retained-drain|rendered-drained-controller|rendered-drained-controller-wrong-scope|retained-active-min|retained-active-max|retained-suspended|retained-missing-bound|retained-wrong-scale-set) + printf '\n---\n' + case "$ARC_TEST_CASE" in + retained-active-min) yq '.spec.values.minRunners=1' "$ARC_TEST_ROOT/retained-release.yaml" ;; + retained-active-max) yq '.spec.values.maxRunners=1' "$ARC_TEST_ROOT/retained-release.yaml" ;; + retained-suspended) yq '.spec.suspend=true' "$ARC_TEST_ROOT/retained-release.yaml" ;; + retained-missing-bound) yq 'del(.spec.values.maxRunners)' "$ARC_TEST_ROOT/retained-release.yaml" ;; + retained-wrong-scale-set) yq '.spec.values.runnerScaleSetName="unapproved"' "$ARC_TEST_ROOT/retained-release.yaml" ;; + *) cat "$ARC_TEST_ROOT/retained-release.yaml" ;; + esac + printf '\n---\n' + ;; + esac + fi + if jq -e --arg component "$reference" 'any(.[]; contains($component))' <<<"$references" >/dev/null; then + case "${ARC_TEST_CASE:-}" in + rendered-suspended) yq '.spec.suspend=true' "$release" ;; + rendered-drained-controller) yq '.metadata.annotations."platform.devantler.tech/arc-recovery"="drain-only"' "$release" ;; + rendered-drained-controller-wrong-scope) yq '.metadata.annotations."platform.devantler.tech/arc-recovery"="drain-only" | .spec.values.flags.watchSingleNamespace="unapproved"' "$release" ;; + rendered-wrong-image) yq '.spec.values.template.spec.containers[0].image="unverified:latest"' "$release" ;; + *) cat "$release" ;; + esac + fi + exit 0 +fi +[[ "$1" == --context && "$2" == admin@prod ]] || exit 94 +touch "$ARC_TEST_ROOT/runtime-access" +shift 3 # context pair and request-timeout +args="$*" +case "$args" in + *'get providerconfigs.github.m.upbound.io --all-namespaces '*) + jq -cn '{items:[{apiVersion:"github.m.upbound.io/v1beta1",kind:"ProviderConfig", + metadata:{name:"arc-runtime-platform-app",namespace:"arc-runners"}, + spec:{credentials:{source:"Secret",secretRef:{namespace:"arc-runners",name:"arc-github-app",key:"provider-credentials"}}}}]}' ;; + *'get clusterproviderconfigs.github.m.upbound.io '*|*'get providerconfigs.github.upbound.io '*) + printf '{"items":[]}' ;; + *'get deployment cluster-autoscaler-hetzner-cluster-autoscaler '*) + printf '%s\n' --cloud-provider=hetzner --max-nodes-total=9 --nodes=0:1:cx53:fsn1:autoscale-arc-runners + [[ "$ARC_TEST_CASE" != wider-ceiling ]] || printf '%s\n' --max-nodes-total=10 ;; + *'get kustomization '*) + count=0 + [[ ! -e "$ARC_TEST_ROOT/flux-reads" ]] || count=$(cat "$ARC_TEST_ROOT/flux-reads") + count=$((count + 1)) + printf '%s\n' "$count" >"$ARC_TEST_ROOT/flux-reads" + if [[ "$ARC_TEST_CASE" == stale-flux || ( "$ARC_TEST_CASE" == delayed-flux && "$count" -le 2 ) ]]; then + printf '{"metadata":{"generation":2},"status":{"observedGeneration":1,"lastAppliedRevision":"latest@old","conditions":[{"type":"Ready","status":"True"}]}}' + exit 0 + fi + jq -n --arg revision "latest@$digest" '{metadata:{generation:1},status:{observedGeneration:1, + lastAppliedRevision:$revision,conditions:[{type:"Ready",status:"True"}]}}' ;; + *'get runnergroups.actions.github.m.upbound.io platform '*) + jq -cn '{apiVersion:"actions.github.m.upbound.io/v1alpha1",kind:"RunnerGroup", + metadata:{name:"platform",namespace:"arc-runners",uid:"group-uid",generation:2,annotations:{"crossplane.io/external-name":"27"}}, + spec:{providerConfigRef:{name:"arc-runtime-platform-app",kind:"ProviderConfig"},forProvider:{name:"platform",visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:true,selectedRepositoryIds:[737584922],selectedWorkflows:["devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main"]}}, + status:{atProvider:{id:"27",name:"platform",default:false,inherited:false,visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:true,selectedRepositoryIds:[737584922],selectedWorkflows:["devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main"], + runnersUrl:"https://api.github.com/orgs/devantler-tech/actions/runner-groups/27/runners", + selectedRepositoriesUrl:"https://api.github.com/orgs/devantler-tech/actions/runner-groups/27/repositories"}, + conditions:[{type:"Ready",status:"True",observedGeneration:2},{type:"Synced",status:"True",observedGeneration:2}]}}' ;; + *'get autoscalingrunnerset.actions.github.com '*) + count=0 + [[ ! -e "$ARC_TEST_ROOT/registration-reads" ]] || count=$(cat "$ARC_TEST_ROOT/registration-reads") + count=$((count + 1)) + printf '%s\n' "$count" >"$ARC_TEST_ROOT/registration-reads" + if [[ "$ARC_TEST_CASE" == registration-timeout || ( "$ARC_TEST_CASE" == delayed-registration && "$count" -eq 1 ) ]]; then + jq '.status.phase="Pending" | .status.observedGeneration=0 | .metadata.annotations["runner-scale-set-id"]="0"' "$ARC_TEST_ROOT/ars" + exit 0 + fi + case "$ARC_TEST_CASE" in + no-registration) jq '.metadata.annotations["runner-scale-set-id"]="0"' "$ARC_TEST_ROOT/ars" ;; + unexpected-hook) jq '.spec.template.spec.containers[0].env[0].value="/unverified"' "$ARC_TEST_ROOT/ars" ;; + writable-metrics) jq '.spec.template.spec.containers[0].volumeMounts[2].readOnly=false' "$ARC_TEST_ROOT/ars" ;; + wrong-metrics-configmap) jq '.spec.template.spec.volumes[2].configMap.name="unverified"' "$ARC_TEST_ROOT/ars" ;; + runner-deadline) jq '.spec.template.spec.activeDeadlineSeconds=60' "$ARC_TEST_ROOT/ars" ;; + template-churn) + if [[ -e "$ARC_TEST_ROOT/admission-fence" ]]; then jq '.spec.template.spec.activeDeadlineSeconds=60' "$ARC_TEST_ROOT/ars" + else cat "$ARC_TEST_ROOT/ars"; fi ;; + invalid-runner-identity) jq 'del(.metadata.uid)' "$ARC_TEST_ROOT/ars" ;; + drift-during-probe) + if [[ -e "$ARC_TEST_ROOT/metrics-executed" ]]; then jq '.metadata.generation=2 | .spec.template.spec.activeDeadlineSeconds=60' "$ARC_TEST_ROOT/ars" + else cat "$ARC_TEST_ROOT/ars"; fi ;; + *) cat "$ARC_TEST_ROOT/ars" ;; + esac ;; + *'get ephemeralrunnersets.actions.github.com '*|*'get ephemeralrunners.actions.github.com '*) + printf 'continue=\n' + if [[ "$ARC_TEST_CASE" == retained-runner-deadline ]]; then printf 'uid=retained-uid;deadline=60\n' + else printf 'uid=retained-uid;deadline=\n'; fi ;; + *'get configmap ksail-arc-job-metrics-'*) + if [[ "$ARC_TEST_CASE" == live-api-error ]]; then + printf 'private-api-error-canary\n' >&2 + exit 42 + fi + jq -cn --rawfile script "$ARC_TEST_ROOT/scripts/ksail-arc-job-metrics.sh" \ + --arg scenario "$ARC_TEST_CASE" '{metadata:{uid:"metrics-uid",annotations:{"kustomize.toolkit.fluxcd.io/substitute":"disabled"}},immutable:($scenario!="mutable-metrics"), + data:{"job-metrics.sh":(if $scenario=="tampered-metrics" then "unverified" else $script end)}}' ;; + *'get resourcequotas -l '*) + printf '{"items":[]}' ;; + *'get resourcequotas '*) + { + if [[ "$ARC_TEST_CASE" == full-quota ]]; then + printf '{"items":[{"spec":{"hard":{"pods":"1"}},"status":{"hard":{"pods":"1"},"used":{"pods":"1"}}}]}' + elif [[ "$ARC_TEST_CASE" == stale-quota ]]; then + printf '{"items":[{"spec":{"hard":{"pods":"1"}},"status":{"hard":{"pods":"3"},"used":{"pods":"0"}}}]}' + else printf '{"items":[]}'; fi + } | jq --slurpfile fence "$ARC_TEST_ROOT/admission-fence" '.items += $fence' ;; + *'get resourcequota arc-runtime-admission '*) + [[ -e "$ARC_TEST_ROOT/admission-fence" ]] || exit 0 + if [[ "$ARC_TEST_CASE" == own-fence-empty && -e "$ARC_TEST_ROOT/fence-mutated" ]]; then exit 0; fi + if [[ "$ARC_TEST_CASE" == orphan-quota-churn ]]; then + if [[ -e "$ARC_TEST_ROOT/prior-fence-read" ]]; then + jq '.metadata.uid="replacement-fence-uid"' "$ARC_TEST_ROOT/admission-fence" >"$ARC_TEST_ROOT/replaced-fence" + mv "$ARC_TEST_ROOT/replaced-fence" "$ARC_TEST_ROOT/admission-fence" + fi + touch "$ARC_TEST_ROOT/prior-fence-read" + fi + cat "$ARC_TEST_ROOT/admission-fence" ;; + *'patch resourcequota arc-runtime-admission --type=json --patch-file '*) + patch=${!#} + jq -e '.[0] == {op:"test",path:"/metadata/uid",value:"fence-uid"} and + .[1] == {op:"test",path:"/metadata/resourceVersion",value:"1"} and + .[2] == {op:"add",path:"/metadata/annotations/platform.devantler.tech~1arc-runtime-probe-uid",value:"owned-uid"}' \ + "$patch" >/dev/null || exit 96 + jq '.metadata.annotations["platform.devantler.tech/arc-runtime-probe-uid"]="owned-uid" | + .metadata.resourceVersion="2"' "$ARC_TEST_ROOT/admission-fence" >"$ARC_TEST_ROOT/patched-fence" + mv "$ARC_TEST_ROOT/patched-fence" "$ARC_TEST_ROOT/admission-fence" ;; + *'get pods -l platform.devantler.tech/arc-role=runner '*|*'get pods -l platform.devantler.tech/arc-runtime-probe'*) + if [[ "$ARC_TEST_CASE" == orphan-incomplete-probes && "$args" == *'arc-runtime-probe='* ]]; then + printf '{"metadata":{"continue":"next"},"items":[]}' + exit 0 + fi + if [[ "$args" == *'arc-role=runner'* && -e "$ARC_TEST_ROOT/live-pod" ]]; then + jq '{items:[.]}' "$ARC_TEST_ROOT/live-pod" + elif [[ "$args" == *'arc-role=runner'* && "$ARC_TEST_CASE" == busy-runner && ! -e "$ARC_TEST_ROOT/job-completed" ]]; then + touch "$ARC_TEST_ROOT/job-completed" + printf '{"items":[{"metadata":{"name":"legitimate-runner","uid":"job-uid"},"spec":{},"status":{"phase":"Running"}}]}' + else printf '{"items":[]}'; fi ;; + 'get nodes -o json') + if [[ "$ARC_TEST_CASE" == orphan-node-api && -e "$ARC_TEST_ROOT/prior-probe-deleted" ]]; then exit 42; fi + if [[ -e "$ARC_TEST_ROOT/deleted" && ! -e "$ARC_TEST_ROOT/admission-fence" ]]; then exit 97; fi + if [[ "$ARC_TEST_CASE" == failed-node-cleanup && -e "$ARC_TEST_ROOT/deleted" ]]; then exit 1; fi + if [[ "$ARC_TEST_CASE" == unknown-create && -e "$ARC_TEST_ROOT/live-pod" ]]; then exit 1; fi + if [[ "$ARC_TEST_CASE" == replacement-race && -e "$ARC_TEST_ROOT/replacement-preserved" ]]; then exit 1; fi + if [[ -e "$ARC_TEST_ROOT/live-pod" ]]; then + printf '{"items":[{"metadata":{"name":"autoscale-arc-runners-0123456789abcdef","uid":"autoscale-arc-runners-0123456789abcdef-uid","labels":{"platform.devantler.tech/ci-runner":"enabled"}}}]}' + else + printf '{"items":[{"metadata":{"name":"baseline-node","uid":"baseline-uid","labels":{}}}]}' + fi ;; + 'create --dry-run=server -f '*) + file=$4 + if [[ -e "$ARC_TEST_ROOT/admission-fence" ]] && jq -e '.spec.activeDeadlineSeconds == null' "$file" >/dev/null; then + [[ "$ARC_TEST_CASE" != fence-transport ]] || { printf 'connection refused\n' >&2; exit 1; } + printf 'pods forbidden: exceeded quota: arc-runtime-admission, requested: pods=1, limited: pods=0\n' >&2 + exit 1 + fi + if jq -e '.spec.containers[0].securityContext.privileged == true or any(.spec.volumes[]; .hostPath != null)' "$file" >/dev/null; then + if [[ "$ARC_TEST_CASE" == admission-transport ]]; then printf 'connection refused\n' >&2 + else printf 'violates PodSecurity restricted: privileged hostPath\n' >&2; fi + exit 1 + fi + case "$ARC_TEST_CASE" in + own-fence-origin-churn|own-fence-controller-churn|own-fence-deleting|own-fence-empty) + jq --arg scenario "$ARC_TEST_CASE" ' + if $scenario=="own-fence-origin-churn" then .metadata.annotations["platform.devantler.tech/arc-runtime-origin"]="changed" + elif $scenario=="own-fence-controller-churn" then .metadata.ownerReferences=[{uid:"controller-uid"}] + elif $scenario=="own-fence-deleting" then .metadata.deletionTimestamp="2026-10-05T22:00:00Z" + else . end' "$ARC_TEST_ROOT/admission-fence" >"$ARC_TEST_ROOT/mutated-fence" + mv "$ARC_TEST_ROOT/mutated-fence" "$ARC_TEST_ROOT/admission-fence" + touch "$ARC_TEST_ROOT/fence-mutated" ;; + esac ;; + 'create -f '*) + if jq -e '.kind == "ResourceQuota"' "$3" >/dev/null; then + [[ ! -e "$ARC_TEST_ROOT/admission-fence" ]] || exit 1 + jq -e '.metadata.name == "arc-runtime-admission" and .metadata.namespace == "arc-runners" and + .metadata.labels["platform.devantler.tech/arc-runtime-probe"] == "arc-proof-123-1" and + .spec == {hard:{pods:"0"},scopes:["NotTerminating"]}' "$3" >/dev/null || exit 96 + [[ "$ARC_TEST_CASE" != foreign-fence ]] || exit 1 + jq '.metadata.uid="fence-uid" | .metadata.resourceVersion="1" | + .status={hard:{pods:"0"},used:{pods:"0"}}' "$3" >"$ARC_TEST_ROOT/admission-fence" + cat "$ARC_TEST_ROOT/admission-fence" + exit 0 + fi + if [[ "$ARC_TEST_CASE" == late-runner ]]; then + [[ -e "$ARC_TEST_ROOT/admission-fence" ]] || { touch "$ARC_TEST_ROOT/late-job-admitted"; exit 1; } + touch "$ARC_TEST_ROOT/late-job-blocked" + fi + cp "$3" "$ARC_TEST_ROOT/desired-pod" + jq --arg image "ghcr.io/devantler-tech/ksail-analysis-runner@$digest" ' + .metadata.uid="owned-uid" | .spec.nodeName="autoscale-arc-runners-0123456789abcdef" | + .status={podIP:"192.0.2.10", + containerStatuses:[{name:"runner",ready:true,restartCount:0,containerID:"runtime-id",imageID:$image}], + initContainerStatuses:[{name:"init-runner-home",restartCount:0,imageID:$image,state:{terminated:{exitCode:0}}}]}' \ + "$3" >"$ARC_TEST_ROOT/live-pod" + if [[ "$ARC_TEST_CASE" == unknown-create ]]; then exit 1; fi + cat "$ARC_TEST_ROOT/live-pod" ;; + *'wait pod/'*'--for=condition=Ready --timeout=600s') + [[ "$ARC_TEST_TIMEOUT" == 630s ]] || exit 95 ;; + *'wait pod/'*'--for=delete --timeout=25s') ;; + *'get pod arc-proof-'*) + [[ -e "$ARC_TEST_ROOT/live-pod" ]] || exit 0 + if [[ "$ARC_TEST_CASE" == mutated-init ]]; then + jq '.spec.initContainers[0].securityContext.privileged=true' "$ARC_TEST_ROOT/live-pod" + else cat "$ARC_TEST_ROOT/live-pod"; fi ;; + 'delete --raw='*) + if [[ "$args" == *'/resourcequotas/'* ]]; then + if jq -e '.preconditions.uid == "prior-fence-uid"' "$4" >/dev/null; then + [[ "$ARC_TEST_CASE" != orphan-replacement ]] || { touch "$ARC_TEST_ROOT/prior-replacement-preserved"; exit 1; } + jq -e '.preconditions == {uid:"prior-fence-uid",resourceVersion:"1"}' "$4" >/dev/null || exit 96 + [[ ! -e "$ARC_TEST_ROOT/live-pod" ]] || exit 96 + rm "$ARC_TEST_ROOT/admission-fence" + touch "$ARC_TEST_ROOT/prior-fence-deleted" + exit 0 + fi + [[ "$ARC_TEST_CASE" != replacement-fence ]] || { touch "$ARC_TEST_ROOT/fence-replacement-preserved"; exit 1; } + jq -e '.kind == "DeleteOptions" and .preconditions.uid == "fence-uid" and + (.preconditions.resourceVersion == "1" or .preconditions.resourceVersion == "2")' "$4" >/dev/null || exit 96 + [[ ! -e "$ARC_TEST_ROOT/live-pod" ]] || exit 96 + rm "$ARC_TEST_ROOT/admission-fence" + touch "$ARC_TEST_ROOT/fence-deleted" + exit 0 + fi + if [[ "$args" == *'/pods/arc-proof-122-1'* ]]; then + jq -e '.preconditions.uid == "prior-probe-uid" and .gracePeriodSeconds == 5' "$4" >/dev/null || exit 96 + rm "$ARC_TEST_ROOT/live-pod" + touch "$ARC_TEST_ROOT/prior-probe-deleted" + exit 0 + fi + [[ "$ARC_TEST_CASE" != replacement-race ]] || { touch "$ARC_TEST_ROOT/replacement-preserved"; exit 1; } + jq -e '.kind=="DeleteOptions" and .preconditions.uid=="owned-uid" and .gracePeriodSeconds==5' "$4" >/dev/null || exit 96 + rm "$ARC_TEST_ROOT/live-pod" + touch "$ARC_TEST_ROOT/deleted" ;; + 'get node autoscale-arc-runners-0123456789abcdef '*) + jq -n '{metadata:{name:"autoscale-arc-runners-0123456789abcdef",uid:"autoscale-arc-runners-0123456789abcdef-uid",labels:{"platform.devantler.tech/ci-runner":"enabled","node.kubernetes.io/instance-type":"cx53"}}, + spec:{taints:[{key:"platform.devantler.tech/ci-runner",value:"enabled",effect:"NoSchedule"}]}, + status:{nodeInfo:{operatingSystem:"linux",architecture:"amd64"},allocatable:{cpu:"15",memory:"30Gi","ephemeral-storage":"100Gi"}, + conditions:[{type:"Ready",status:"True"}]}}' ;; + 'get pods -A --field-selector spec.nodeName=autoscale-arc-runners-0123456789abcdef '*) + printf 'P\towned-uid\tRunning\nR\t3\t12Gi\t32Gi\nP\tsystem-uid\tRunning\nR\t100m\t1Gi\t1Gi\n' + if [[ "$ARC_TEST_CASE" == insufficient-memory ]]; then printf 'R\t100m\t25Gi\t1Gi\n'; fi ;; + *'get pods -l k8s-app=cilium '*) + printf '{"items":[{"metadata":{"name":"observer","uid":"observer-uid"},"spec":{"nodeName":"autoscale-arc-runners-0123456789abcdef"}, + "status":{"podIP":"192.0.2.11","containerStatuses":[{"name":"cilium-agent","ready":true,"restartCount":0,"containerID":"observer-id"}],"conditions":[{"type":"Ready","status":"True"}]}}]}' ;; + *'get configmap cilium-config '*) + printf '{"data":{"cluster-name":"fixture"}}' ;; + *'exec observer -c cilium-agent -- hubble status '*) ;; + *'get pods -l k8s-app=kube-dns '*) + printf '{"items":[{"metadata":{"name":"dns","uid":"dns-uid"},"spec":{"nodeName":"baseline-node"}, + "status":{"podIP":"192.0.2.12","containerStatuses":[{"name":"dns","ready":true,"restartCount":0,"containerID":"dns-id"}],"conditions":[{"type":"Ready","status":"True"}]}}]}' ;; + *'get endpoints kubernetes '*) + printf '{"subsets":[{"addresses":[{"ip":"192.0.2.13"}],"ports":[{"name":"https","port":6443}]}]}' ;; + 'get --raw /api/v1/namespaces/kube-system/pods/dns:8181/proxy/ready') + [[ "$ARC_TEST_CASE" != unhealthy-target ]] || { printf 'not ready'; exit 0; } + printf OK ;; + '--server=https://192.0.2.13:6443 get --raw /readyz') printf ok ;; + *'exec arc-proof-'*'-- curl '*) + if [[ "$ARC_TEST_CASE" == curl-transport ]]; then exit 1; fi + if [[ "$ARC_TEST_CASE" == allowed-egress ]]; then exit 0; fi + exit 28 ;; + *'exec arc-proof-'*'-- test -f /tmp/arc-proof-ready') ;; + *'exec arc-proof-'*'-- /etc/ksail-arc-metrics/job-metrics.sh') + mkdir -p "$ARC_TEST_ROOT/cgroup" + printf '15032385536\n' >"$ARC_TEST_ROOT/cgroup/memory.max" + if [[ "$ARC_TEST_CASE" == failed-metrics-hook ]]; then + printf '0\n' >"$ARC_TEST_ROOT/cgroup/memory.peak" + else printf '9876543210\n' >"$ARC_TEST_ROOT/cgroup/memory.peak"; fi + printf 'low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0\n' >"$ARC_TEST_ROOT/cgroup/memory.events" + source "$ARC_TEST_ROOT/scripts/ksail-arc-job-metrics.sh" + arc_job_metrics "$ARC_TEST_ROOT/cgroup" + touch "$ARC_TEST_ROOT/metrics-executed" ;; + *'exec observer -c cilium-agent -- hubble observe '*) + destination=192.0.2.12 port=8181 target_ns=kube-system target_pod=dns + [[ "$args" != *'--to-ip 192.0.2.13'* ]] || { destination=192.0.2.13; port=6443; target_ns=''; target_pod=''; } + verdict=DROPPED source=192.0.2.10 + [[ "$ARC_TEST_CASE" != forwarded-flow ]] || verdict=FORWARDED + [[ "$ARC_TEST_CASE" != wrong-source ]] || source=192.0.2.99 + jq -n --arg verdict "$verdict" --arg source "$source" --arg destination "$destination" --argjson port "$port" \ + --arg target_ns "$target_ns" --arg target_pod "$target_pod" '{flow:{verdict:$verdict,drop_reason_desc:"POLICY_DENIED", + traffic_direction:"EGRESS",source:{namespace:"arc-runners",pod_name:"arc-proof-123-1"}, + destination:{namespace:$target_ns,pod_name:$target_pod},IP:{source:$source,destination:$destination}, + l4:{TCP:{destination_port:$port,flags:{SYN:true}}},node_name:"fixture/autoscale-arc-runners-0123456789abcdef",time:"2026-10-05T22:00:00Z"}}' + [[ "$ARC_TEST_CASE" != observer-loss ]] || printf '{"lost_events":{"num_events_lost":"1"}}' + [[ "$ARC_TEST_CASE" != observer-diagnostics ]] || printf 'observer warning\n' >&2 ;; + *'get pod observer '*) + printf '{"metadata":{"name":"observer","uid":"observer-uid"},"spec":{"nodeName":"autoscale-arc-runners-0123456789abcdef"}, + "status":{"podIP":"192.0.2.11","containerStatuses":[{"name":"cilium-agent","ready":true,"restartCount":0,"containerID":"observer-id"}]}}' ;; + *'get pod dns '*) + printf '{"metadata":{"name":"dns","uid":"dns-uid"},"spec":{"nodeName":"baseline-node"}, + "status":{"podIP":"192.0.2.12","containerStatuses":[{"name":"dns","ready":true,"restartCount":0,"containerID":"dns-id"}]}}' ;; + *) printf 'unrecognized fixture command\n' >&2; exit 97 ;; +esac +SH +chmod +x "$scratch/bin/"* + +expected_refusal() { + case "$1" in + orphan-current-empty-api|orphan-current-job-list-empty|orphan-unauthorized-current) printf 'runner-admission-fence' ;; + orphan-*) printf 'runner-fence-recovery' ;; + own-fence-*) printf 'restricted-admission' ;; + retained-active-min|retained-active-max|retained-suspended|retained-missing-bound|retained-wrong-scale-set) + printf 'retained-source-state' ;; + foreign-fence|retained-runner-deadline) printf 'runner-admission-fence' ;; + replacement-fence|replacement-race|failed-node-cleanup) printf 'cleanup' ;; + fence-transport) printf 'unproven-runner-admission-fence' ;; + template-churn) printf 'runner-job-drain' ;; + invalid-runner-identity|unexpected-hook|writable-metrics|wrong-metrics-configmap|runner-deadline) printf 'registration-and-bounds' ;; + drift-during-probe|failed-metrics-hook) printf 'job-cgroup-measurement' ;; + live-api-error|mutable-metrics|tampered-metrics) printf 'immutable-job-metrics' ;; + registration-timeout|stale-flux|no-registration|registration-diagnostic-redaction) printf 'registration-convergence' ;; + rendered-hidden-activation) printf 'partial-activation' ;; + source-render-failure) printf 'source-render' ;; + rendered-suspended|rendered-wrong-image) printf 'rendered-source-state' ;; + invalid-node-suffix-*|wrong-node-pool|full-quota|stale-quota|mutated-init|unknown-create) printf 'restricted-admission' ;; + bad-signature|tampered-image) printf 'immutable-image' ;; + wider-ceiling) printf 'autoscaler-boundary' ;; + admission-transport) printf 'unproven-admission-denial' ;; + insufficient-memory) printf 'allocatable-headroom' ;; + unhealthy-target) printf 'unhealthy-internal-control' ;; + curl-transport) printf 'unproven-curl-timeout' ;; + allowed-egress) printf 'network-isolation' ;; + forwarded-flow|wrong-source|observer-loss) printf 'intercepted-egress-denials' ;; + observer-diagnostics) printf 'observer-diagnostics' ;; + *) printf 'FAIL: no expected refusal for %s\n' "$1" >&2; return 1 ;; + esac +} + +run_case() { + local name=$1 expected=$2 code=0 producer_job=deploy-prod + [[ "$name" != orphan-unauthorized-current ]] || producer_job=unrelated + rm -f "$scratch/live-pod" "$scratch/deleted" "$scratch/replacement-preserved" "$scratch/runtime-access" + rm -f "$scratch/flux-reads" "$scratch/registration-reads" "$scratch/registration-budget" + rm -f "$scratch/metrics-executed" + rm -f "$scratch/admission-fence" "$scratch/fence-deleted" "$scratch/job-completed" \ + "$scratch/late-job-admitted" "$scratch/late-job-blocked" "$scratch/fence-replacement-preserved" + rm -f "$scratch/prior-fence-deleted" "$scratch/prior-probe-deleted" "$scratch/prior-replacement-preserved" + rm -f "$scratch/prior-fence-read" "$scratch/fence-mutated" + if [[ "$name" == orphan-* ]]; then + jq -cn --arg manifest "$digest" --arg key platform.devantler.tech/arc-runtime-origin \ + --arg scenario "$name" '{metadata:{name:"arc-runtime-admission",namespace:"arc-runners",uid:"prior-fence-uid",resourceVersion:"1", + labels:{"platform.devantler.tech/arc-runtime-probe":"arc-proof-122-1"}, + annotations:{($key):({repository:"devantler-tech/platform",workflow:".github/workflows/ci.yaml",run:"122",attempt:"1", + head:"0000000000000000000000000000000000000001",source:"0000000000000000000000000000000000000001", + manifest:$manifest,job:"455",job_name:"๐Ÿš€ Deploy to Prod"}|tojson), + "platform.devantler.tech/arc-runtime-probe-uid":"prior-probe-uid"}}, + spec:{hard:{pods:"0"},scopes:["NotTerminating"]},status:{hard:{pods:"0"},used:{pods:"0"}}} | + if $scenario=="orphan-missing-provenance" then del(.metadata.annotations[$key]) + elif $scenario=="orphan-missing-probe-uid" then del(.metadata.annotations["platform.devantler.tech/arc-runtime-probe-uid"]) + elif $scenario=="orphan-current-attempt" then + .metadata.labels["platform.devantler.tech/arc-runtime-probe"]="arc-proof-123-1" | + .metadata.annotations[$key] |= (fromjson | .run="123" | .job="456" | tojson) + elif $scenario=="orphan-foreign-provenance" then + .metadata.annotations[$key] |= (fromjson | .repository="devantler-tech/ksail" | tojson) + elif $scenario=="orphan-wrong-quota" then .spec.scopes=["Terminating"] + else . end' >"$scratch/admission-fence" + if [[ "$name" != orphan-empty ]]; then + jq -cn --arg scenario "$name" '{metadata:{name:"arc-proof-122-1",namespace:"arc-runners", + uid:(if $scenario=="orphan-wrong-probe" then "replacement-probe-uid" else "prior-probe-uid" end), + labels:{"platform.devantler.tech/arc-runtime-probe":"arc-proof-122-1"}}, + spec:{activeDeadlineSeconds:1200,restartPolicy:"Never"}} | + if $scenario=="orphan-controlled-probe" then .metadata.ownerReferences=[{uid:"controller-uid"}] + elif $scenario=="orphan-wrong-deadline" then .spec.activeDeadlineSeconds=60 + else . end' >"$scratch/live-pod" + fi + fi + ( + cd "$scratch" + PATH="$scratch/bin:$PATH" GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform \ + GITHUB_RUN_ID=123 GITHUB_RUN_ATTEMPT=1 PLATFORM_MANIFEST_DIGEST="$digest" ARC_TEST_CASE="$name" \ + GITHUB_SHA=0000000000000000000000000000000000000001 GITHUB_JOB="$producer_job" \ + GITHUB_WORKFLOW_REF=devantler-tech/platform/.github/workflows/ci.yaml@refs/heads/main GH_TOKEN=fixture-only \ + bash scripts/verify-ksail-arc-runtime.sh --if-active + ) >"$scratch/stdout" 2>"$scratch/stderr" || code=$? + if [[ "$expected" == pass ]]; then + [[ "$code" == 0 && -e "$scratch/deleted" && -e "$scratch/metrics-executed" ]] || { printf 'FAIL: %s (exit %s)\n' "$name" "$code" >&2; cat "$scratch/stderr" >&2; exit 1; } + grep -q '^PASS: ARC registration' "$scratch/stdout" || exit 1 + else + [[ "$code" != 0 ]] || { printf 'FAIL: accepted %s\n' "$name" >&2; exit 1; } + [[ "$code" -lt 91 || "$code" -gt 98 ]] || { + printf 'FAIL: %s failed with an unexpected fixture/transport exit %s\n' "$name" "$code" >&2 + exit 1 + } + local refusal marker + refusal=$(expected_refusal "$name") || exit 1 + marker="ARC acceptance: FAIL at $refusal" + [[ "$refusal" != cleanup ]] || marker='ARC acceptance: FAIL cleanup' + grep -Fxq "$marker" "$scratch/stderr" || { + printf 'FAIL: %s refused for an unrelated reason (expected %s, exit %s)\n' "$name" "$refusal" "$code" >&2 + cat "$scratch/stderr" >&2 + exit 1 + } + case "$name" in + replacement-race) [[ -e "$scratch/replacement-preserved" && ! -e "$scratch/deleted" && "$code" == 4 ]] || exit 1 ;; + unknown-create) [[ -e "$scratch/live-pod" && ! -e "$scratch/deleted" && "$code" == 4 ]] || exit 1 ;; + failed-node-cleanup) [[ -e "$scratch/deleted" && "$code" == 4 ]] || exit 1 ;; + esac + fi + printf 'PASS: %s\n' "$name" +} + +printf 'resources: []\n' >"$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" +printf 'resources: []\n' >"$scratch/k8s/providers/hetzner/infrastructure/kustomization.yaml" +( + cd "$scratch" + PATH="$scratch/bin:$PATH" bash scripts/verify-ksail-arc-runtime.sh --if-active +) >"$scratch/inactive" +grep -Fq 'inactive source; no runtime access' "$scratch/inactive" +[[ ! -e "$scratch/runtime-access" ]] +printf 'resources: ["../../../../bases/infrastructure/controllers/actions-runner-controller/"]\n' \ + >"$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" +( + cd "$scratch" + PATH="$scratch/bin:$PATH" ARC_TEST_CASE=rendered-suspended bash scripts/verify-ksail-arc-runtime.sh --if-active +) >"$scratch/retained-controller" +grep -Fq 'inactive source; no runtime access' "$scratch/retained-controller" +[[ ! -e "$scratch/runtime-access" ]] +( + cd "$scratch" + PATH="$scratch/bin:$PATH" ARC_TEST_CASE=rendered-drained-controller bash scripts/verify-ksail-arc-runtime.sh --if-active +) >"$scratch/drained-controller" +grep -Fq 'inactive source; no runtime access' "$scratch/drained-controller" +[[ ! -e "$scratch/runtime-access" ]] +if (cd "$scratch"; PATH="$scratch/bin:$PATH" ARC_TEST_CASE=rendered-drained-controller-wrong-scope bash scripts/verify-ksail-arc-runtime.sh --if-active) \ + >"$scratch/wrong-scope-out" 2>"$scratch/wrong-scope-error"; then exit 1; fi +grep -Fq 'FAIL at partial-activation' "$scratch/wrong-scope-error" +[[ ! -e "$scratch/runtime-access" ]] +if (cd "$scratch"; PATH="$scratch/bin:$PATH" bash scripts/verify-ksail-arc-runtime.sh --if-active) \ + >"$scratch/active-controller-out" 2>"$scratch/active-controller-error"; then exit 1; fi +grep -Fq 'FAIL at partial-activation' "$scratch/active-controller-error" +[[ ! -e "$scratch/runtime-access" ]] +printf 'resources: ["../../../bases/infrastructure/actions-runners/"]\n' \ + >"$scratch/k8s/providers/hetzner/infrastructure/kustomization.yaml" +if (cd "$scratch"; PATH="$scratch/bin:$PATH" GITHUB_ACTIONS=false bash scripts/verify-ksail-arc-runtime.sh --if-active) \ + >"$scratch/unauthorized-out" 2>"$scratch/unauthorized-error"; then exit 1; fi +grep -Fq 'FAIL at deployment-identity' "$scratch/unauthorized-error" +run_case rendered-retained-drain pass +for name in retained-active-min retained-active-max retained-suspended retained-missing-bound retained-wrong-scale-set; do + run_case "$name" fail + grep -Fq 'FAIL at retained-source-state' "$scratch/stderr" + [[ ! -e "$scratch/runtime-access" && ! -e "$scratch/live-pod" ]] +done +run_case complete-proof pass +run_case orphan-completed pass +[[ -e "$scratch/prior-fence-deleted" && -e "$scratch/prior-probe-deleted" ]] +run_case orphan-empty pass +[[ -e "$scratch/prior-fence-deleted" && ! -e "$scratch/prior-probe-deleted" ]] +run_case orphan-paginated pass +[[ -e "$scratch/prior-fence-deleted" && -e "$scratch/prior-probe-deleted" ]] +for name in orphan-active orphan-api-unknown orphan-incomplete-jobs orphan-missing-provenance orphan-wrong-probe \ + orphan-replacement orphan-run-mismatch orphan-wrong-api-head orphan-wrong-api-workflow orphan-wrong-api-repository \ + orphan-duplicate-job orphan-job-active orphan-wrong-job orphan-quota-churn orphan-incomplete-probes orphan-node-api \ + orphan-missing-probe-uid orphan-current-attempt orphan-foreign-provenance orphan-wrong-quota orphan-controlled-probe orphan-wrong-deadline \ + orphan-current-empty-api orphan-current-job-list-empty orphan-unauthorized-current orphan-final-api-unknown; do + run_case "$name" fail + [[ ! -e "$scratch/prior-fence-deleted" ]] + case "$name" in + orphan-replacement|orphan-incomplete-probes|orphan-node-api|orphan-final-api-unknown) [[ -e "$scratch/prior-probe-deleted" ]] ;; + *) [[ ! -e "$scratch/prior-probe-deleted" ]] ;; + esac +done +for name in own-fence-origin-churn own-fence-controller-churn own-fence-deleting own-fence-empty; do + run_case "$name" fail + [[ ! -e "$scratch/live-pod" && ! -e "$scratch/fence-deleted" && -e "$scratch/admission-fence" ]] +done +run_case busy-runner pass +[[ -e "$scratch/job-completed" && -e "$scratch/fence-deleted" ]] +run_case late-runner pass +[[ -e "$scratch/late-job-blocked" && ! -e "$scratch/late-job-admitted" && -e "$scratch/fence-deleted" ]] +run_case foreign-fence fail +[[ ! -e "$scratch/live-pod" && ! -e "$scratch/fence-deleted" ]] +run_case replacement-fence fail +[[ -e "$scratch/fence-replacement-preserved" && ! -e "$scratch/fence-deleted" ]] +run_case fence-transport fail +[[ ! -e "$scratch/live-pod" && -e "$scratch/fence-deleted" ]] +for name in runner-deadline retained-runner-deadline; do + run_case "$name" fail + [[ ! -e "$scratch/live-pod" && ! -e "$scratch/admission-fence" ]] +done +run_case template-churn fail +[[ ! -e "$scratch/live-pod" && -e "$scratch/fence-deleted" ]] +run_case invalid-runner-identity fail +[[ ! -e "$scratch/admission-fence" ]] +run_case drift-during-probe fail +[[ -e "$scratch/deleted" && -e "$scratch/fence-deleted" ]] +run_case live-api-error fail +grep -Fq 'ARC acceptance: FAIL at immutable-job-metrics' "$scratch/stderr" || { + printf 'FAIL: live API error did not identify the failing acceptance stage\n' >&2 + exit 1 +} +grep -Fq 'ARC acceptance: cleanup verified' "$scratch/stdout" +if grep -Fq 'private-api-error-canary' "$scratch/stdout" "$scratch/stderr"; then + printf 'FAIL: private API diagnostics reached acceptance output\n' >&2 + exit 1 +fi +run_case delayed-flux pass +[[ $(cat "$scratch/flux-reads") -gt 2 && -e "$scratch/registration-budget" ]] +run_case delayed-registration pass +[[ $(cat "$scratch/registration-reads") -gt 2 && -e "$scratch/registration-budget" ]] +for name in registration-timeout stale-flux; do + run_case "$name" fail + [[ -e "$scratch/registration-budget" && ! -e "$scratch/live-pod" && ! -e "$scratch/deleted" ]] + if [[ "$name" == registration-timeout ]]; then label=runner-scale-set + else label=flux-infrastructure; fi + grep -Fxq "ARC registration pending: $label" "$scratch/stderr" || { + printf 'FAIL: safe registration diagnostic was suppressed\n' >&2; exit 1; + } +done +run_case registration-diagnostic-redaction fail +grep -Fxq 'ARC registration pending: runner-scale-set' "$scratch/stderr" || exit 1 +if grep -Fq 'private-registration-error-canary' "$scratch/stdout" "$scratch/stderr"; then + printf 'FAIL: raw or unrecognized registration diagnostic escaped\n' >&2; exit 1 +fi +cp "$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" "$scratch/controller-aggregate" +cp "$scratch/k8s/providers/hetzner/infrastructure/kustomization.yaml" "$scratch/runner-aggregate" +sed 's/^resources:/bases:/' "$scratch/controller-aggregate" >"$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" +sed 's/^resources:/bases:/' "$scratch/runner-aggregate" >"$scratch/k8s/providers/hetzner/infrastructure/kustomization.yaml" +run_case rendered-hidden-activation fail +[[ ! -e "$scratch/runtime-access" ]] +cp "$scratch/controller-aggregate" "$scratch/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" +cp "$scratch/runner-aggregate" "$scratch/k8s/providers/hetzner/infrastructure/kustomization.yaml" +for name in source-render-failure rendered-suspended rendered-wrong-image; do + run_case "$name" fail + [[ ! -e "$scratch/runtime-access" ]] +done +# Hetzner formats rand.Int63 with %x, so valid suffixes contain one to sixteen +# hex digits. Exercise the complete lifecycle with the shortest legal name. +cp "$scratch/bin/kubectl" "$scratch/kubectl-original" +for suffix in a 0123456789abcdef0 nothex; do + sed "s/autoscale-arc-runners-0123456789abcdef/autoscale-arc-runners-$suffix/g" \ + "$scratch/kubectl-original" >"$scratch/bin/kubectl" + if [[ "$suffix" == a ]]; then run_case short-node-suffix pass + else run_case "invalid-node-suffix-$suffix" fail; fi +done +sed 's/autoscale-arc-runners-0123456789abcdef/autoscale-other-pool-0123456789abcdef/g' \ + "$scratch/kubectl-original" >"$scratch/bin/kubectl" +run_case wrong-node-pool fail +cp "$scratch/kubectl-original" "$scratch/bin/kubectl" +for name in no-registration unexpected-hook writable-metrics wrong-metrics-configmap mutable-metrics tampered-metrics \ + failed-metrics-hook bad-signature tampered-image wider-ceiling full-quota stale-quota admission-transport mutated-init \ + unknown-create replacement-race insufficient-memory unhealthy-target curl-transport allowed-egress \ + forwarded-flow wrong-source observer-loss observer-diagnostics failed-node-cleanup; do + run_case "$name" fail +done +# Prove that an unrelated API failure cannot stand in for an egress refusal. +sed 's/== live-api-error/== allowed-egress/' "$scratch/kubectl-original" >"$scratch/bin/kubectl" +if (run_case allowed-egress fail) >"$scratch/wrong-cause-out" 2>"$scratch/wrong-cause-error"; then + printf 'FAIL: an unrelated API error satisfied the egress negative control\n' >&2 + exit 1 +fi +cp "$scratch/kubectl-original" "$scratch/bin/kubectl" +printf 'PASS: inactive and unauthorized sources never reach runtime access\n' diff --git a/scripts/tests/test-wait-for-ksail-arc-registration.sh b/scripts/tests/test-wait-for-ksail-arc-registration.sh new file mode 100755 index 0000000000..85133542a6 --- /dev/null +++ b/scripts/tests/test-wait-for-ksail-arc-registration.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/bin" +cat >"$scratch/bin/timeout" <<'SH' +#!/usr/bin/env bash +shift +exec "$@" +SH +cat >"$scratch/bin/sleep" <<'SH' +#!/usr/bin/env bash +exit 124 +SH +cat >"$scratch/bin/kubectl" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +[[ "$*" == *'--context admin@prod'* ]] || exit 92 +case "$*" in + *'get providerconfigs.github.m.upbound.io --all-namespaces '*) + [[ "$GROUP_CASE" != unreadable-provider ]] || { printf 'private-registration-error-canary\n' >&2; exit 1; } + jq -cn --arg scenario "$GROUP_CASE" '{items:[{ + apiVersion:"github.m.upbound.io/v1beta1",kind:"ProviderConfig", + metadata:{name:"arc-runtime-platform-app",namespace:"arc-runners"}, + spec:{credentials:{source:"Secret",secretRef:{namespace:"arc-runners",name:"arc-github-app",key:"provider-credentials"}}} + }]} | + if $scenario=="duplicate-provider" then .items+=[.items[0]|.metadata.namespace="other"] + elif $scenario=="wrong-provider-secret" then .items[0].spec.credentials.secretRef.name="management-app" + else . end' ;; + *'get clusterproviderconfigs.github.m.upbound.io '*) + jq -cn --arg scenario "$GROUP_CASE" '{items:(if $scenario=="cluster-provider-collision" then [{metadata:{name:"arc-runtime-platform-app"}}] else [] end)}' ;; + *'get providerconfigs.github.upbound.io '*) + jq -cn --arg scenario "$GROUP_CASE" '{items:(if $scenario=="legacy-provider-collision" then [{metadata:{name:"arc-runtime-platform-app"}}] else [] end)}' ;; + *'get kustomization '*) + if [[ ( "$GROUP_CASE" == unready-infrastructure && "$*" == *'get kustomization infrastructure '* ) || + ( "$GROUP_CASE" == unready-apps && "$*" == *'get kustomization apps '* ) ]]; then + printf '{"metadata":{"generation":2},"status":{"observedGeneration":1,"conditions":[]}}' + exit 0 + fi + jq -cn --arg revision "latest@$GROUP_DIGEST" '{metadata:{generation:1},status:{observedGeneration:1,lastAppliedRevision:$revision,conditions:[{type:"Ready",status:"True"}]}}' ;; + *'get runnergroups.actions.github.m.upbound.io platform '*) + [[ "$GROUP_CASE" != unreadable-group ]] || exit 1 + jq -cn --arg scenario "$GROUP_CASE" '{ + apiVersion:"actions.github.m.upbound.io/v1alpha1",kind:"RunnerGroup", + metadata:{name:"platform",namespace:"arc-runners",uid:"00000000-0000-0000-0000-000000000002",generation:2,annotations:{"crossplane.io/external-name":"27"}}, + spec:{providerConfigRef:{name:"arc-runtime-platform-app",kind:"ProviderConfig"},forProvider:{name:"platform",visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:true,selectedRepositoryIds:[737584922],selectedWorkflows:["devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main"]}}, + status:{atProvider:{id:"27",name:"platform",default:false,inherited:false,visibility:"selected",allowsPublicRepositories:true,restrictedToWorkflows:true,selectedRepositoryIds:[737584922],selectedWorkflows:["devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main"], + runnersUrl:"https://api.github.com/orgs/devantler-tech/actions/runner-groups/27/runners", + selectedRepositoriesUrl:"https://api.github.com/orgs/devantler-tech/actions/runner-groups/27/repositories"}, + conditions:[{type:"Ready",status:"True",observedGeneration:2},{type:"Synced",status:"True",observedGeneration:2}]} + } | + if $scenario=="all-repositories" then .status.atProvider.visibility="all" + elif $scenario=="extra-repository" then .status.atProvider.selectedRepositoryIds+=[1] + elif $scenario=="default-group" then .status.atProvider.default=true + elif $scenario=="inherited-group" then .status.atProvider.inherited=true + elif $scenario=="wrong-remote-org" then .status.atProvider.runnersUrl="https://api.github.com/orgs/other/actions/runner-groups/27/runners" + elif $scenario=="stale-observation" then .status.conditions[1].observedGeneration=1 + elif $scenario=="missing-observation" then del(.status.conditions[1].observedGeneration) + elif $scenario=="unbound-id" then .metadata.annotations["crossplane.io/external-name"]="1" + elif $scenario=="unrestricted-workflows" then .status.atProvider.restrictedToWorkflows=false + elif $scenario=="extra-workflow" then .status.atProvider.selectedWorkflows+=["devantler-tech/ksail/.github/workflows/ci.yaml@refs/heads/main"] + elif $scenario=="wrong-workflow-ref" then .status.atProvider.selectedWorkflows=["devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/pull/1/head"] + elif $scenario=="unrestricted-desired" then .spec.forProvider.restrictedToWorkflows=false + else . end' ;; + *'get autoscalingrunnerset.actions.github.com platform-linux '*) + jq -cn --arg scenario "$GROUP_CASE" '{metadata:{generation:1,annotations:{"runner-scale-set-id":"12","actions.github.com/runner-group-name":"platform","actions.github.com/runner-scale-set-name":"platform-linux"}}, + spec:{githubConfigUrl:"https://github.com/devantler-tech",runnerGroup:"platform",runnerScaleSetName:"platform-linux"}, + status:{phase:"Running",observedGeneration:1}} | + if $scenario=="wrong-registered-group" then .metadata.annotations["actions.github.com/runner-group-name"]="Default" else . end' ;; + *) exit 93 ;; +esac +SH +chmod 700 "$scratch/bin/"* +for scenario in normal all-repositories extra-repository default-group inherited-group wrong-remote-org \ + stale-observation missing-observation unbound-id unreadable-group wrong-registered-group \ + unrestricted-workflows extra-workflow wrong-workflow-ref unrestricted-desired \ + duplicate-provider wrong-provider-secret cluster-provider-collision legacy-provider-collision unreadable-provider \ + unready-infrastructure unready-apps; do + if PATH="$scratch/bin:$PATH" GROUP_DIGEST="sha256:$(printf 'a%.0s' {1..64})" GROUP_CASE="$scenario" \ + GITHUB_ACTIONS=true GITHUB_REPOSITORY=devantler-tech/platform \ + bash "$root/scripts/wait-for-ksail-arc-registration.sh" "sha256:$(printf 'a%.0s' {1..64})" \ + >"$scratch/out" 2>"$scratch/err"; then + [[ "$scenario" == normal ]] || { echo "accepted $scenario"; exit 1; } + else + [[ "$scenario" != normal ]] || { cat "$scratch/err"; exit 1; } + case "$scenario" in + duplicate-provider|wrong-provider-secret|unreadable-provider) label=provider-config ;; + cluster-provider-collision) label=provider-collision-cluster ;; + legacy-provider-collision) label=provider-collision-legacy ;; + wrong-registered-group) label=runner-scale-set ;; + unready-infrastructure) label=flux-infrastructure ;; + unready-apps) label=flux-apps ;; + *) label=runner-group ;; + esac + grep -Fxq "ARC registration pending: $label" "$scratch/err" || { + printf 'FAIL: missing safe diagnostic for %s\n' "$scenario" >&2; exit 1; + } + fi + if grep -Fq 'private-registration-error-canary' "$scratch/out" "$scratch/err"; then + printf 'FAIL: raw registration error escaped\n' >&2; exit 1 + fi +done +printf 'PASS: organization registration requires an observed KSail-only group\n' diff --git a/scripts/validate-eks-ci-role-policy/approved-surface-history.md b/scripts/validate-eks-ci-role-policy/approved-surface-history.md index 2e5631b777..8a9b7e6821 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface-history.md +++ b/scripts/validate-eks-ci-role-policy/approved-surface-history.md @@ -3322,3 +3322,56 @@ devantler-tech/ksail could list usage but not the limit. These values were computed locally with kubectl v1.37.1 / Kustomize v5.8.1, which is not the approved renderer; the required CI job re-renders with the approved kubectl v1.36.2 / Kustomize v5.8.1 and is the authority. + +## 2026-10-09 โ€” ARC metrics source repair retains its authorization boundary + +The checksum-verified kubectl v1.36.2 / Kustomize v5.8.1 renderer reproduces +exactly one changed authorization entry for #4544 at `d82bcd49`, compared with +its preceding reviewed source at `b0a77dd4`: HelmRelease +arc-runners/platform-runners. Its fingerprint moves from +`0dd16e9d373fe4e3aa26d6b0fc3e31d4d977cf623acfc0e172b8c5b99033a95f` to +`52f89508024a6a9d9547f85a1f292b6fa05819f679fed5f5e3ee3972cd6eb388`. + +All five production layers contain 686 rendered documents before and after. +The generated metrics ConfigMap changes name from +`ksail-arc-job-metrics-d6e7291092` to `ksail-arc-job-metrics-6f02a2616f`; the +HelmRelease's corresponding ConfigMap volume reference is its only scalar +change. Every other complete rendered document is equal. Integrating main at +`ce0be361` adds no rendered change. The metrics script repair makes regular +offline fixture reads portable and accepts the optional kernel counter while +retaining bounded production reads and every mandatory memory/OOM counter. +No IAM/RBAC grant, App permission, runner identity, image, resource bound or +security setting changes. + +The old entry fails with precisely this mismatch. Renewing only this entry +passes the complete authorization package (74 tests and 253 subtests, no +skips) and its CLI. Negative controls retain rejection of expanded grants, +missing, duplicated or newly added authorization resources, changed Flux +handoff and unauthorized AWS identity bindings. Local validation uses cached +Go 1.26.9; required native CI uses the repository's Go 1.26.6 pin and remains +the authority. This approval establishes no runtime or activation clearance. + +## 2026-10-09 โ€” ARC over-budget measurements remain visible without accepting them + +The approved kubectl v1.36.2 / Kustomize v5.8.1 renderer compares all five +production layers against #4544 at `1451f9cf`. Both renders contain 686 +documents, with no duplicate identities. Only the generated metrics ConfigMap +and its HelmRelease volume reference change. The ConfigMap name moves from +`ksail-arc-job-metrics-6f02a2616f` to `ksail-arc-job-metrics-2fbcf0b1d7`. +Removing that single reference scalar makes the complete HelmRelease equal; +every other complete rendered document is equal. + +The script now emits a canonical measurement for a transient peak above the +kernel limit, then returns the existing budget failure with a nonzero exit. +The 14 GiB limit, positive numeric peak, complete event counters and zero-OOM +requirements remain enforced. Over-budget measurements with invalid events or +OOM events still emit no receipt. Runtime acceptance continues to reject an +over-budget peak. No IAM/RBAC grant, App permission, runner identity, image, +resource bound or security setting changes. + +The validator reproduces only the pool entry's fingerprint change from +`52f89508024a6a9d9547f85a1f292b6fa05819f679fed5f5e3ee3972cd6eb388` to +`442c4bc77b77c3a927b1c479592d6960d535627050bf408ce55e794aa3ec505a`. +Renew only this entry; all authorization checks and negative controls remain +enabled. Local validation uses cached Go 1.26.9; native CI's repository pin +remains authoritative. This audit grants no runtime or activation clearance. diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index 6cd6295a5e..121cbdf50e 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -16,7 +16,9 @@ helm.toolkit.fluxcd.io/v2|HelmRelease|actual-budget|actual-budget adc009b64f2a9d helm.toolkit.fluxcd.io/v2|HelmRelease|arc-ksail-analysis|ksail-analysis-runners f59980a0d056bb796135803ce4cfb1a84947957e943abe9854a78eaf1249b2b0 -helm.toolkit.fluxcd.io/v2|HelmRelease|arc-systems|arc-controller 5af946415fcf05e49fb79e9a07cd82cd1da6d3c8ee33a6ebbe0483fc6e00afa1 +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-runners|platform-runners 442c4bc77b77c3a927b1c479592d6960d535627050bf408ce55e794aa3ec505a + +helm.toolkit.fluxcd.io/v2|HelmRelease|arc-systems|arc-controller 844e32e3408c2ad6e85b10c6628de7208c447ca3f1cffe5a0da74dcf3cdc11cf helm.toolkit.fluxcd.io/v2|HelmRelease|backstage|backstage 5ff19be19e3525e641b0a7a22e2a7047e1fbda16ef9ca7b4fa731ce3a61e686a @@ -406,6 +408,8 @@ source.toolkit.fluxcd.io/v1|HelmRepository|whoami|whoami 28dc88b5a5e69a086e4f7fb source.toolkit.fluxcd.io/v1|OCIRepository|arc-ksail-analysis|ksail-analysis-runners 5f89c75739fa50d998e046bc2a388ccf9c7fb4413377510e43f9f12ae4ff4c70 +source.toolkit.fluxcd.io/v1|OCIRepository|arc-runners|platform-runners 7224602dcdde3bce2c8bdf1c6077806e3b38ee13adfcbbf544418da4fc7e0913 + source.toolkit.fluxcd.io/v1|OCIRepository|arc-systems|arc-controller 64fb325741489bbd5ece40cf2cbf6e4aef3e87e89efea97b7b0f2ec498671fa1 source.toolkit.fluxcd.io/v1|OCIRepository|ascoachingogvaner|ascoachingogvaner e7717c1d618e5d2dd1f3ab32d589ac06205d040e867f33db22ab9651bcb105bc diff --git a/scripts/verify-ksail-arc-runtime.sh b/scripts/verify-ksail-arc-runtime.sh new file mode 100755 index 0000000000..d96f8c89d3 --- /dev/null +++ b/scripts/verify-ksail-arc-runtime.sh @@ -0,0 +1,532 @@ +#!/usr/bin/env bash +# CI-owned, bounded runtime acceptance. No Secret or generated JIT configuration is read. +set -Eeuo pipefail +umask 077 + +fail() { printf 'ARC acceptance: FAIL at %s\n' "$1" >&2; exit 1; } +[[ "$#" -eq 1 && "$1" == --if-active ]] || fail arguments +readonly controller_overlay=k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml +readonly pool_overlay=k8s/providers/hetzner/infrastructure/kustomization.yaml +controller_active=$(yq '[.resources[] | select(. == "../../../../bases/infrastructure/controllers/actions-runner-controller/")] | length' "$controller_overlay") +pool_active=$(yq '[.resources[] | select(. == "../../../bases/infrastructure/actions-runners/")] | length' "$pool_overlay") +# Render every independently reconciled production layer before deciding to +# skip. Legacy bases, aliases and overlay patches cannot activate ARC unseen. +source_scratch=$(mktemp -d) +trap 'rm -rf "$source_scratch"' EXIT +for overlay in k8s/providers/hetzner/apps k8s/providers/hetzner/infrastructure \ + k8s/providers/hetzner/infrastructure/controllers k8s/clusters/prod/bootstrap k8s/clusters/prod; do + timeout 95s kubectl kustomize "$overlay" >>"$source_scratch/rendered.yaml" \ + 2>"$source_scratch/render-error" || fail source-render + printf '\n---\n' >>"$source_scratch/rendered.yaml" +done +yq ea -o=json -I=0 '[select(.kind == "HelmRelease" and + ((.metadata.namespace == "arc-systems" and .metadata.name == "arc-controller") or + (.metadata.namespace == "arc-runners" and .metadata.name == "platform-runners") or + .metadata.namespace == "arc-ksail-analysis"))]' \ + "$source_scratch/rendered.yaml" >"$source_scratch/releases.json" +rendered_controller=$(jq '[.[] | select(.metadata.namespace == "arc-systems")] | length' "$source_scratch/releases.json") +rendered_pool=$(jq '[.[] | select(.metadata.namespace == "arc-runners")] | length' "$source_scratch/releases.json") +rendered_legacy=$(jq '[.[] | select(.metadata.namespace == "arc-ksail-analysis")] | length' "$source_scratch/releases.json") +[[ "$rendered_legacy" -le 1 ]] || fail retained-source-state +jq -e 'all(.[] | select(.metadata.namespace == "arc-ksail-analysis"); + .metadata.name == "ksail-analysis-runners" and .spec.suspend == false and + .spec.values.runnerScaleSetName == "ksail-code-quality" and + .spec.values.minRunners == 0 and .spec.values.maxRunners == 0)' \ + "$source_scratch/releases.json" >/dev/null || fail retained-source-state +if [[ "$controller_active" == 0 && "$pool_active" == 0 && "$rendered_controller" == 0 && "$rendered_pool" == 0 ]]; then + printf 'ARC acceptance: inactive source; no runtime access\n' + exit 0 +fi +if [[ "$controller_active" == 1 && "$pool_active" == 0 && "$rendered_controller" == 1 && "$rendered_pool" == 0 ]] && + jq -e 'all(.[] | select(.metadata.namespace == "arc-systems"); .spec.suspend == true or + (.spec.suspend == false and .metadata.annotations["platform.devantler.tech/arc-recovery"] == "drain-only" and + .spec.values.flags.watchSingleNamespace == "arc-runners"))' "$source_scratch/releases.json" >/dev/null; then + printf 'ARC acceptance: inactive source; no runtime access\n' + exit 0 +fi +[[ "$controller_active" == 1 && "$pool_active" == 1 && "$rendered_controller" == 1 && "$rendered_pool" == 1 ]] || fail partial-activation +source_image=$(yq '.spec.values.template.spec.containers[0].image' k8s/bases/infrastructure/actions-runners/helm-release.yaml) +jq -e --arg image "$source_image" 'all(.[] | select(.metadata.namespace != "arc-ksail-analysis"); .spec.suspend == false) and + all(.[] | select(.metadata.namespace == "arc-runners"); + .spec.values.template.spec.containers[0].image == $image and + .spec.values.template.spec.initContainers[0].image == $image)' "$source_scratch/releases.json" >/dev/null || fail rendered-source-state +rm -rf "$source_scratch" +trap - EXIT +[[ "${GITHUB_ACTIONS:-}" == true && "${GITHUB_REPOSITORY:-}" == devantler-tech/platform ]] || fail deployment-identity +[[ "${GITHUB_RUN_ID:-}" =~ ^[1-9][0-9]{0,19}$ && "${GITHUB_RUN_ATTEMPT:-}" =~ ^[1-9][0-9]{0,2}$ ]] || fail run-identity +[[ "${PLATFORM_MANIFEST_DIGEST:-}" =~ ^sha256:[0-9a-f]{64}$ ]] || fail revision +readonly context=admin@prod namespace=arc-runners +readonly ownership_label=platform.devantler.tech/arc-runtime-probe +readonly probe="arc-proof-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" +readonly fence_name=arc-runtime-admission +scratch=$(mktemp -d) +readonly scratch +created=false +probe_uid='' +fence_created=false +fence_uid='' +stage=initialization +trap 'printf "ARC acceptance: FAIL at %s\n" "$stage" >&2' ERR + +kc() { timeout 35s kubectl --context "$context" --request-timeout=30s "$@" 2>/dev/null; } +wait_ready() { + timeout 630s kubectl --context "$context" --request-timeout=610s -n "$namespace" \ + wait "pod/$probe" --for=condition=Ready --timeout=600s 2>/dev/null +} +quiet() { + local code=0 + "$@" >/dev/null 2>"$scratch/command-error" || code=$? + if [[ "$code" != 0 ]]; then printf 'ARC acceptance: FAIL at %s\n' "$stage" >&2; fi + return "$code" +} +# shellcheck source=scripts/ksail-arc-admission-fence.sh +source scripts/ksail-arc-admission-fence.sh +verify_admission_fence() { + kc -n "$namespace" get resourcequota "$fence_name" -o json >"$scratch/live-fence.json" || return 1 + [[ -s "$scratch/live-fence.json" ]] || return 1 + jq -e --arg name "$fence_name" --arg ns "$namespace" --arg owner "$probe" --arg key "$ownership_label" --arg uid "$fence_uid" \ + --arg origin_key "$arc_origin_key" --slurpfile origin "$scratch/current-origin.json" ' + .metadata.name == $name and .metadata.labels[$key] == $owner and .metadata.uid == $uid and + .metadata.namespace == $ns and (.metadata.ownerReferences // [] | length) == 0 and + .metadata.annotations[$origin_key] == ($origin[0]|tojson) and + .metadata.deletionTimestamp == null and .spec == {hard:{pods:"0"},scopes:["NotTerminating"]} and + .status.hard == {pods:"0"} + ' "$scratch/live-fence.json" >/dev/null +} +verify_runner_deadlines() { + local resource path line first + # Project only metadata and the deadline: EphemeralRunner objects can carry + # generated JIT credentials, which this acceptance must never read. + for resource in ephemeralrunnersets.actions.github.com ephemeralrunners.actions.github.com; do + if [[ "$resource" == ephemeralrunnersets.actions.github.com ]]; then + path='{.spec.ephemeralRunnerSpec.spec.activeDeadlineSeconds}' + else + path='{.spec.spec.activeDeadlineSeconds}' + fi + kc -n "$namespace" get "$resource" \ + -o "jsonpath={\"continue=\"}{.metadata.continue}{\"\\n\"}{range .items[*]}{\"uid=\"}{.metadata.uid}{\";deadline=\"}$path{\"\\n\"}{end}" \ + >"$scratch/runner-deadlines" + first=true + while IFS= read -r line; do + if "$first"; then [[ "$line" == continue= ]] || return 1; first=false + else [[ "$line" =~ ^uid=[a-zA-Z0-9-]+\;deadline=$ ]] || return 1; fi + done <"$scratch/runner-deadlines" + ! "$first" || return 1 + done +} +verify_runner_source() { + verify_runner_deadlines + kc -n "$namespace" get autoscalingrunnerset.actions.github.com platform-linux -o json >"$scratch/ars-current.json" + jq -e --slurpfile before "$scratch/ars.json" '.metadata.uid == $before[0].metadata.uid and + .metadata.generation == $before[0].metadata.generation and .spec == $before[0].spec' \ + "$scratch/ars-current.json" >/dev/null +} + +cleanup() { + local result=$? failed=0 remaining='' + trap - EXIT ERR INT TERM + if "$created"; then + # A timed-out create may have succeeded; without its response UID, fail closed. + # A DELETE precondition binds the mutation atomically, including replacement races. + local current + if [[ -z "$probe_uid" ]]; then + failed=1 + else + current=$(kc -n "$namespace" get pod "$probe" --ignore-not-found -o json) || failed=1 + if [[ -n "$current" ]]; then + if ! jq -e --arg name "$probe" --arg key "$ownership_label" --arg owner "$probe" --arg uid "$probe_uid" \ + '.metadata.name == $name and .metadata.labels[$key] == $owner and .metadata.uid == $uid' \ + <<<"$current" >/dev/null; then + failed=1 + else + jq -n --arg uid "$probe_uid" '{apiVersion:"v1",kind:"DeleteOptions", + preconditions:{uid:$uid},gracePeriodSeconds:5,propagationPolicy:"Background"}' >"$scratch/delete.json" + quiet kc delete --raw="/api/v1/namespaces/$namespace/pods/$probe" -f "$scratch/delete.json" || failed=1 + quiet kc -n "$namespace" wait "pod/$probe" --for=delete --timeout=25s || failed=1 + fi + fi + fi + current=$(kc -n "$namespace" get pod "$probe" --ignore-not-found -o json) || failed=1 + [[ -z "$current" ]] || failed=1 + remaining=$(kc -n "$namespace" get pods -l "$ownership_label=$probe" -o json) || failed=1 + jq -e '.items | length == 0' <<<"$remaining" >/dev/null || failed=1 + # The dedicated min-zero pool may take the configured unneeded interval to drain. + # Never delete nodes or resize a shared pool to manufacture cleanup. + local deadline=$((SECONDS + 1200)) + while ((SECONDS < deadline)); do + verify_admission_fence || { failed=1; break; } + remaining=$(kc get nodes -o json) || { failed=1; break; } + if jq -e '.items | all(.[]; .metadata.labels["platform.devantler.tech/ci-runner"] != "enabled" and + (.metadata.name | startswith("autoscale-arc-runners-") | not))' <<<"$remaining" >/dev/null; then break; fi + sleep 10 + done + jq -e '.items | all(.[]; .metadata.labels["platform.devantler.tech/ci-runner"] != "enabled" and + (.metadata.name | startswith("autoscale-arc-runners-") | not))' <<<"$remaining" >/dev/null || failed=1 + fi + if "$fence_created" && [[ "$failed" == 0 ]]; then + # Keep admission closed until the probe and its node are gone. A replacement + # quota, a timed-out create with no UID, or an incomplete cleanup needs recovery. + local fence + if [[ -z "$fence_uid" ]]; then + failed=1 + else + if ! verify_admission_fence || ! jq -e ' + .metadata.resourceVersion | type == "string" and length > 0 + ' "$scratch/live-fence.json" >/dev/null; then + failed=1 + else + jq '{apiVersion:"v1",kind:"DeleteOptions",preconditions:{uid:.metadata.uid, + resourceVersion:.metadata.resourceVersion},propagationPolicy:"Background"}' \ + "$scratch/live-fence.json" >"$scratch/delete-fence.json" + quiet kc delete --raw="/api/v1/namespaces/$namespace/resourcequotas/$fence_name" \ + -f "$scratch/delete-fence.json" || failed=1 + fence=$(kc -n "$namespace" get resourcequota "$fence_name" --ignore-not-found -o json) || failed=1 + [[ -z "$fence" ]] || failed=1 + fi + fi + fi + rm -rf "$scratch" + if [[ "$failed" != 0 ]]; then printf 'ARC acceptance: FAIL cleanup\n' >&2; exit 4; fi + printf 'ARC acceptance: cleanup verified\n' + exit "$result" +} +trap cleanup EXIT +trap 'exit 143' INT TERM + +stage=deployed-revision +quiet bash scripts/wait-for-platform-flux-revision.sh "$PLATFORM_MANIFEST_DIGEST" +# Helm readiness precedes asynchronous ARC registration. Await the full +# revision/generation join before taking the immutable acceptance snapshots. +stage=registration-convergence +if ! quiet timeout 660s bash scripts/wait-for-ksail-arc-registration.sh "$PLATFORM_MANIFEST_DIGEST"; then + # Preserve fixed check labels even when the outer timeout stops the waiter. + # Captured command diagnostics may contain private cluster values. + while IFS= read -r line; do + case "$line" in + 'ARC registration pending: flux-infrastructure'|\ + 'ARC registration pending: flux-apps'|\ + 'ARC registration pending: provider-config'|\ + 'ARC registration pending: provider-collision-cluster'|\ + 'ARC registration pending: provider-collision-legacy'|\ + 'ARC registration pending: runner-group'|\ + 'ARC registration pending: runner-scale-set') printf '%s\n' "$line" >&2 ;; + esac + done <"$scratch/command-error" + fail registration-convergence +fi +for layer in infrastructure apps; do + kc -n flux-system get kustomization "$layer" -o json >"$scratch/layer.json" + jq -e --arg revision "latest@$PLATFORM_MANIFEST_DIGEST" \ + '.status.lastAppliedRevision == $revision and .status.observedGeneration == .metadata.generation and + any(.status.conditions[]; .type == "Ready" and .status == "True")' "$scratch/layer.json" >/dev/null +done + +stage=registration-and-bounds +kc -n "$namespace" get autoscalingrunnerset.actions.github.com platform-linux -o json >"$scratch/ars.json" +jq -e ' + .spec.template.spec.volumes[2].configMap.name as $metrics | + (.metadata.uid | type == "string" and length > 0) and + (.metadata.generation | type == "number" and . > 0) and + .status.phase == "Running" and .status.observedGeneration == .metadata.generation and + ((.metadata.annotations["runner-scale-set-id"] | tonumber) > 0) and + .spec.githubConfigUrl == "https://github.com/devantler-tech" and + .spec.runnerGroup == "platform" and + .metadata.annotations["actions.github.com/runner-group-name"] == "platform" and + .metadata.annotations["actions.github.com/runner-scale-set-name"] == "platform-linux" and + .spec.githubConfigSecret == "arc-github-app" and .spec.runnerScaleSetName == "platform-linux" and + .spec.minRunners == 0 and .spec.maxRunners == 1 and + .spec.template.spec.activeDeadlineSeconds == null and + .spec.template.spec.serviceAccountName == "platform-linux-gha-rs-no-permission" and + .spec.template.spec.automountServiceAccountToken == false and + .spec.template.spec.securityContext.runAsUser == 1001 and + .spec.template.spec.securityContext.runAsGroup == 1001 and + .spec.template.spec.securityContext.fsGroup == 1001 and + .spec.template.spec.securityContext.seccompProfile.type == "RuntimeDefault" and + (.spec.template.spec.containers | length) == 1 and + (.spec.template.spec.initContainers | length) == 1 and + .spec.template.spec.containers[0].resources.requests.memory == "12Gi" and + .spec.template.spec.containers[0].resources.limits.memory == "14Gi" and + .spec.template.spec.containers[0].resources.requests.cpu == "3" and + .spec.template.spec.containers[0].resources.limits.cpu == "3500m" and + .spec.template.spec.containers[0].resources.requests["ephemeral-storage"] == "32Gi" and + .spec.template.spec.containers[0].resources.limits["ephemeral-storage"] == "48Gi" and + .spec.template.spec.nodeSelector["platform.devantler.tech/ci-runner"] == "enabled" and + .spec.template.spec.tolerations == [{"key":"platform.devantler.tech/ci-runner","operator":"Equal","value":"enabled","effect":"NoSchedule"}] and + ($metrics | test("^ksail-arc-job-metrics-[a-z0-9]{10}$")) and + .spec.template.spec.volumes == [{"name":"runner-home","emptyDir":{"sizeLimit":"40Gi"}},{"name":"runner-tmp","emptyDir":{"sizeLimit":"2Gi"}}, + {"name":"runner-metrics","configMap":{"name":$metrics,"defaultMode":365,"items":[{"key":"job-metrics.sh","path":"job-metrics.sh"}]}}] and + .spec.template.spec.containers[0].env == [{"name":"ACTIONS_RUNNER_HOOK_JOB_COMPLETED","value":"/etc/ksail-arc-metrics/job-metrics.sh"}] and + .spec.template.spec.containers[0].volumeMounts == [{"name":"runner-home","mountPath":"/home/runner"},{"name":"runner-tmp","mountPath":"/tmp"}, + {"name":"runner-metrics","mountPath":"/etc/ksail-arc-metrics","readOnly":true}] and + (.spec.template.spec.initContainers[0].env // [] | length) == 0 and + .spec.template.spec.initContainers[0].volumeMounts == [{"name":"runner-home","mountPath":"/runner-data"}] and + all((.spec.template.spec.containers + .spec.template.spec.initContainers)[]; + .securityContext.readOnlyRootFilesystem == true and .securityContext.privileged == false and + .securityContext.allowPrivilegeEscalation == false and .securityContext.capabilities.drop == ["ALL"] and + .envFrom == null and + (.image | test("^ghcr.io/devantler-tech/ksail-analysis-runner@sha256:[0-9a-f]{64}$"))) and + .spec.template.spec.initContainers[0].image == .spec.template.spec.containers[0].image +' "$scratch/ars.json" >/dev/null +stage=immutable-job-metrics +metrics_name=$(jq -er '.spec.template.spec.volumes[2].configMap.name' "$scratch/ars.json") +kc -n "$namespace" get configmap "$metrics_name" -o json >"$scratch/metrics-config.json" +jq -e --rawfile expected scripts/ksail-arc-job-metrics.sh ' + .immutable == true and (.metadata.uid | type == "string" and length > 0) and + .metadata.annotations["kustomize.toolkit.fluxcd.io/substitute"] == "disabled" and + (.data | length) == 1 and (.binaryData // {} | length) == 0 and .data["job-metrics.sh"] == $expected +' "$scratch/metrics-config.json" >/dev/null +image=$(jq -r '.spec.template.spec.containers[0].image' "$scratch/ars.json") +expected_image=$(yq '.spec.values.template.spec.containers[0].image' k8s/bases/infrastructure/actions-runners/helm-release.yaml) +[[ "$image" == "$expected_image" ]] || fail image-drift +stage=immutable-image +pinned_digest=${image##*@} +quiet timeout 90s cosign verify \ + --certificate-identity 'https://github.com/devantler-tech/ksail/.github/workflows/publish-ksail-analysis-runner.yaml@refs/heads/main' \ + --certificate-oidc-issuer 'https://token.actions.githubusercontent.com' "$image" +timeout 90s docker buildx imagetools inspect "$image" --raw >"$scratch/image.json" 2>"$scratch/registry-error" +[[ ! -s "$scratch/registry-error" ]] || fail registry-diagnostics +go run ./scripts/verify-ksail-arc-runtime verify-image --digest "$pinned_digest" \ + <"$scratch/image.json" >"$scratch/runtime-digest" +runtime_digest=$(cat "$scratch/runtime-digest") +readonly runtime_image="ghcr.io/devantler-tech/ksail-analysis-runner@$runtime_digest" +if [[ "$runtime_digest" != "$pinned_digest" ]]; then + timeout 90s docker buildx imagetools inspect "$runtime_image" --raw >"$scratch/runtime-image.json" 2>"$scratch/registry-error" + [[ ! -s "$scratch/registry-error" ]] || fail registry-diagnostics + go run ./scripts/verify-ksail-arc-runtime verify-image --digest "$runtime_digest" \ + <"$scratch/runtime-image.json" >"$scratch/runtime-readback" + cmp -s "$scratch/runtime-digest" "$scratch/runtime-readback" || fail runtime-descriptor +fi +stage=runner-admission-fence +# A scoped quota leaves every existing job running and rejects new ordinary +# runner Pods, including creations already queued by a stale ARC controller. +# The bounded probe has activeDeadlineSeconds and is outside NotTerminating. +# Source: https://kubernetes.io/docs/concepts/policy/resource-quotas/ +verify_runner_deadlines +arc_current_origin +stage=runner-fence-recovery +arc_recover_admission_fence +stage=runner-admission-fence +kc -n "$namespace" get resourcequotas -l "$ownership_label" -o json \ + | jq -e '.items | length == 0' >/dev/null +jq -n --arg name "$fence_name" --arg owner "$probe" --arg ns "$namespace" --arg key "$ownership_label" \ + --arg origin_key "$arc_origin_key" --slurpfile origin "$scratch/current-origin.json" ' + {apiVersion:"v1",kind:"ResourceQuota",metadata:{name:$name,namespace:$ns,labels:{($key):$owner}, + annotations:{($origin_key):($origin[0]|tojson)}}, + spec:{hard:{pods:"0"},scopes:["NotTerminating"]}}' >"$scratch/fence.json" +fence_created=true +kc create -f "$scratch/fence.json" -o json >"$scratch/created-fence.json" +fence_uid=$(jq -er --arg name "$fence_name" --arg owner "$probe" --arg ns "$namespace" --arg key "$ownership_label" ' + select(.metadata.name == $name and .metadata.namespace == $ns and .metadata.labels[$key] == $owner) | + .metadata.uid | select(type == "string" and length > 0)' "$scratch/created-fence.json") +fence_deadline=$((SECONDS + 120)) +while ((SECONDS < fence_deadline)); do + kc -n "$namespace" get resourcequota "$fence_name" -o json >"$scratch/live-fence.json" + if jq -e --arg uid "$fence_uid" '.metadata.uid == $uid and .spec == {hard:{pods:"0"},scopes:["NotTerminating"]} and + .status.hard == {pods:"0"}' "$scratch/live-fence.json" >/dev/null; then break; fi + sleep 10 +done +jq -e --arg uid "$fence_uid" '.metadata.uid == $uid and .spec == {hard:{pods:"0"},scopes:["NotTerminating"]} and + .status.hard == {pods:"0"}' "$scratch/live-fence.json" >/dev/null +# Prove the actual API denies the unchanged runner template for this quota. +# A transport error, unrelated admission denial, or dry-run success proves nothing. +jq --arg name "$probe-admission" --arg ns "$namespace" ' + {apiVersion:"v1",kind:"Pod",metadata:{name:$name,namespace:$ns, + labels:.spec.template.metadata.labels},spec:.spec.template.spec}' "$scratch/ars.json" >"$scratch/runner-admission.json" +if timeout 35s kubectl --context "$context" --request-timeout=30s create --dry-run=server \ + -f "$scratch/runner-admission.json" >"$scratch/admission-out" 2>"$scratch/admission-error"; then + fail runner-admission-fence +fi +grep -Fq "exceeded quota: $fence_name," "$scratch/admission-error" || fail unproven-runner-admission-fence +stage=runner-job-drain +drain_deadline=$((SECONDS + 1200)) +while ((SECONDS < drain_deadline)); do + kc -n "$namespace" get pods -l platform.devantler.tech/arc-role=runner -o json >"$scratch/runners.json" + kc get nodes -o json >"$scratch/drain-nodes.json" + if jq -e '.items | length == 0' "$scratch/runners.json" >/dev/null && + jq -e '.items | all(.[]; .metadata.labels["platform.devantler.tech/ci-runner"] != "enabled" and + (.metadata.name | startswith("autoscale-arc-runners-") | not))' "$scratch/drain-nodes.json" >/dev/null; then break; fi + sleep 10 +done +jq -e '.items | length == 0' "$scratch/runners.json" >/dev/null +jq -e '.items | all(.[]; .metadata.labels["platform.devantler.tech/ci-runner"] != "enabled" and + (.metadata.name | startswith("autoscale-arc-runners-") | not))' "$scratch/drain-nodes.json" >/dev/null +verify_runner_source +stage=idle-pool-baseline +kc -n kube-system get deployment cluster-autoscaler-hetzner-cluster-autoscaler \ + -o jsonpath='{range .spec.template.spec.containers[*].args[*]}{.}{"\n"}{end}' >"$scratch/autoscaler-args" +declared_ceiling=$(grep '^--max-nodes-total=' "$scratch/autoscaler-args") +declared_pool=$(grep '^--nodes=.*:autoscale-arc-runners$' "$scratch/autoscaler-args") +declared_provider=$(grep '^--cloud-provider=' "$scratch/autoscaler-args") +[[ "$declared_ceiling" == --max-nodes-total=9 && \ + "$declared_pool" == --nodes=0:1:cx53:fsn1:autoscale-arc-runners && \ + "$declared_provider" == --cloud-provider=hetzner ]] || fail autoscaler-boundary +kc -n "$namespace" get pods -l platform.devantler.tech/arc-role=runner -o json \ + | jq -e '.items | length == 0' >/dev/null +kc -n "$namespace" get pods -l "$ownership_label" -o json \ + | jq -e '.items | length == 0' >/dev/null +kc get nodes -o json >"$scratch/nodes.json" +jq -e --arg key platform.devantler.tech/ci-runner \ + '(.items | length) < 9 and all(.items[]; .metadata.labels[$key] != "enabled" and + (.metadata.name | startswith("autoscale-arc-runners-") | not) and + (.metadata.uid | type == "string" and length > 0))' "$scratch/nodes.json" >/dev/null + +stage=restricted-admission +kc -n "$namespace" get resourcequotas -o json >"$scratch/quotas.json" +# Only our immutable, deadline-scoped fence excludes this probe. Every other +# quota still goes through the existing conservative budget verification. +jq -e --arg name "$fence_name" --arg owner "$probe" --arg key "$ownership_label" --arg uid "$fence_uid" ' + [.items[] | select(.metadata.name == $name)] as $fences | + ($fences | length) == 1 and $fences[0].metadata.uid == $uid and + $fences[0].metadata.labels[$key] == $owner and + $fences[0].spec == {hard:{pods:"0"},scopes:["NotTerminating"]} and + $fences[0].status.hard == {pods:"0"} +' "$scratch/quotas.json" >/dev/null +jq --arg uid "$fence_uid" '.items |= map(select(.metadata.uid != $uid))' \ + "$scratch/quotas.json" >"$scratch/probe-quotas.json" +quiet go run ./scripts/verify-ksail-arc-runtime verify-quota <"$scratch/probe-quotas.json" +# Retain the actual chart-generated template; replace only process/lifecycle and run identity. +jq --arg name "$probe" --arg ns "$namespace" --arg key "$ownership_label" ' + {apiVersion:"v1",kind:"Pod",metadata:{name:$name,namespace:$ns, + labels:(.spec.template.metadata.labels + {($key):$name})},spec:.spec.template.spec} | + .spec.restartPolicy="Never" | .spec.activeDeadlineSeconds=1200 | + .spec.containers[0].command=["/bin/sh","-ec", + "/usr/local/bin/ksail-analysis-smoke --live-runner; test ! -e /var/run/secrets/kubernetes.io/serviceaccount/token; curl --proto =https --tlsv1.2 --fail --silent --show-error --max-time 20 --output /dev/null https://api.github.com/zen; printf ready > /tmp/arc-proof-ready; sleep 900"] | + .spec.containers[0].readinessProbe={exec:{command:["/bin/sh","-ec","test -f /tmp/arc-proof-ready"]},periodSeconds:2} +' "$scratch/ars.json" >"$scratch/pod.json" +quiet kc create --dry-run=server -f "$scratch/pod.json" +for negative in privileged hostpath; do + if [[ "$negative" == privileged ]]; then + jq '.spec.containers[0].securityContext.privileged=true' "$scratch/pod.json" >"$scratch/negative.json" + else + jq '.spec.volumes += [{"name":"forbidden-host","hostPath":{"path":"/","type":"Directory"}}]' "$scratch/pod.json" >"$scratch/negative.json" + fi + if timeout 35s kubectl --context "$context" --request-timeout=30s create --dry-run=server \ + -f "$scratch/negative.json" >"$scratch/denied-out" 2>"$scratch/denied-error"; then fail negative-admission; fi + # An API/transport failure does not prove admission interception. + grep -Eq 'violates PodSecurity.*restricted|admission webhook.*denied the request' "$scratch/denied-error" || fail unproven-admission-denial + if [[ "$negative" == privileged ]]; then + grep -qi 'privileged' "$scratch/denied-error" || fail unrelated-admission-denial + else + grep -Eqi 'hostPath|host.path' "$scratch/denied-error" || fail unrelated-admission-denial + fi +done +verify_admission_fence +created=true +kc create -f "$scratch/pod.json" -o json >"$scratch/created-pod.json" +probe_uid=$(jq -er --arg name "$probe" --arg ns "$namespace" --arg key "$ownership_label" \ + 'select(.metadata.name == $name and .metadata.namespace == $ns and .metadata.labels[$key] == $name) | + .metadata.uid | select(type == "string" and length > 0)' "$scratch/created-pod.json") +arc_record_probe_uid +quiet wait_ready +kc -n "$namespace" get pod "$probe" -o json >"$scratch/live-pod.json" +jq -e --arg uid "$probe_uid" '.metadata.uid == $uid' "$scratch/live-pod.json" >/dev/null +probe_ip=$(jq -r '.status.podIP' "$scratch/live-pod.json") +probe_node=$(jq -r '.spec.nodeName' "$scratch/live-pod.json") +quiet go run ./scripts/verify-ksail-arc-runtime verify-pod --desired "$scratch/pod.json" \ + --actual "$scratch/live-pod.json" --runtime-image "$runtime_image" +kc get node "$probe_node" -o json >"$scratch/node.json" +jq -e --slurpfile before "$scratch/nodes.json" ' + .metadata.uid as $uid | ($uid | type == "string" and length > 0) and + all($before[0].items[]; .metadata.uid != $uid) and + (.metadata.name | test("^autoscale-arc-runners-[0-9a-f]{1,16}$")) and + .metadata.labels["platform.devantler.tech/ci-runner"] == "enabled" and + .metadata.labels["node.kubernetes.io/instance-type"] == "cx53" and + .status.nodeInfo.operatingSystem == "linux" and .status.nodeInfo.architecture == "amd64" and + any(.spec.taints[]; .key == "platform.devantler.tech/ci-runner" and .value == "enabled" and .effect == "NoSchedule") and + any(.status.conditions[]; .type == "Ready" and .status == "True")' "$scratch/node.json" >/dev/null +node_uid=$(jq -er '.metadata.uid' "$scratch/node.json") +kc get nodes -o json | jq -e --arg uid "$node_uid" --arg name "$probe_node" \ + '(.items | length) <= 9 and any(.items[]; .metadata.uid == $uid and .metadata.name == $name) and + ([.items[] | select(.metadata.labels["platform.devantler.tech/ci-runner"] == "enabled" or + (.metadata.name | startswith("autoscale-arc-runners-")))] | length) == 1' >/dev/null +stage=allocatable-headroom +# Print only UIDs/phases and resource quantities, never runner environment/JIT data. +kc get pods -A --field-selector "spec.nodeName=$probe_node" -o jsonpath='{range .items[*]}P{"\t"}{.metadata.uid}{"\t"}{.status.phase}{"\n"}{range .spec.containers[*]}R{"\t"}{.resources.requests.cpu}{"\t"}{.resources.requests.memory}{"\t"}{.resources.requests.ephemeral-storage}{"\n"}{end}{range .spec.initContainers[*]}R{"\t"}{.resources.requests.cpu}{"\t"}{.resources.requests.memory}{"\t"}{.resources.requests.ephemeral-storage}{"\n"}{end}R{"\t"}{.spec.overhead.cpu}{"\t"}{.spec.overhead.memory}{"\t"}{.spec.overhead.ephemeral-storage}{"\n"}{end}' >"$scratch/reservations" +quiet go run ./scripts/verify-ksail-arc-runtime verify-budget --node "$scratch/node.json" \ + --reservations "$scratch/reservations" --probe-uid "$probe_uid" + +stage='observer-and-healthy-targets' +kc -n kube-system get pods -l k8s-app=cilium --field-selector "spec.nodeName=$probe_node" -o json \ + | jq -e '[.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))] | + if length == 1 then .[0] else error("observer count") end' >"$scratch/agent.json" +agent=$(jq -r '.metadata.name' "$scratch/agent.json") +kc -n kube-system get configmap cilium-config -o json >"$scratch/cilium-config.json" +cluster=$(jq -er '.data["cluster-name"] | select(length > 0)' "$scratch/cilium-config.json") +observer="$cluster/$probe_node" +quiet kc -n kube-system exec "$agent" -c cilium-agent -- hubble status \ + --server unix:///var/run/cilium/hubble.sock --timeout 5s --request-timeout 5s --output json +kc -n kube-system get pods -l k8s-app=kube-dns -o json \ + | jq -e '[.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))][0] | + select(.metadata.uid != null and .status.podIP != null)' >"$scratch/dns.json" +dns_pod=$(jq -r '.metadata.name' "$scratch/dns.json") +dns_ip=$(jq -r '.status.podIP' "$scratch/dns.json") +kc -n default get endpoints kubernetes -o json >"$scratch/api.json" +api_ip=$(jq -er '.subsets[0].addresses[0].ip' "$scratch/api.json") +api_port=$(jq -er '.subsets[0].ports[] | select(.name == "https") | .port' "$scratch/api.json") + +healthy_dns() { + local reply + reply=$(kc get --raw "/api/v1/namespaces/kube-system/pods/$dns_pod:8181/proxy/ready") + [[ "$reply" == OK ]] || fail unhealthy-internal-control +} +healthy_api() { + local reply + reply=$(kc --server="https://$api_ip:$api_port" get --raw /readyz) + [[ "$reply" == ok ]] || fail unhealthy-api-control +} +identity() { + jq -c '{uid:.metadata.uid,ip:.status.podIP,node:.spec.nodeName, + containers:[.status.containerStatuses[]|{name,containerID,restartCount,ready}]}' +} +negative_egress() { + local destination=$1 port=$2 url=$3 target_ns=$4 target_pod=$5 start end result + start=$(date -u +%Y-%m-%dT%H:%M:%S.%NZ) + # Only curl's connect/total timeout counts as a denied attempt; exec/transport failure does not. + if kc -n "$namespace" exec "$probe" -c runner -- curl --noproxy '*' --connect-timeout 3 \ + --max-time 5 --silent --show-error --output /dev/null "$url" >"$scratch/attempt-out" 2>"$scratch/attempt-error"; then + fail network-isolation + else + result=$? + fi + [[ "$result" == 28 ]] || fail unproven-curl-timeout + # Re-prove that exec still reaches this process after the bounded attempt. + quiet kc -n "$namespace" exec "$probe" -c runner -- test -f /tmp/arc-proof-ready + end=$(date -u +%Y-%m-%dT%H:%M:%S.%NZ) + timeout 20s kubectl --context "$context" --request-timeout=15s -n kube-system exec "$agent" -c cilium-agent -- \ + hubble observe --server unix:///var/run/cilium/hubble.sock --timeout 5s --since "$start" --until "$end" \ + --from-ip "$probe_ip" --from-pod "$namespace/$probe" --to-ip "$destination" --to-port "$port" \ + --protocol tcp --traffic-direction egress --verdict DROPPED --drop-reason-desc POLICY_DENIED --output jsonpb \ + >"$scratch/flows" 2>"$scratch/observer-error" + [[ ! -s "$scratch/observer-error" ]] || fail observer-diagnostics + quiet go run ./scripts/verify-ksail-arc-runtime --pod "$probe" --source "$probe_ip" \ + --destination "$destination" --port "$port" --node "$observer" --since "$start" --until "$end" \ + --destination-namespace "$target_ns" --destination-pod "$target_pod" <"$scratch/flows" +} +stage=intercepted-egress-denials +healthy_dns +healthy_api +negative_egress "$dns_ip" 8181 "http://$dns_ip:8181/ready" kube-system "$dns_pod" +negative_egress "$api_ip" "$api_port" "https://$api_ip:$api_port/readyz" '' '' +healthy_dns +healthy_api +kc -n "$namespace" get pod "$probe" -o json | identity >"$scratch/probe-after" +identity <"$scratch/live-pod.json" >"$scratch/probe-before" +cmp -s "$scratch/probe-before" "$scratch/probe-after" || fail probe-churn +kc -n kube-system get pod "$agent" -o json | identity >"$scratch/agent-after" +identity <"$scratch/agent.json" >"$scratch/agent-before" +cmp -s "$scratch/agent-before" "$scratch/agent-after" || fail observer-churn +kc -n kube-system get pod "$dns_pod" -o json | identity >"$scratch/dns-after" +identity <"$scratch/dns.json" >"$scratch/dns-before" +cmp -s "$scratch/dns-before" "$scratch/dns-after" || fail target-churn +kc -n default get endpoints kubernetes -o json | jq -c '.subsets' >"$scratch/api-after" +jq -c '.subsets' "$scratch/api.json" >"$scratch/api-before" +cmp -s "$scratch/api-before" "$scratch/api-after" || fail api-target-churn + +stage='job-cgroup-measurement' +quiet kc -n "$namespace" exec "$probe" -- /etc/ksail-arc-metrics/job-metrics.sh +verify_admission_fence +verify_runner_source +kc -n "$namespace" get pods -l platform.devantler.tech/arc-role=runner -o json \ + | jq -e --arg uid "$probe_uid" '.items | length == 1 and .[0].metadata.uid == $uid' >/dev/null +stage=complete +printf 'PASS: ARC registration, restricted image, admission and two intercepted egress denials; manifest=%s\n' "$PLATFORM_MANIFEST_DIGEST" diff --git a/scripts/verify-ksail-arc-runtime/budget.go b/scripts/verify-ksail-arc-runtime/budget.go new file mode 100644 index 0000000000..24918628f4 --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/budget.go @@ -0,0 +1,157 @@ +package main + +import ( + "bufio" + "encoding/json" + "errors" + "math/big" + "regexp" + "strings" +) + +var quantityPattern = regexp.MustCompile(`^([0-9]+(?:\.[0-9]+)?)([a-zA-Z]*)$`) + +func quantity(value string) (*big.Rat, error) { + if value == "" { + return new(big.Rat), nil + } + parts := quantityPattern.FindStringSubmatch(value) + if parts == nil { + return nil, errors.New("unknown resource quantity") + } + number, ok := new(big.Rat).SetString(parts[1]) + if !ok { + return nil, errors.New("invalid resource quantity") + } + scale := map[string]string{"": "1", "n": "0.000000001", "u": "0.000001", "m": "0.001", + "k": "1000", "M": "1000000", "G": "1000000000", "T": "1000000000000", + "Ki": "1024", "Mi": "1048576", "Gi": "1073741824", "Ti": "1099511627776"} + factor, ok := scale[parts[2]] + if !ok { + return nil, errors.New("unsupported resource unit") + } + ratio, _ := new(big.Rat).SetString(factor) + return number.Mul(number, ratio), nil +} + +// Sum main, init and overhead reservations conservatively. Serial init requests +// are deliberately added rather than subtracted, preserving node headroom. +func verifyBudget(nodeJSON, reservations []byte, probeUID string) error { + var node struct { + Status struct{ Allocatable map[string]string } + } + if json.Unmarshal(nodeJSON, &node) != nil { + return errors.New("invalid node capacity") + } + total := map[string]*big.Rat{} + for resource, budget := range map[string]string{"cpu": "3500m", "memory": "14Gi", "ephemeral-storage": "48Gi"} { + total[resource], _ = quantity(budget) + if node.Status.Allocatable[resource] == "" { + return errors.New("missing allocatable capacity") + } + } + scanner := bufio.NewScanner(strings.NewReader(string(reservations))) + seenHeader, skip := false, false + for scanner.Scan() { + fields := strings.Split(scanner.Text(), "\t") + switch fields[0] { + case "P": + if len(fields) != 3 || fields[1] == "" { + return errors.New("invalid reservation identity") + } + seenHeader = true + skip = fields[1] == probeUID || fields[2] == "Succeeded" || fields[2] == "Failed" + case "R": + if !seenHeader || len(fields) != 4 { + return errors.New("incomplete reservations") + } + if skip { + continue + } + for i, resource := range []string{"cpu", "memory", "ephemeral-storage"} { + value, err := quantity(fields[i+1]) + if err != nil { + return err + } + total[resource].Add(total[resource], value) + } + default: + return errors.New("unknown reservation record") + } + } + if scanner.Err() != nil || !seenHeader { + return errors.New("missing complete reservation read") + } + for resource, reserved := range total { + allocatable, err := quantity(node.Status.Allocatable[resource]) + if err != nil { + return err + } + if reserved.Cmp(allocatable) > 0 { + return errors.New("insufficient allocatable headroom") + } + } + return nil +} + +func verifyQuota(data []byte) error { + var quotas struct { + Items []struct { + Spec struct { + Scopes []string + ScopeSelector json.RawMessage + Hard map[string]string + } + Status struct{ Hard, Used map[string]string } + } + } + if json.Unmarshal(data, "as) != nil || quotas.Items == nil { + return errors.New("invalid quota read") + } + for _, q := range quotas.Items { + if len(q.Spec.Scopes) != 0 || len(q.Spec.ScopeSelector) != 0 { + return errors.New("unresolved scoped quota") + } + if len(q.Spec.Hard) == 0 { + return errors.New("missing declared quota limits") + } + for resource, needed := range map[string]string{ + "pods": "1", "count/pods": "1", "requests.cpu": "3", "requests.memory": "12Gi", "requests.ephemeral-storage": "32Gi", + "limits.cpu": "3500m", "limits.memory": "14Gi", "limits.ephemeral-storage": "48Gi", + "cpu": "3", "memory": "12Gi", "ephemeral-storage": "32Gi", + } { + hard, exists := q.Spec.Hard[resource] + if !exists { + if _, stale := q.Status.Hard[resource]; stale { + return errors.New("stale quota limit") + } + continue + } + accounted, exists := q.Status.Hard[resource] + if !exists { + return errors.New("missing quota limit accounting") + } + used, exists := q.Status.Used[resource] + if !exists { + return errors.New("incomplete quota accounting") + } + limit, err := quantity(hard) + if err != nil { + return err + } + observed, err := quantity(accounted) + if err != nil || observed.Cmp(limit) != 0 { + return errors.New("stale quota limit accounting") + } + current, err := quantity(used) + if err != nil { + return err + } + additional, _ := quantity(needed) + if new(big.Rat).Add(current, additional).Cmp(limit) > 0 { + return errors.New("insufficient namespace quota") + } + } + } + return nil +} diff --git a/scripts/verify-ksail-arc-runtime/budget_test.go b/scripts/verify-ksail-arc-runtime/budget_test.go new file mode 100644 index 0000000000..e94fba5e64 --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/budget_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "strings" + "testing" +) + +func TestCapacityAccountsForNeighborsAndInitReservations(t *testing.T) { + node := `{"status":{"allocatable":{"cpu":"7.8","memory":"30Gi","ephemeral-storage":"100Gi"}}}` + reservations := "P\towned\tRunning\nR\t3\t12Gi\t32Gi\nP\tsystem\tRunning\nR\t200m\t1Gi\t1Gi\nR\t100m\t256Mi\t1Gi\n" + if err := verifyBudget([]byte(node), []byte(reservations), "owned"); err != nil { + t.Fatal(err) + } + for _, item := range []struct{ name, node, records string }{ + {"memory", strings.ReplaceAll(node, "30Gi", "14Gi"), reservations}, + {"cpu", strings.ReplaceAll(node, "7.8", "3.5"), reservations}, + {"storage", strings.ReplaceAll(node, "100Gi", "48Gi"), reservations}, + {"unknown-quantity", node, strings.ReplaceAll(reservations, "200m", "not-known")}, + {"missing-read", node, ""}, + {"truncated-record", node, "P\tsystem\tRunning\nR\t200m\t1Gi"}, + } { + t.Run(item.name, func(t *testing.T) { + if verifyBudget([]byte(item.node), []byte(item.records), "owned") == nil { + t.Fatal("accepted insufficient or unknown headroom") + } + }) + } +} + +func TestQuotaRequiresCompleteHeadroom(t *testing.T) { + good := `{"items":[{"spec":{"hard":{"pods":"3","requests.memory":"16Gi","limits.memory":"16Gi"}},"status":{"hard":{"pods":"3","requests.memory":"16Gi","limits.memory":"16Gi"},"used":{"pods":"1","requests.memory":"1Gi","limits.memory":"1Gi"}}}]}` + if err := verifyQuota([]byte(good)); err != nil { + t.Fatal(err) + } + if err := verifyQuota([]byte(`{"items":[]}`)); err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ name, input, want string }{ + {"pods", strings.ReplaceAll(good, `"pods":"3"`, `"pods":"1"`), "insufficient namespace quota"}, + {"memory", strings.ReplaceAll(good, `"limits.memory":"16Gi"`, `"limits.memory":"14Gi"`), "insufficient namespace quota"}, + {"used accounting", strings.ReplaceAll(good, `"requests.memory":"1Gi",`, ``), "incomplete quota accounting"}, + {"limit accounting", `{"items":[{"spec":{"hard":{"pods":"1"}},"status":{}}]}`, "missing quota limit accounting"}, + {"stale accounting", `{"items":[{"spec":{"hard":{"pods":"1"}},"status":{"hard":{"pods":"5"},"used":{"pods":"1"}}}]}`, "stale quota limit accounting"}, + {"scoped", `{"items":[{"spec":{"hard":{"pods":"1"},"scopes":["Terminating"]},"status":{"hard":{"pods":"1"},"used":{"pods":"0"}}}]}`, "unresolved scoped quota"}, + {"missing read", `{}`, "invalid quota read"}, + {"unknown quantity", `{"items":[{"spec":{"hard":{"pods":"unknown"}},"status":{"hard":{"pods":"unknown"},"used":{"pods":"0"}}}]}`, "unknown resource quantity"}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := verifyQuota([]byte(tc.input)); err == nil || err.Error() != tc.want { + t.Fatalf("got %v, want %q", err, tc.want) + } + }) + } +} diff --git a/scripts/verify-ksail-arc-runtime/denial.go b/scripts/verify-ksail-arc-runtime/denial.go new file mode 100644 index 0000000000..db26e7fa08 --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/denial.go @@ -0,0 +1,94 @@ +package main + +import ( + "encoding/json" + "errors" + "io" + "time" +) + +type denialTarget struct { + namespace, pod, source, destination, node string + destinationNamespace, destinationPod string + port uint16 + since, until time.Time +} + +type endpointIdentity struct { + Namespace string `json:"namespace"` + Pod string `json:"pod_name"` +} +type deniedFlow struct { + Verdict string `json:"verdict"` + Reason string `json:"drop_reason_desc"` + Direction string `json:"traffic_direction"` + Source endpointIdentity `json:"source"` + Destination endpointIdentity `json:"destination"` + IP struct { + Source string `json:"source"` + Destination string `json:"destination"` + } `json:"IP"` + L4 struct { + TCP struct { + DestinationPort uint16 `json:"destination_port"` + Flags struct { + SYN bool `json:"SYN"` + } `json:"flags"` + } `json:"TCP"` + } `json:"l4"` + Node string `json:"node_name"` + Time string `json:"time"` +} + +func verifyDenial(input io.Reader, expected denialTarget) error { + limited := &io.LimitedReader{R: input, N: 4 << 20} + decoder := json.NewDecoder(limited) + matched := false + for { + var response map[string]json.RawMessage + if err := decoder.Decode(&response); err != nil { + if errors.Is(err, io.EOF) { + break + } + return errors.New("malformed observer response") + } + for _, field := range []string{"lost_events", "lostEvents", "node_status", "nodeStatus"} { + if _, exists := response[field]; exists { + return errors.New("incomplete observer evidence") + } + } + raw, exists := response["flow"] + if !exists { + return errors.New("unknown observer response") + } + var flow deniedFlow + if err := json.Unmarshal(raw, &flow); err != nil { + return errors.New("malformed flow") + } + if matchesDenial(flow, expected) { + matched = true + } + } + if limited.N <= 0 { + return errors.New("observer evidence exceeds the bound") + } + if !matched { + return errors.New("no correlated policy denial") + } + return nil +} + +func matchesDenial(flow deniedFlow, expected denialTarget) bool { + observed, err := time.Parse(time.RFC3339Nano, flow.Time) + if err != nil || observed.Before(expected.since) || observed.After(expected.until) { + return false + } + if flow.Verdict != "DROPPED" || flow.Reason != "POLICY_DENIED" || flow.Direction != "EGRESS" || + flow.Source.Namespace != expected.namespace || flow.Source.Pod != expected.pod || + flow.IP.Source != expected.source || flow.IP.Destination != expected.destination || + flow.L4.TCP.DestinationPort != expected.port || !flow.L4.TCP.Flags.SYN || flow.Node != expected.node { + return false + } + return (expected.destinationNamespace == "" || flow.Destination.Namespace == expected.destinationNamespace) && + (expected.destinationPod == "" || flow.Destination.Pod == expected.destinationPod) +} diff --git a/scripts/verify-ksail-arc-runtime/denial_test.go b/scripts/verify-ksail-arc-runtime/denial_test.go new file mode 100644 index 0000000000..36782d0ec4 --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/denial_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +func TestPolicyDenialMustBindEveryIdentityAndTheAttemptWindow(t *testing.T) { + t.Parallel() + expected := denialTarget{ + namespace: "arc-runners", pod: "probe-123", source: "192.0.2.2", + destination: "192.0.2.3", port: 8181, node: "default/worker", + since: time.Date(2026, 10, 5, 22, 0, 0, 0, time.UTC), + until: time.Date(2026, 10, 5, 22, 0, 10, 0, time.UTC), + } + valid := `{"flow":{"verdict":"DROPPED","drop_reason_desc":"POLICY_DENIED","traffic_direction":"EGRESS","source":{"namespace":"arc-runners","pod_name":"probe-123"},"IP":{"source":"192.0.2.2","destination":"192.0.2.3"},"l4":{"TCP":{"destination_port":8181,"flags":{"SYN":true}}},"node_name":"default/worker","time":"2026-10-05T22:00:05Z"}}` + cases := []struct { + name, input string + want bool + }{ + {"correlated policy drop", valid, true}, + {"timeout only", "", false}, + {"malformed observer", "{", false}, + {"other source", strings.Replace(valid, "192.0.2.2", "192.0.2.4", 1), false}, + {"same pod prefix", strings.Replace(valid, "probe-123", "probe-123-other", 1), false}, + {"other namespace", strings.Replace(valid, "arc-runners", "other", 1), false}, + {"other destination", strings.Replace(valid, "192.0.2.3", "192.0.2.4", 1), false}, + {"other port", strings.Replace(valid, "8181", "443", 1), false}, + {"other node", strings.Replace(valid, "default/worker", "default/other", 1), false}, + {"outside attempt", strings.Replace(valid, "22:00:05", "21:59:59", 1), false}, + {"unrecognized timestamp", strings.Replace(valid, "2026-10-05T22:00:05Z", "unknown", 1), false}, + {"other direction", strings.Replace(valid, "EGRESS", "INGRESS", 1), false}, + {"other reason", strings.Replace(valid, "POLICY_DENIED", "POLICY_DENY", 1), false}, + {"forwarded event", strings.Replace(valid, "DROPPED", "FORWARDED", 1), false}, + {"not a connection attempt", strings.Replace(valid, `"SYN":true`, `"SYN":false`, 1), false}, + {"lost events", valid + "\n" + `{"lost_events":{"num_events_lost":1}}`, false}, + {"observer node failure", valid + "\n" + `{"node_status":{"state_change":"NODE_ERROR"}}`, false}, + {"trailing malformed response", valid + "\n{", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := verifyDenial(strings.NewReader(tc.input), expected) + if (err == nil) != tc.want { + t.Fatalf("verdict success = %v, want %v", err == nil, tc.want) + } + }) + } +} diff --git a/scripts/verify-ksail-arc-runtime/image.go b/scripts/verify-ksail-arc-runtime/image.go new file mode 100644 index 0000000000..3d24905dce --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/image.go @@ -0,0 +1,63 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" +) + +func runtimeDigest(data []byte, digest string) (string, error) { + if !validDigest(digest) || len(data) > 4<<20 { + return "", errors.New("invalid image evidence") + } + // Buildx --raw may append one display newline. Accept only bytes whose hash + // equals the immutable signed descriptor, never reserialized JSON or a tag. + if fmt.Sprintf("sha256:%x", sha256.Sum256(data)) != digest { + if len(data) == 0 || data[len(data)-1] != '\n' || fmt.Sprintf("sha256:%x", sha256.Sum256(data[:len(data)-1])) != digest { + return "", errors.New("image descriptor mismatch") + } + data = data[:len(data)-1] + } + var manifest struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Manifests []struct { + Digest string `json:"digest"` + Platform struct{ OS, Architecture, Variant string } `json:"platform"` + } `json:"manifests"` + } + if json.Unmarshal(data, &manifest) != nil || manifest.SchemaVersion != 2 { + return "", errors.New("invalid image manifest") + } + switch manifest.MediaType { + case "application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json": + return digest, nil + case "application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json": + runtime := "" + for _, item := range manifest.Manifests { + if item.Platform.OS != "linux" || item.Platform.Architecture != "amd64" { + continue + } + if runtime != "" || item.Platform.Variant != "" || !validDigest(item.Digest) { + return "", errors.New("ambiguous amd64 runtime") + } + runtime = item.Digest + } + if runtime == "" { + return "", errors.New("missing amd64 runtime") + } + return runtime, nil + default: + return "", errors.New("unsupported image manifest") + } +} + +func validDigest(digest string) bool { + if len(digest) != 71 || digest[:7] != "sha256:" { + return false + } + data, err := hex.DecodeString(digest[7:]) + return err == nil && len(data) == 32 +} diff --git a/scripts/verify-ksail-arc-runtime/image_test.go b/scripts/verify-ksail-arc-runtime/image_test.go new file mode 100644 index 0000000000..a4a069f78a --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/image_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "crypto/sha256" + "fmt" + "strings" + "testing" +) + +func TestImmutableRuntimeDescriptorJoin(t *testing.T) { + child := "sha256:" + strings.Repeat("a", 64) + index := fmt.Sprintf(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"digest":%q,"platform":{"os":"linux","architecture":"amd64"}},{"digest":"attestation","platform":{"os":"unknown","architecture":"unknown"}}]}`, child) + digest := func(data string) string { return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(data))) } + for _, suffix := range []string{"", "\n"} { + got, err := runtimeDigest([]byte(index+suffix), digest(index)) + if err != nil || got != child { + t.Fatalf("signed index join: %s %v", got, err) + } + } + for name, data := range map[string]string{ + "tampered": index + " ", + "no-runtime": strings.ReplaceAll(index, "amd64", "arm64"), + "malformed-digest": strings.ReplaceAll(index, child, "latest"), + "unsupported-schema": strings.ReplaceAll(index, `"schemaVersion":2`, `"schemaVersion":1`), + "unknown-media": strings.ReplaceAll(index, "application/vnd.oci.image.index.v1+json", "unknown"), + "trailing-json": index + "{}", + "ambiguous": fmt.Sprintf(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"digest":%q,"platform":{"os":"linux","architecture":"amd64"}},{"digest":%q,"platform":{"os":"linux","architecture":"amd64"}}]}`, child, child), + } { + t.Run(name, func(t *testing.T) { + expected := digest(data) + if name == "tampered" { + expected = digest(index) + } + if _, err := runtimeDigest([]byte(data), expected); err == nil { + t.Fatal("accepted unproven runtime descriptor") + } + }) + } + manifest := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{},"layers":[]}` + if got, err := runtimeDigest([]byte(manifest), digest(manifest)); err != nil || got != digest(manifest) { + t.Fatal(got, err) + } +} diff --git a/scripts/verify-ksail-arc-runtime/main.go b/scripts/verify-ksail-arc-runtime/main.go new file mode 100644 index 0000000000..85028127bf --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/main.go @@ -0,0 +1,120 @@ +package main + +import ( + "flag" + "fmt" + "io" + "net" + "os" + "time" +) + +func main() { + if len(os.Args) > 1 && os.Args[1] == "verify-image" { + flags := flag.NewFlagSet("verify-image", flag.ContinueOnError) + digest := flags.String("digest", "", "signed image descriptor") + err := flags.Parse(os.Args[2:]) + var selected string + if err == nil && flags.NArg() == 0 { + var data []byte + data, err = io.ReadAll(io.LimitReader(os.Stdin, (4<<20)+1)) + if err == nil { + selected, err = runtimeDigest(data, *digest) + } + } + if err != nil || selected == "" { + fmt.Fprintln(os.Stderr, "ARC acceptance: immutable image proof failed") + os.Exit(1) + } + fmt.Println(selected) + return + } + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "ARC acceptance: correlated denial proof failed") + os.Exit(1) + } + fmt.Println("PASS: ARC runtime evidence verified") +} + +func run() error { + if len(os.Args) > 1 && os.Args[1] == "verify-budget" { + flags := flag.NewFlagSet("verify-budget", flag.ContinueOnError) + node := flags.String("node", "", "private node readback") + reservations := flags.String("reservations", "", "filtered resource requests") + uid := flags.String("probe-uid", "", "owned probe UID") + if err := flags.Parse(os.Args[2:]); err != nil { + return err + } + if flags.NArg() != 0 || *node == "" || *reservations == "" || *uid == "" { + return fmt.Errorf("missing capacity readback") + } + data, err := os.ReadFile(*node) + if err != nil { + return err + } + requests, err := os.ReadFile(*reservations) + if err != nil { + return err + } + return verifyBudget(data, requests, *uid) + } + if len(os.Args) > 1 && os.Args[1] == "verify-quota" { + if len(os.Args) != 2 { + return fmt.Errorf("invalid quota arguments") + } + data, err := io.ReadAll(io.LimitReader(os.Stdin, (4<<20)+1)) + if err != nil || len(data) > 4<<20 { + return fmt.Errorf("invalid quota evidence") + } + return verifyQuota(data) + } + if len(os.Args) > 1 && os.Args[1] == "verify-pod" { + flags := flag.NewFlagSet("verify-pod", flag.ContinueOnError) + desired := flags.String("desired", "", "private desired Pod file") + actual := flags.String("actual", "", "private admitted Pod file") + image := flags.String("runtime-image", "", "verified platform image") + if err := flags.Parse(os.Args[2:]); err != nil { + return err + } + if flags.NArg() != 0 || *desired == "" || *actual == "" { + return fmt.Errorf("missing pod readback") + } + want, err := os.ReadFile(*desired) + if err != nil { + return err + } + got, err := os.ReadFile(*actual) + if err != nil { + return err + } + return verifyPod(want, got, *image) + } + pod := flag.String("pod", "", "exact owned probe name") + source := flag.String("source", "", "pinned probe address") + destination := flag.String("destination", "", "healthy target address") + destinationPod := flag.String("destination-pod", "", "optional pinned target pod") + destinationNamespace := flag.String("destination-namespace", "", "optional pinned target namespace") + node := flag.String("node", "", "exact Hubble observer identity") + port := flag.Uint("port", 0, "target TCP port") + since := flag.String("since", "", "attempt start") + until := flag.String("until", "", "attempt end") + flag.Parse() + start, err := time.Parse(time.RFC3339Nano, *since) + if err != nil { + return err + } + end, err := time.Parse(time.RFC3339Nano, *until) + if err != nil { + return err + } + if flag.NArg() != 0 || *pod == "" || *node == "" || + net.ParseIP(*source) == nil || net.ParseIP(*destination) == nil || + *port == 0 || *port > 65535 || end.Before(start) || end.Sub(start) > 30*time.Second { + return fmt.Errorf("invalid bounded target") + } + return verifyDenial(os.Stdin, denialTarget{ + namespace: "arc-runners", pod: *pod, source: *source, destination: *destination, + destinationNamespace: *destinationNamespace, destinationPod: *destinationPod, + node: *node, port: uint16(*port), since: start, until: end, + }) +} diff --git a/scripts/verify-ksail-arc-runtime/pod.go b/scripts/verify-ksail-arc-runtime/pod.go new file mode 100644 index 0000000000..49d3d2811e --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/pod.go @@ -0,0 +1,151 @@ +package main + +import ( + "encoding/hex" + "encoding/json" + "errors" + "reflect" + "strings" +) + +func verifyPod(desired, actual []byte, runtimeImage string) error { + var expected, observed map[string]any + if json.Unmarshal(desired, &expected) != nil || json.Unmarshal(actual, &observed) != nil { + return errors.New("invalid pod readback") + } + want, ok := expected["spec"].(map[string]any) + if !ok { + return errors.New("missing desired spec") + } + got, ok := observed["spec"].(map[string]any) + if !ok { + return errors.New("missing actual spec") + } + if err := normalizePodSpec(want); err != nil { + return err + } + if err := normalizePodSpec(got); err != nil { + return err + } + if !reflect.DeepEqual(want, got) { + return errors.New("admitted pod changed runtime fields") + } + digest := strings.TrimPrefix(runtimeImage, "ghcr.io/devantler-tech/ksail-analysis-runner@sha256:") + decoded, decodeErr := hex.DecodeString(digest) + if digest == runtimeImage || decodeErr != nil || len(decoded) != 32 { + return errors.New("unproven runtime image") + } + status, ok := observed["status"].(map[string]any) + if !ok { + return errors.New("missing runtime status") + } + for _, pair := range [][2]string{{"containers", "containerStatuses"}, {"initContainers", "initContainerStatuses"}} { + containers, ok := got[pair[0]].([]any) + if !ok || len(containers) != 1 { + return errors.New("unexpected container count") + } + statuses, ok := status[pair[1]].([]any) + if !ok || len(statuses) != 1 { + return errors.New("incomplete runtime status") + } + c := containers[0].(map[string]any) + s, ok := statuses[0].(map[string]any) + imageID := strings.TrimPrefix(stringField(s, "imageID"), "docker-pullable://") + // Containerd may report the requested index identity. Both representations + // are bound to the verified signed index and its unique linux/amd64 child; + // the caller separately verifies the actual node architecture. + requested := stringField(c, "image") + if !strings.HasPrefix(requested, "ghcr.io/devantler-tech/ksail-analysis-runner@") || + !validDigest(strings.TrimPrefix(requested, "ghcr.io/devantler-tech/ksail-analysis-runner@")) { + return errors.New("unproven requested image") + } + if !ok || s["name"] != c["name"] || s["restartCount"] != float64(0) || + (imageID != runtimeImage && imageID != requested) { + return errors.New("runtime identity mismatch") + } + if pair[0] == "containers" { + if s["ready"] != true { + return errors.New("runtime not ready") + } + } else { + state, ok := s["state"].(map[string]any) + if !ok { + return errors.New("missing init result") + } + terminated, ok := state["terminated"].(map[string]any) + if !ok || terminated["exitCode"] != float64(0) { + return errors.New("init failed") + } + } + } + return nil +} + +func stringField(m map[string]any, key string) string { value, _ := m[key].(string); return value } + +// Kubernetes core/v1 defaults and the defaulttolerationseconds admission plugin +// add these exact values. Unknown fields and every nondefault value remain in the +// comparison, including added containers, credentials, mounts and permissions. +func normalizePodSpec(spec map[string]any) error { + delete(spec, "nodeName") // Scheduler assignment is verified separately against the dedicated node. + for key, value := range map[string]any{ + "dnsPolicy": "ClusterFirst", "schedulerName": "default-scheduler", + "terminationGracePeriodSeconds": float64(30), "enableServiceLinks": true, + "priority": float64(0), "preemptionPolicy": "PreemptLowerPriority", + } { + removeDefault(spec, key, value) + } + if alias, exists := spec["serviceAccount"]; exists { + if alias != spec["serviceAccountName"] { + return errors.New("service account alias mismatch") + } + delete(spec, "serviceAccount") + } + if tolerations, ok := spec["tolerations"].([]any); ok { + filtered := make([]any, 0, len(tolerations)) + for _, item := range tolerations { + t, ok := item.(map[string]any) + if !ok { + return errors.New("invalid toleration") + } + if len(t) == 4 && (t["key"] == "node.kubernetes.io/not-ready" || t["key"] == "node.kubernetes.io/unreachable") && + t["operator"] == "Exists" && t["effect"] == "NoExecute" && t["tolerationSeconds"] == float64(300) { + continue + } + filtered = append(filtered, item) + } + spec["tolerations"] = filtered + } + for _, key := range []string{"containers", "initContainers"} { + containers, ok := spec[key].([]any) + if !ok { + return errors.New("missing container specification") + } + for _, item := range containers { + c, ok := item.(map[string]any) + if !ok { + return errors.New("invalid container") + } + removeDefault(c, "imagePullPolicy", "IfNotPresent") + removeDefault(c, "terminationMessagePath", "/dev/termination-log") + removeDefault(c, "terminationMessagePolicy", "File") + for _, name := range []string{"readinessProbe", "livenessProbe", "startupProbe"} { + if probe, ok := c[name].(map[string]any); ok { + for field, value := range map[string]any{ + "timeoutSeconds": float64(1), "periodSeconds": float64(10), + "successThreshold": float64(1), "failureThreshold": float64(3), + } { + removeDefault(probe, field, value) + } + } + } + } + } + return nil +} + +func removeDefault(m map[string]any, key string, value any) { + if reflect.DeepEqual(m[key], value) { + delete(m, key) + } +} diff --git a/scripts/verify-ksail-arc-runtime/pod_test.go b/scripts/verify-ksail-arc-runtime/pod_test.go new file mode 100644 index 0000000000..f7f81ba5eb --- /dev/null +++ b/scripts/verify-ksail-arc-runtime/pod_test.go @@ -0,0 +1,136 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestPodReadbackBindsAllRuntimeFields(t *testing.T) { + image := "ghcr.io/devantler-tech/ksail-analysis-runner@sha256:" + strings.Repeat("a", 64) + runtimeImage := "ghcr.io/devantler-tech/ksail-analysis-runner@sha256:" + strings.Repeat("b", 64) + var desired map[string]any + if err := json.Unmarshal([]byte(`{"spec":{ + "automountServiceAccountToken":false,"serviceAccountName":"no-permission", + "securityContext":{"runAsUser":1001,"runAsGroup":1001,"fsGroup":1001,"seccompProfile":{"type":"RuntimeDefault"}}, + "nodeSelector":{"analysis":"enabled"},"tolerations":[{"key":"analysis","operator":"Equal","value":"enabled","effect":"NoSchedule"}], + "volumes":[{"name":"scratch","emptyDir":{"sizeLimit":"40Gi"}}], + "restartPolicy":"Never","activeDeadlineSeconds":1200, + "containers":[{"name":"runner","image":"IMAGE","command":["/bin/sh","-ec","proof"], + "securityContext":{"privileged":false,"allowPrivilegeEscalation":false,"readOnlyRootFilesystem":true,"capabilities":{"drop":["ALL"]}}, + "resources":{"requests":{"cpu":"3","memory":"12Gi"},"limits":{"cpu":"3500m","memory":"14Gi"}}, + "volumeMounts":[{"name":"scratch","mountPath":"/home/runner"}], + "readinessProbe":{"exec":{"command":["true"]},"periodSeconds":2}}], + "initContainers":[{"name":"init","image":"IMAGE","command":["cp"],"securityContext":{"readOnlyRootFilesystem":true}}] + }}`), &desired); err != nil { + t.Fatal(err) + } + spec := desired["spec"].(map[string]any) + for _, key := range []string{"containers", "initContainers"} { + spec[key].([]any)[0].(map[string]any)["image"] = image + } + encode := func(v any) []byte { + data, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return data + } + desiredJSON := encode(desired) + actual := func() map[string]any { + var pod map[string]any + if err := json.Unmarshal(desiredJSON, &pod); err != nil { + t.Fatal(err) + } + s := pod["spec"].(map[string]any) + s["nodeName"] = "assigned-node" + s["dnsPolicy"] = "ClusterFirst" + s["schedulerName"] = "default-scheduler" + s["terminationGracePeriodSeconds"] = float64(30) + s["enableServiceLinks"] = true + s["serviceAccount"] = s["serviceAccountName"] + s["tolerations"] = append(s["tolerations"].([]any), map[string]any{ + "key": "node.kubernetes.io/not-ready", "operator": "Exists", "effect": "NoExecute", "tolerationSeconds": float64(300)}) + for _, key := range []string{"containers", "initContainers"} { + c := s[key].([]any)[0].(map[string]any) + c["imagePullPolicy"] = "IfNotPresent" + c["terminationMessagePath"] = "/dev/termination-log" + c["terminationMessagePolicy"] = "File" + } + s["containers"].([]any)[0].(map[string]any)["readinessProbe"].(map[string]any)["timeoutSeconds"] = float64(1) + pod["status"] = map[string]any{ + "containerStatuses": []any{map[string]any{"name": "runner", "ready": true, "restartCount": float64(0), "imageID": runtimeImage}}, + "initContainerStatuses": []any{map[string]any{"name": "init", "restartCount": float64(0), "imageID": runtimeImage, "state": map[string]any{"terminated": map[string]any{"exitCode": float64(0)}}}}, + } + return pod + } + t.Run("known-api-defaults-and-verified-runtime", func(t *testing.T) { + if err := verifyPod(desiredJSON, encode(actual()), runtimeImage); err != nil { + t.Fatal(err) + } + }) + t.Run("containerd-reports-requested-signed-index", func(t *testing.T) { + pod := actual() + for _, field := range []string{"containerStatuses", "initContainerStatuses"} { + pod["status"].(map[string]any)[field].([]any)[0].(map[string]any)["imageID"] = image + } + if err := verifyPod(desiredJSON, encode(pod), runtimeImage); err != nil { + t.Fatal(err) + } + }) + mutations := map[string]func(map[string]any){ + "host-network": func(s map[string]any) { s["hostNetwork"] = true }, + "host-pid": func(s map[string]any) { s["hostPID"] = true }, + "runtime-class": func(s map[string]any) { s["runtimeClassName"] = "unexpected" }, + "service-account": func(s map[string]any) { s["serviceAccountName"] = "privileged" }, + "token-mount": func(s map[string]any) { s["automountServiceAccountToken"] = true }, + "pod-security": func(s map[string]any) { s["securityContext"].(map[string]any)["runAsUser"] = float64(0) }, + "node-selector": func(s map[string]any) { s["nodeSelector"] = map[string]any{"other": "enabled"} }, + "extra-toleration": func(s map[string]any) { + s["tolerations"] = append(s["tolerations"].([]any), map[string]any{"operator": "Exists"}) + }, + "host-path": func(s map[string]any) { + s["volumes"] = append(s["volumes"].([]any), map[string]any{"name": "host", "hostPath": map[string]any{"path": "/"}}) + }, + "sidecar": func(s map[string]any) { + s["containers"] = append(s["containers"].([]any), map[string]any{"name": "sidecar", "image": "other"}) + }, + "init-image": func(s map[string]any) { s["initContainers"].([]any)[0].(map[string]any)["image"] = "other" }, + "init-security": func(s map[string]any) { + s["initContainers"].([]any)[0].(map[string]any)["securityContext"] = map[string]any{"privileged": true} + }, + "runner-image": func(s map[string]any) { s["containers"].([]any)[0].(map[string]any)["image"] = "other" }, + "runner-env": func(s map[string]any) { + s["containers"].([]any)[0].(map[string]any)["env"] = []any{map[string]any{"name": "INJECTED", "value": "unsafe"}} + }, + "runner-envfrom": func(s map[string]any) { + s["containers"].([]any)[0].(map[string]any)["envFrom"] = []any{map[string]any{"secretRef": map[string]any{"name": "unsafe"}}} + }, + "runner-mount": func(s map[string]any) { s["containers"].([]any)[0].(map[string]any)["volumeMounts"] = []any{} }, + "runner-resources": func(s map[string]any) { + s["containers"].([]any)[0].(map[string]any)["resources"].(map[string]any)["limits"] = map[string]any{"memory": "1Gi"} + }, + "runner-lifecycle": func(s map[string]any) { + s["containers"].([]any)[0].(map[string]any)["lifecycle"] = map[string]any{"postStart": map[string]any{"exec": map[string]any{"command": []any{"unsafe"}}}} + }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + pod := actual() + mutate(pod["spec"].(map[string]any)) + if verifyPod(desiredJSON, encode(pod), runtimeImage) == nil { + t.Fatal("accepted changed runtime") + } + }) + } + for _, field := range []string{"containerStatuses", "initContainerStatuses"} { + t.Run(field+"-unverified-image", func(t *testing.T) { + pod := actual() + pod["status"].(map[string]any)[field].([]any)[0].(map[string]any)["imageID"] = + "ghcr.io/devantler-tech/ksail-analysis-runner@sha256:" + strings.Repeat("c", 64) + if verifyPod(desiredJSON, encode(pod), runtimeImage) == nil { + t.Fatal("accepted unrelated runtime digest") + } + }) + } +} diff --git a/scripts/wait-for-ksail-arc-registration.sh b/scripts/wait-for-ksail-arc-registration.sh new file mode 100755 index 0000000000..e45facd265 --- /dev/null +++ b/scripts/wait-for-ksail-arc-registration.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Read-only convergence polling; the caller also imposes a 660-second hard deadline. +set -euo pipefail +[[ "$#" -eq 1 && "$1" =~ ^sha256:[0-9a-f]{64}$ ]] || exit 2 +[[ "${GITHUB_ACTIONS:-}" == true && "${GITHUB_REPOSITORY:-}" == devantler-tech/platform ]] || exit 1 +readonly revision="latest@$1" +readonly deadline=$((SECONDS + 600)) +kc() { timeout 35s kubectl --context admin@prod --request-timeout=30s "$@" 2>/dev/null; } +pending() { printf 'ARC registration pending: %s\n' "$1" >&2; } + +while ((SECONDS < deadline)); do + ready=true + for layer in infrastructure apps; do + if ! state=$(kc -n flux-system get kustomization "$layer" -o json) || + ! jq -e --arg revision "$revision" ' + .status.lastAppliedRevision == $revision and + .status.observedGeneration == .metadata.generation and + any(.status.conditions[]; .type == "Ready" and .status == "True") + ' <<<"$state" >/dev/null 2>&1; then + ready=false + pending "flux-$layer" + fi + done + # The pinned provider caches by reference name, across namespaces and kinds. + # Bind this distinct name to exactly one native configuration and Secret. + if ! state=$(kc get providerconfigs.github.m.upbound.io --all-namespaces -o json) || + ! jq -e ' + [.items[] | select(.metadata.name == "arc-runtime-platform-app")] as $configs | + ($configs | length) == 1 and + $configs[0].apiVersion == "github.m.upbound.io/v1beta1" and + $configs[0].kind == "ProviderConfig" and + $configs[0].metadata.namespace == "arc-runners" and + $configs[0].metadata.deletionTimestamp == null and + $configs[0].spec.credentials == {"source":"Secret","secretRef":{ + "namespace":"arc-runners","name":"arc-github-app","key":"provider-credentials"}} + ' <<<"$state" >/dev/null 2>&1; then + ready=false + pending provider-config + fi + for resource in clusterproviderconfigs.github.m.upbound.io providerconfigs.github.upbound.io; do + if ! state=$(kc get "$resource" -o json) || + ! jq -e '[.items[] | select(.metadata.name == "arc-runtime-platform-app")] | length == 0' \ + <<<"$state" >/dev/null 2>&1; then + ready=false + case "$resource" in + clusterproviderconfigs.github.m.upbound.io) pending provider-collision-cluster ;; + providerconfigs.github.upbound.io) pending provider-collision-legacy ;; + esac + fi + done + # The provider observation comes from the existing organization's App. + # Desired selection alone cannot establish the remote access boundary. + if ! state=$(kc -n arc-runners get runnergroups.actions.github.m.upbound.io platform -o json) || + ! jq -e ' + .metadata.generation as $generation | + .status.atProvider.id as $id | + .apiVersion == "actions.github.m.upbound.io/v1alpha1" and .kind == "RunnerGroup" and + .metadata.name == "platform" and .metadata.namespace == "arc-runners" and + (.metadata.uid | type == "string" and length > 0) and .metadata.deletionTimestamp == null and + ($generation | type == "number" and . > 0) and + ($id | type == "string" and test("^[1-9][0-9]{0,18}$")) and + .metadata.annotations["crossplane.io/external-name"] == $id and + .spec.providerConfigRef == {"name":"arc-runtime-platform-app","kind":"ProviderConfig"} and + .spec.forProvider.name == "platform" and .spec.forProvider.visibility == "selected" and + .spec.forProvider.selectedRepositoryIds == [737584922] and + .spec.forProvider.allowsPublicRepositories == true and + .spec.forProvider.restrictedToWorkflows == true and + .spec.forProvider.selectedWorkflows == ["devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main"] and + .status.atProvider.name == "platform" and .status.atProvider.visibility == "selected" and + .status.atProvider.selectedRepositoryIds == [737584922] and + .status.atProvider.allowsPublicRepositories == true and + .status.atProvider.restrictedToWorkflows == true and + .status.atProvider.selectedWorkflows == ["devantler-tech/ksail/.github/workflows/verify-ksail-arc-delivery.yaml@refs/heads/main"] and + .status.atProvider.default == false and .status.atProvider.inherited == false and + .status.atProvider.runnersUrl == ("https://api.github.com/orgs/devantler-tech/actions/runner-groups/" + $id + "/runners") and + .status.atProvider.selectedRepositoriesUrl == ("https://api.github.com/orgs/devantler-tech/actions/runner-groups/" + $id + "/repositories") and + any(.status.conditions[]; .type == "Ready" and .status == "True" and .observedGeneration == $generation) and + any(.status.conditions[]; .type == "Synced" and .status == "True" and .observedGeneration == $generation) + ' <<<"$state" >/dev/null 2>&1; then + ready=false + pending runner-group + fi + if ! state=$(kc -n arc-runners get autoscalingrunnerset.actions.github.com platform-linux -o json) || + ! jq -e ' + .status.phase == "Running" and .status.observedGeneration == .metadata.generation and + ((.metadata.annotations["runner-scale-set-id"] | tonumber) > 0) and + .spec.githubConfigUrl == "https://github.com/devantler-tech" and + .spec.runnerGroup == "platform" and .spec.runnerScaleSetName == "platform-linux" and + .metadata.annotations["actions.github.com/runner-group-name"] == "platform" and + .metadata.annotations["actions.github.com/runner-scale-set-name"] == "platform-linux" + ' <<<"$state" >/dev/null 2>&1; then + ready=false + pending runner-scale-set + fi + if "$ready"; then + printf 'ARC layers and registration converged at the deployed revision\n' + exit 0 + fi + sleep 10 +done +printf 'ARC layers or registration did not converge before the deadline\n' >&2 +exit 1