fix(deploy): fail the deploy when a kubelet runs without a declared setting

.github/actions/deploy-prod/action.yml

@@ -473,16 +473,14 @@ runs:
       # A green machine-config sync does not prove that every running kubelet
       # took the declared settings: two autoscaler nodes ran without the disk
       # eviction thresholds for weeks after the fix deployed, and nothing
-      # reported it (#3137). Observe-only for now: the comparison has not yet
-      # been read against production, so a finding is a warning on a green
-      # deploy. Drop KUBELET_READBACK_ENFORCE once one deploy has logged a
-      # clean PASS (tracked on #3137). The check only reads Kubernetes.
+      # reported it (#3137). A finding fails the deploy: the comparison passed on
+      # every production deploy for a day before it was made blocking. The
+      # steps after it that reassert credentials run on a failure too. The
+      # check only reads Kubernetes.
       if: >-
         steps.cluster_update.outcome == 'success' &&
         steps.wait_prod_api_stability.outcome == 'success'
       shell: bash
-      env:
-        KUBELET_READBACK_ENFORCE: "false"
       run: ./scripts/verify-kubelet-config-readback.sh
 
     - name: 📣 Report suppressed machine-config sync

scripts/tests/test-kubelet-config-readback.sh

@@ -25,14 +25,15 @@ if [[ -z "$stability_line" || -z "$readback_line" ]] || (( stability_line >= rea
   exit 1
 fi
 
-# The deploy step must stay observe-only until the switch is made on purpose,
-# and must run only after a successful update and a stable API.
+# The deploy step must fail the deploy on a finding, so it may not set the
+# observe-only switch, and must run only after a successful update and a
+# stable API.
 step=$(yq -o=json -I=0 '.runs.steps[] | select(.run == "./scripts/verify-kubelet-config-readback.sh")' "$deploy_action")
 if [[ "$(jq -s 'length' <<<"$step")" != 1 ]] ||
-  ! jq -e '.env.KUBELET_READBACK_ENFORCE == "false" and
+  ! jq -e '(.env // {} | has("KUBELET_READBACK_ENFORCE") | not) and
     (.if | contains("steps.cluster_update.outcome == \u0027success\u0027") and
            contains("steps.wait_prod_api_stability.outcome == \u0027success\u0027"))' <<<"$step" >/dev/null; then
-  printf 'the deploy must run the kubelet readback once, observe-only, after a successful update\n' >&2
+  printf 'the deploy must run the kubelet readback once, blocking, after a successful update\n' >&2
   exit 1
 fi
 

scripts/verify-kubelet-config-readback.sh

@@ -29,8 +29,8 @@ kube_context=${KUBE_CONTEXT:-admin@prod}
 max_attempts=${KUBELET_READBACK_ATTEMPTS:-30}
 interval_seconds=${KUBELET_READBACK_INTERVAL_SECONDS:-10}
 max_seconds=${KUBELET_READBACK_MAX_SECONDS:-300}
-# false reports a failure as a warning and exits 0. The production deploy
-# starts that way until one clean production reading is on record (#3137).
+# false reports a failure as a warning and exits 0, for reading a cluster
+# whose result is not yet known. The production deploy does not set it.
 enforce=${KUBELET_READBACK_ENFORCE:-true}
 
 [[ "$enforce" =~ ^(true|false)$ ]] || {
@@ -39,8 +39,7 @@ enforce=${KUBELET_READBACK_ENFORCE:-true}
 }
 
 # Every way this check can end without a pass goes through here, so that
-# observe-only really cannot fail a deploy: a red step would skip the steps
-# after it, and those reassert production credentials.
+# observe-only really cannot fail a deploy.
 give_up() { # <exit code> <message>
   printf '%s\n' "$2" >&2
   if [[ "$enforce" == false ]]; then