diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index 62e883e05d..b35184e780 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -473,16 +473,14 @@ runs: # A green machine-config sync does not prove that every running kubelet # took the declared settings: two autoscaler nodes ran without the disk # eviction thresholds for weeks after the fix deployed, and nothing - # reported it (#3137). Observe-only for now: the comparison has not yet - # been read against production, so a finding is a warning on a green - # deploy. Drop KUBELET_READBACK_ENFORCE once one deploy has logged a - # clean PASS (tracked on #3137). The check only reads Kubernetes. + # reported it (#3137). A finding fails the deploy: the comparison passed on + # every production deploy for a day before it was made blocking. The + # steps after it that reassert credentials run on a failure too. The + # check only reads Kubernetes. if: >- steps.cluster_update.outcome == 'success' && steps.wait_prod_api_stability.outcome == 'success' shell: bash - env: - KUBELET_READBACK_ENFORCE: "false" run: ./scripts/verify-kubelet-config-readback.sh - name: 📣 Report suppressed machine-config sync diff --git a/scripts/tests/test-kubelet-config-readback.sh b/scripts/tests/test-kubelet-config-readback.sh index 9b89c1922a..bd091dbf68 100755 --- a/scripts/tests/test-kubelet-config-readback.sh +++ b/scripts/tests/test-kubelet-config-readback.sh @@ -25,14 +25,15 @@ if [[ -z "$stability_line" || -z "$readback_line" ]] || (( stability_line >= rea exit 1 fi -# The deploy step must stay observe-only until the switch is made on purpose, -# and must run only after a successful update and a stable API. +# The deploy step must fail the deploy on a finding, so it may not set the +# observe-only switch, and must run only after a successful update and a +# stable API. step=$(yq -o=json -I=0 '.runs.steps[] | select(.run == "./scripts/verify-kubelet-config-readback.sh")' "$deploy_action") if [[ "$(jq -s 'length' <<<"$step")" != 1 ]] || - ! jq -e '.env.KUBELET_READBACK_ENFORCE == "false" and + ! jq -e '(.env // {} | has("KUBELET_READBACK_ENFORCE") | not) and (.if | contains("steps.cluster_update.outcome == \u0027success\u0027") and contains("steps.wait_prod_api_stability.outcome == \u0027success\u0027"))' <<<"$step" >/dev/null; then - printf 'the deploy must run the kubelet readback once, observe-only, after a successful update\n' >&2 + printf 'the deploy must run the kubelet readback once, blocking, after a successful update\n' >&2 exit 1 fi diff --git a/scripts/verify-kubelet-config-readback.sh b/scripts/verify-kubelet-config-readback.sh index ac8979012a..5190eb9b9c 100755 --- a/scripts/verify-kubelet-config-readback.sh +++ b/scripts/verify-kubelet-config-readback.sh @@ -29,8 +29,8 @@ kube_context=${KUBE_CONTEXT:-admin@prod} max_attempts=${KUBELET_READBACK_ATTEMPTS:-30} interval_seconds=${KUBELET_READBACK_INTERVAL_SECONDS:-10} max_seconds=${KUBELET_READBACK_MAX_SECONDS:-300} -# false reports a failure as a warning and exits 0. The production deploy -# starts that way until one clean production reading is on record (#3137). +# false reports a failure as a warning and exits 0, for reading a cluster +# whose result is not yet known. The production deploy does not set it. enforce=${KUBELET_READBACK_ENFORCE:-true} [[ "$enforce" =~ ^(true|false)$ ]] || { @@ -39,8 +39,7 @@ enforce=${KUBELET_READBACK_ENFORCE:-true} } # Every way this check can end without a pass goes through here, so that -# observe-only really cannot fail a deploy: a red step would skip the steps -# after it, and those reassert production credentials. +# observe-only really cannot fail a deploy. give_up() { # printf '%s\n' "$2" >&2 if [[ "$enforce" == false ]]; then