diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml
index 230326a286..17969c9f29 100644
--- a/.github/actions/deploy-prod/action.yml
+++ b/.github/actions/deploy-prod/action.yml
@@ -459,6 +459,22 @@ runs:
shell: bash
run: ./scripts/verify-talos-schematic-readback.sh
+ - name: 🧾 Read back the kubelet settings on every node
+ # A green machine-config sync does not prove that every running kubelet
+ # took the declared settings: two autoscaler nodes ran without the disk
+ # eviction thresholds for weeks after the fix deployed, and nothing
+ # reported it (#3137). Observe-only for now: the comparison has not yet
+ # been read against production, so a finding is a warning on a green
+ # deploy. Drop KUBELET_READBACK_ENFORCE once one deploy has logged a
+ # clean PASS (tracked on #3137). The check only reads Kubernetes.
+ if: >-
+ steps.cluster_update.outcome == 'success' &&
+ steps.wait_prod_api_stability.outcome == 'success'
+ shell: bash
+ env:
+ KUBELET_READBACK_ENFORCE: "false"
+ run: ./scripts/verify-kubelet-config-readback.sh
+
- name: 📣 Report suppressed machine-config sync
# The step above is the pipeline's ONLY Talos machine-config sync, and the
# rollout gate skips it. A skipped step inside a green deploy reads as
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 787667d246..d9eaf1630a 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -957,6 +957,8 @@ jobs:
- 'scripts/verify-talos-schematic-readback.sh'
- 'scripts/tests/test-talos-factory-schematic-registration.sh'
- 'scripts/tests/test-talos-schematic-readback.sh'
+ - 'scripts/verify-kubelet-config-readback.sh'
+ - 'scripts/tests/test-kubelet-config-readback.sh'
- 'scripts/tests/test-talos-apparmor-lsm.sh'
- 'ksail.prod.yaml'
- 'ksail.yaml'
@@ -2842,6 +2844,11 @@ jobs:
shellcheck scripts/verify-talos-schematic-readback.sh scripts/tests/test-talos-schematic-readback.sh
bash scripts/tests/test-talos-schematic-readback.sh
+ - name: 🧾 Validate kubelet settings readback
+ run: |
+ shellcheck scripts/verify-kubelet-config-readback.sh scripts/tests/test-kubelet-config-readback.sh
+ bash scripts/tests/test-kubelet-config-readback.sh
+
- name: 🧱 Prove folded Talos render and invalid-patch rejection
run: |
shellcheck scripts/tests/test-talos-render-kernel-args.sh
diff --git a/scripts/tests/test-kubelet-config-readback.sh b/scripts/tests/test-kubelet-config-readback.sh
new file mode 100755
index 0000000000..9983196934
--- /dev/null
+++ b/scripts/tests/test-kubelet-config-readback.sh
@@ -0,0 +1,267 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+root_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+readback="$root_dir/scripts/verify-kubelet-config-readback.sh"
+deploy_action="$root_dir/.github/actions/deploy-prod/action.yml"
+ci_workflow="$root_dir/.github/workflows/ci.yaml"
+
+[[ -x "$readback" ]] || {
+ printf 'kubelet settings readback is missing or not executable\n' >&2
+ exit 1
+}
+
+if ! grep -Fq 'scripts/verify-kubelet-config-readback.sh' "$deploy_action" ||
+ ! grep -Fq "'scripts/tests/test-kubelet-config-readback.sh'" "$ci_workflow" ||
+ ! grep -Fq 'bash scripts/tests/test-kubelet-config-readback.sh' "$ci_workflow"; then
+ printf 'CI and deployment must execute the kubelet settings readback contract\n' >&2
+ exit 1
+fi
+
+stability_line=$(grep -nF './scripts/wait-for-prod-api-stability.sh' "$deploy_action" | cut -d: -f1)
+readback_line=$(grep -nF './scripts/verify-kubelet-config-readback.sh' "$deploy_action" | cut -d: -f1)
+if [[ -z "$stability_line" || -z "$readback_line" ]] || (( stability_line >= readback_line )); then
+ printf 'kubelet settings readback must follow cluster update and API stability\n' >&2
+ exit 1
+fi
+
+# The deploy step must stay observe-only until the switch is made on purpose,
+# and must run only after a successful update and a stable API.
+step=$(yq -o=json -I=0 '.runs.steps[] | select(.run == "./scripts/verify-kubelet-config-readback.sh")' "$deploy_action")
+if [[ "$(jq -s 'length' <<<"$step")" != 1 ]] ||
+ ! jq -e '.env.KUBELET_READBACK_ENFORCE == "false" and
+ (.if | contains("steps.cluster_update.outcome == \u0027success\u0027") and
+ contains("steps.wait_prod_api_stability.outcome == \u0027success\u0027"))' <<<"$step" >/dev/null; then
+ printf 'the deploy must run the kubelet readback once, observe-only, after a successful update\n' >&2
+ exit 1
+fi
+
+tmp_dir=$(mktemp -d)
+trap 'rm -rf "$tmp_dir"' EXIT
+
+# The mock answers the two reads the check makes and nothing else. A node's
+# reply is the file named after it; a missing file is a refused read.
+mock_kubectl="$tmp_dir/kubectl"
+cat >"$mock_kubectl" <<'MOCK'
+#!/usr/bin/env bash
+set -euo pipefail
+[[ "$1 $2 $3" == "--context admin@prod --request-timeout=15s" ]] || exit 64
+shift 3
+if [[ "$*" == "get nodes -o json" ]]; then
+ [[ -z "${MOCK_NODES_FAIL:-}" ]] || exit 1
+ if [[ -n "${MOCK_FIRST_DIR:-}" && ! -f "$MOCK_DIR/first-used" ]]; then
+ cat "$MOCK_FIRST_DIR/nodes.json"
+ else
+ cat "$MOCK_DIR/nodes.json"
+ fi
+ exit 0
+fi
+[[ "$1 $2" == "get --raw" && "$3" == /api/v1/nodes/*/proxy/configz && $# -eq 3 ]] || exit 64
+node=${3#/api/v1/nodes/}
+node=${node%/proxy/configz}
+dir=$MOCK_DIR
+if [[ -n "${MOCK_FIRST_DIR:-}" && ! -f "$MOCK_DIR/first-used" ]]; then
+ dir=$MOCK_FIRST_DIR
+ [[ "$node" != "${MOCK_LAST_NODE:-}" ]] || : >"$MOCK_DIR/first-used"
+fi
+[[ -f "$dir/$node.json" ]] || { printf 'Error from server (Forbidden)\n' >&2; exit 1; }
+cat "$dir/$node.json"
+MOCK
+chmod +x "$mock_kubectl"
+
+nodes=(prod-control-plane-2 prod-worker-1 autoscale-cx43-example)
+
+# What a healthy kubelet reports for the settings the repository declares
+# today, plus fields the check must ignore.
+healthy='{"kubeletconfig":{
+ "systemReserved":{"memory":"256Mi","cpu":"50m"},
+ "kubeReserved":{"memory":"256Mi"},
+ "evictionSoft":{"memory.available":"500Mi"},
+ "evictionSoftGracePeriod":{"memory.available":"1m30s"},
+ "evictionMinimumReclaim":{"memory.available":"200Mi"},
+ "evictionMaxPodGracePeriod":60,
+ "mergeDefaultEvictionSettings":true,
+ "evictionHard":{"memory.available":"100Mi","nodefs.available":"10%","nodefs.inodesFree":"5%","imagefs.available":"15%","imagefs.inodesFree":"5%"},
+ "imageGCHighThresholdPercent":75,
+ "imageGCLowThresholdPercent":70,
+ "maxPods":110}}'
+# The exact shape #3137 found on two autoscaler nodes.
+drifted=$(jq '.kubeletconfig.mergeDefaultEvictionSettings = false
+ | .kubeletconfig.evictionHard = {"memory.available":"100Mi"}' <<<"$healthy")
+
+new_fixture() { #
: every node healthy
+ mkdir -p "$1"
+ printf '%s\n' "${nodes[@]}" | jq -R '{metadata: {name: .}}' | jq -s '{items: .}' >"$1/nodes.json"
+ for node in "${nodes[@]}"; do printf '%s\n' "$healthy" >"$1/$node.json"; done
+}
+
+run() { # [env assignments...]: sets rc and output
+ local dir=$1
+ shift
+ rc=0
+ output=$(env KUBECTL_BIN="$mock_kubectl" MOCK_DIR="$dir" \
+ KUBELET_READBACK_ATTEMPTS=1 KUBELET_READBACK_INTERVAL_SECONDS=0 "$@" "$readback" 2>&1) || rc=$?
+}
+
+expect() { # ...
+ local name=$1 want=$2
+ shift 2
+ if [[ "$rc" -ne "$want" ]]; then
+ printf 'FAIL %s: expected exit %s, got %s:\n%s\n' "$name" "$want" "$rc" "$output" >&2
+ exit 1
+ fi
+ for fragment in "$@"; do
+ # A here-string, not a pipe: grep -q stops at the first match.
+ if ! grep -qF -- "$fragment" <<<"$output"; then
+ printf 'FAIL %s: missing "%s":\n%s\n' "$name" "$fragment" "$output" >&2
+ exit 1
+ fi
+ done
+ printf 'ok: %s\n' "$name"
+}
+
+new_fixture "$tmp_dir/healthy"
+run "$tmp_dir/healthy"
+expect 'every node matches the declared settings' 0 \
+ 'PASS: 3 node(s), 1 of them autoscaler-provisioned' 'mergeDefaultEvictionSettings'
+
+new_fixture "$tmp_dir/drift"
+printf '%s\n' "$drifted" >"$tmp_dir/drift/autoscale-cx43-example.json"
+run "$tmp_dir/drift"
+expect 'the drift #3137 found on an autoscaler node fails and names it' 1 \
+ 'autoscale-cx43-example: mergeDefaultEvictionSettings: declared true, live false' \
+ 'autoscale-cx43-example: evictionHard: live has no nodefs.available threshold' \
+ 'autoscale-cx43-example: evictionHard: live has no imagefs.available threshold'
+if grep -qF 'prod-worker-1:' <<<"$output"; then
+ printf 'FAIL: a healthy node was reported beside the drifted one:\n%s\n' "$output" >&2
+ exit 1
+fi
+
+new_fixture "$tmp_dir/value"
+jq '.kubeletconfig.evictionSoft["memory.available"] = "100Mi"' <<<"$healthy" >"$tmp_dir/value/prod-worker-1.json"
+run "$tmp_dir/value"
+expect 'a changed value inside a declared map fails' 1 \
+ 'prod-worker-1: evictionSoft: declared {"memory.available":"500Mi"}, live {"memory.available":"100Mi"}'
+
+new_fixture "$tmp_dir/absent"
+jq 'del(.kubeletconfig.imageGCHighThresholdPercent)' <<<"$healthy" >"$tmp_dir/absent/prod-worker-1.json"
+run "$tmp_dir/absent"
+expect 'a declared setting the kubelet does not report fails' 1 \
+ 'prod-worker-1: imageGCHighThresholdPercent: declared 75, live null'
+
+# The reproduction in #3137 printed nothing on a refused read. That must
+# never read as a pass.
+new_fixture "$tmp_dir/refused"
+rm "$tmp_dir/refused/autoscale-cx43-example.json"
+run "$tmp_dir/refused"
+expect 'a node whose kubelet cannot be read fails' 1 \
+ 'autoscale-cx43-example: kubelet configuration could not be read'
+
+# jq prints nothing for an empty document, so an empty reply must be refused
+# before it is compared.
+new_fixture "$tmp_dir/blank"
+: >"$tmp_dir/blank/prod-worker-1.json"
+printf ' \n' >"$tmp_dir/blank/prod-control-plane-2.json"
+run "$tmp_dir/blank"
+expect 'an empty or blank reply fails' 1 \
+ 'prod-worker-1: kubelet configuration reply is empty' \
+ 'prod-control-plane-2: kubelet configuration reply is empty'
+
+new_fixture "$tmp_dir/garbage"
+printf 'not json\n' >"$tmp_dir/garbage/prod-worker-1.json"
+run "$tmp_dir/garbage"
+expect 'a reply that is not JSON fails' 1 'prod-worker-1: kubelet configuration reply is not valid JSON'
+
+new_fixture "$tmp_dir/shapeless"
+printf '{}\n' >"$tmp_dir/shapeless/prod-worker-1.json"
+run "$tmp_dir/shapeless"
+expect 'a reply without a kubelet configuration fails' 1 'prod-worker-1: no kubeletconfig in the reply'
+
+mkdir "$tmp_dir/empty"
+printf '{"items": []}\n' >"$tmp_dir/empty/nodes.json"
+run "$tmp_dir/empty"
+expect 'a cluster reporting no nodes fails' 1 'the cluster reported no nodes'
+
+mkdir "$tmp_dir/unreadable"
+printf 'not json\n' >"$tmp_dir/unreadable/nodes.json"
+run "$tmp_dir/unreadable"
+expect 'an unreadable node list fails' 1 'Kubernetes nodes could not be read from context admin@prod'
+
+run "$tmp_dir/healthy" MOCK_NODES_FAIL=1
+expect 'a failed node list read fails' 1 'Kubernetes nodes could not be read from context admin@prod'
+
+mkdir "$tmp_dir/hostile"
+printf '{"items": [{"metadata": {"name": "a/../../b"}}]}\n' >"$tmp_dir/hostile/nodes.json"
+run "$tmp_dir/hostile"
+expect 'a node name that is not a node name is never queried' 1 'unexpected node name; refusing to query it'
+
+# A replacement node that is not answering yet converges within the window.
+new_fixture "$tmp_dir/late-first"
+rm "$tmp_dir/late-first/autoscale-cx43-example.json"
+new_fixture "$tmp_dir/late"
+run "$tmp_dir/late" MOCK_FIRST_DIR="$tmp_dir/late-first" MOCK_LAST_NODE=autoscale-cx43-example \
+ KUBELET_READBACK_ATTEMPTS=3
+expect 'a node that answers on a later attempt passes' 0 'PASS: 3 node(s)'
+[[ -f "$tmp_dir/late/first-used" ]] || {
+ printf 'FAIL: the late-node case never served its first, incomplete reading\n' >&2
+ exit 1
+}
+
+# Drift cannot improve by waiting, so it is reported on the first reading.
+run "$tmp_dir/drift" KUBELET_READBACK_ATTEMPTS=3
+expect 'drift is reported without waiting out the retry window' 1 'did not pass after 1 attempt(s)'
+run "$tmp_dir/refused" KUBELET_READBACK_ATTEMPTS=3
+expect 'an unread node is retried until the last attempt' 1 'did not pass after 3 attempt(s)'
+
+# Observe-only: the finding is reported as a warning and the deploy stays green.
+run "$tmp_dir/drift" KUBELET_READBACK_ENFORCE=false
+expect 'observe-only reports drift as a warning and exits 0' 0 \
+ '::warning title=Kubelet settings readback (observe-only)::' \
+ 'autoscale-cx43-example: mergeDefaultEvictionSettings: declared true, live false'
+run "$tmp_dir/healthy" KUBELET_READBACK_ENFORCE=false
+expect 'observe-only still passes a healthy cluster without a warning' 0 'PASS: 3 node(s)'
+if grep -qF '::warning' <<<"$output"; then
+ printf 'FAIL: a healthy observe-only run printed a warning:\n%s\n' "$output" >&2
+ exit 1
+fi
+run "$tmp_dir/healthy" KUBELET_READBACK_ENFORCE=maybe
+expect 'an unknown enforce value is refused' 2 'invalid KUBELET_READBACK_ENFORCE'
+
+# Nothing declared means nothing was compared; that is not a pass.
+mkdir -p "$tmp_dir/talos-none/cluster"
+printf 'machine:\n network: {}\n' >"$tmp_dir/talos-none/cluster/other.yaml"
+run "$tmp_dir/healthy" TALOS_CONFIG_DIR="$tmp_dir/talos-none"
+expect 'no declared kubelet settings is refused' 2 'nothing to compare'
+
+mkdir -p "$tmp_dir/talos-missing"
+run "$tmp_dir/healthy" TALOS_CONFIG_DIR="$tmp_dir/talos-missing"
+expect 'a missing patch directory is refused' 2 'no cluster-wide Talos patches found'
+
+mkdir -p "$tmp_dir/talos-role/cluster" "$tmp_dir/talos-role/workers"
+cp "$root_dir/talos/cluster/evict-pods-before-oom.yaml" "$tmp_dir/talos-role/cluster/"
+printf 'machine:\n kubelet:\n extraConfig:\n maxPods: 200\n' >"$tmp_dir/talos-role/workers/more-pods.yaml"
+run "$tmp_dir/healthy" TALOS_CONFIG_DIR="$tmp_dir/talos-role"
+expect 'a per-role kubelet setting is refused, not half-compared' 2 'are not modelled by this check'
+
+# Observe-only must not fail the deploy on a refusal either: a red step skips
+# the steps after it.
+run "$tmp_dir/healthy" TALOS_CONFIG_DIR="$tmp_dir/talos-role" KUBELET_READBACK_ENFORCE=false
+expect 'observe-only reports a refusal as a warning and exits 0' 0 \
+ '::warning title=Kubelet settings readback (observe-only)::' 'are not modelled by this check'
+run "$tmp_dir/healthy" TALOS_CONFIG_DIR="$tmp_dir/talos-missing" KUBELET_READBACK_ENFORCE=false
+expect 'observe-only reports a missing patch directory as a warning' 0 '::warning'
+
+# A role file with several documents and no kubelet settings is not a refusal.
+mkdir -p "$tmp_dir/talos-multi/cluster" "$tmp_dir/talos-multi/workers"
+cp "$root_dir"/talos/cluster/*.yaml "$tmp_dir/talos-multi/cluster/"
+printf 'machine:\n network: {}\n---\nmachine:\n sysctls: {}\n' >"$tmp_dir/talos-multi/workers/two-documents.yaml"
+run "$tmp_dir/healthy" TALOS_CONFIG_DIR="$tmp_dir/talos-multi"
+expect 'a multi-document role file without kubelet settings passes' 0 'PASS: 3 node(s)'
+printf 'machine:\n network: {}\n---\nmachine:\n kubelet:\n extraConfig:\n maxPods: 200\n' >"$tmp_dir/talos-multi/workers/two-documents.yaml"
+run "$tmp_dir/healthy" TALOS_CONFIG_DIR="$tmp_dir/talos-multi"
+expect 'a kubelet setting in a later document of a role file is refused' 2 'are not modelled by this check'
+printf 'machine: [unclosed\n' >"$tmp_dir/talos-multi/workers/two-documents.yaml"
+run "$tmp_dir/healthy" TALOS_CONFIG_DIR="$tmp_dir/talos-multi"
+expect 'a role file that cannot be parsed is refused, not read as empty' 2 'to rule out per-role kubelet settings'
+
+printf 'All kubelet settings readback cases passed.\n'
diff --git a/scripts/verify-kubelet-config-readback.sh b/scripts/verify-kubelet-config-readback.sh
new file mode 100755
index 0000000000..cc4460dbfa
--- /dev/null
+++ b/scripts/verify-kubelet-config-readback.sh
@@ -0,0 +1,176 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# A green machine-config sync does not prove that every running kubelet took
+# the settings the repository declares. #3137: two autoscaler nodes kept
+# running without the disk eviction thresholds for weeks after the fix was
+# deployed, and nothing reported it. This reads each node's live kubelet
+# configuration and requires every declared setting to be present in it.
+#
+# Declared settings are the cluster-wide machine.kubelet.extraConfig patches.
+# The comparison is literal and one-directional: a declared key must exist
+# live with exactly the declared value, and anything else the kubelet reports
+# is ignored. Every setting declared today is a number, a boolean or a string
+# the kubelet reports back unchanged. One it rewrites (a duration such as 2m,
+# reported as 2m0s) will fail here on its first deploy, visibly, and has to be
+# declared in the kubelet's own spelling.
+#
+# A node that cannot be read is a failure, never a pass: the reproduction in
+# #3137 printed nothing when the read was refused, which looked clean.
+#
+# Exit codes: 0 every node matches; 1 drift or an unreadable node; 2 nothing
+# to compare against, or a declaration this check does not model.
+root_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+talos_dir=${TALOS_CONFIG_DIR:-$root_dir/talos}
+kubectl_bin=${KUBECTL_BIN:-kubectl}
+# Deployments restore a kubeconfig that contains admin@prod, but its current
+# context is not a trusted cluster selector.
+kube_context=${KUBE_CONTEXT:-admin@prod}
+max_attempts=${KUBELET_READBACK_ATTEMPTS:-30}
+interval_seconds=${KUBELET_READBACK_INTERVAL_SECONDS:-10}
+max_seconds=${KUBELET_READBACK_MAX_SECONDS:-300}
+# false reports a failure as a warning and exits 0. The production deploy
+# starts that way until one clean production reading is on record (#3137).
+enforce=${KUBELET_READBACK_ENFORCE:-true}
+
+[[ "$enforce" =~ ^(true|false)$ ]] || {
+ printf 'invalid KUBELET_READBACK_ENFORCE: expected true or false\n' >&2
+ exit 2
+}
+
+# Every way this check can end without a pass goes through here, so that
+# observe-only really cannot fail a deploy: a red step would skip the steps
+# after it, and those reassert production credentials.
+give_up() { #
+ printf '%s\n' "$2" >&2
+ if [[ "$enforce" == false ]]; then
+ # One annotation, so the finding is visible on a deploy that stays green.
+ printf '::warning title=Kubelet settings readback (observe-only)::%s\n' \
+ "$(printf '%s' "$2" | tr '\n' ';')"
+ exit 0
+ fi
+ exit "$1"
+}
+
+[[ "$max_attempts" =~ ^[1-9][0-9]*$ &&
+ "$interval_seconds" =~ ^(0|[1-9][0-9]*)$ &&
+ "$max_seconds" =~ ^[1-9][0-9]*$ ]] ||
+ give_up 2 'invalid kubelet readback attempt, interval, or deadline setting'
+
+shopt -s nullglob
+cluster_patches=("$talos_dir"/cluster/*.yaml)
+role_patches=("$talos_dir"/control-planes/*.yaml "$talos_dir"/workers/*.yaml)
+shopt -u nullglob
+
+((${#cluster_patches[@]} > 0)) ||
+ give_up 2 "no cluster-wide Talos patches found under $talos_dir/cluster"
+
+# A per-role kubelet setting would make "every node" the wrong expectation.
+# Refuse it instead of comparing those nodes against half their declaration.
+# A file may hold several documents, and a read that fails is not a zero.
+for patch in "${role_patches[@]}"; do
+ role_settings=$(yq eval -o=json -I=0 '.machine.kubelet.extraConfig // {}' "$patch" 2>/dev/null |
+ jq -s 'map(length) | add // 0' 2>/dev/null) ||
+ give_up 2 "could not read $patch to rule out per-role kubelet settings"
+ [[ "$role_settings" == 0 ]] ||
+ give_up 2 "per-role kubelet settings in $patch are not modelled by this check"
+done
+
+# shellcheck disable=SC2016 # yq program, not shell
+declared=$(yq eval-all -o=json -I=0 \
+ '. as $doc ireduce ({}; . * ($doc.machine.kubelet.extraConfig // {}))' \
+ "${cluster_patches[@]}" 2>/dev/null) ||
+ give_up 2 "could not read the cluster-wide Talos patches under $talos_dir/cluster"
+jq -e 'type == "object" and length > 0' <<<"$declared" >/dev/null 2>&1 ||
+ give_up 2 "no declared kubelet settings found under $talos_dir/cluster; nothing to compare"
+
+# shellcheck disable=SC2016 # jq program, not shell
+compare='
+ def covers($want):
+ . as $have
+ | if ($want | type) == "object" then
+ ($have | type) == "object" and
+ all($want | to_entries[]; . as $e | ($have | has($e.key)) and ($have[$e.key] | covers($e.value)))
+ else $have == $want end;
+ (.kubeletconfig // null) as $live
+ | if ($live | type) != "object" then ["no kubeletconfig in the reply"]
+ else
+ [ $declared | to_entries[] | . as $e
+ | select(($live | has($e.key) and (.[$e.key] | covers($e.value))) | not)
+ | "\($e.key): declared \($e.value | tojson), live \($live[$e.key] | tojson)" ]
+ # Without these three the kubelet never evicts on disk pressure, which
+ # is the condition #3137 found. They are kubelet defaults, so they are
+ # only expected when the declaration asks for the defaults to be kept.
+ + ( if $declared.mergeDefaultEvictionSettings == true then
+ [ ("nodefs.available", "nodefs.inodesFree", "imagefs.available") as $signal
+ | select((($live.evictionHard // {}) | has($signal)) | not)
+ | "evictionHard: live has no \($signal) threshold" ]
+ else [] end )
+ end
+ | .[]'
+
+report=''
+unread=false
+node_count=0
+autoscaler_count=0
+deadline_epoch=$(( $(date +%s) + max_seconds ))
+for ((attempt = 1; attempt <= max_attempts; attempt++)); do
+ report=''
+ unread=false
+ node_count=0
+ autoscaler_count=0
+ # A node the autoscaler has just registered may not answer yet. Never accept
+ # an unread node, but allow a normal replacement to converge within a
+ # finite window.
+ if names=$("$kubectl_bin" --context "$kube_context" --request-timeout=15s \
+ get nodes -o json 2>/dev/null | jq -r '.items | if type == "array" then .[].metadata.name else error("no items") end'); then
+ while IFS= read -r node; do
+ [[ -n "$node" ]] || continue
+ node_count=$((node_count + 1))
+ if [[ ! "$node" =~ ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ ]]; then
+ report+="unexpected node name; refusing to query it"$'\n'
+ continue
+ fi
+ [[ "$node" == prod-control-plane-* || "$node" == prod-worker-* ]] ||
+ autoscaler_count=$((autoscaler_count + 1))
+ if ! configz=$("$kubectl_bin" --context "$kube_context" --request-timeout=15s \
+ get --raw "/api/v1/nodes/$node/proxy/configz" 2>/dev/null); then
+ report+="$node: kubelet configuration could not be read"$'\n'
+ unread=true
+ continue
+ fi
+ # jq prints nothing for an empty reply, which would read as no drift.
+ if [[ -z "${configz//[[:space:]]/}" ]]; then
+ report+="$node: kubelet configuration reply is empty"$'\n'
+ unread=true
+ continue
+ fi
+ if ! drift=$(jq -r --argjson declared "$declared" "$compare" <<<"$configz" 2>/dev/null); then
+ report+="$node: kubelet configuration reply is not valid JSON"$'\n'
+ continue
+ fi
+ while IFS= read -r line; do
+ [[ -z "$line" ]] || report+="$node: $line"$'\n'
+ done <<<"$drift"
+ done <<<"$names"
+ ((node_count > 0)) || report='the cluster reported no nodes'$'\n'
+ else
+ report="Kubernetes nodes could not be read from context $kube_context"$'\n'
+ unread=true
+ fi
+
+ if [[ -z "$report" ]]; then
+ printf 'PASS: %d node(s), %d of them autoscaler-provisioned, run every declared kubelet setting (%s)\n' \
+ "$node_count" "$autoscaler_count" "$(jq -r 'keys | join(", ")' <<<"$declared")"
+ exit 0
+ fi
+
+ # Only a read that did not complete can improve by waiting. Drift is a
+ # finding, and waiting out the window would only delay reporting it.
+ if [[ "$unread" == false ]] || ((attempt == max_attempts || $(date +%s) >= deadline_epoch)); then
+ break
+ fi
+ sleep "$interval_seconds"
+done
+
+give_up 1 "Kubelet settings readback did not pass after $attempt attempt(s):"$'\n'"${report%$'\n'}"