fix(ci): bound GPU capacity waits and retain cleanup receipts

.github/workflows/trainer-rank-gpu.yml

@@ -31,7 +31,13 @@ jobs:
       head_sha: ${{ steps.changes.outputs.head_sha }}
       reused: ${{ steps.reuse.outputs.reused }}
       reused_run: ${{ steps.reuse.outputs.run_id }}
+      admission_deadline: ${{ steps.admission.outputs.deadline }}
     steps:
+      - id: admission
+        name: Bound GPU client admission wait
+        # Preserve this classification's cutoff; rerun all jobs for a fresh window.
+        run: echo "deadline=$(( $(date +%s) + 1800 ))" >> "${GITHUB_OUTPUT}"
+
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
@@ -138,20 +144,46 @@ jobs:
             echo "### TrainerRank GPU validation not required" >> "${GITHUB_STEP_SUMMARY}"
           fi
 
+  api-enclosure:
+    name: Qualify CI API enclosure (CPU only)
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+      - name: Qualify detached-child retirement without Sky or credentials
+        run: python3 scripts/ci/trainer_rank_api.py probe "${RUNNER_TEMP}/api-enclosure-probe"
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: api-enclosure-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/api-enclosure-probe
+
   validate:
     name: Run on 2x H200
-    needs: classify
+    needs: [classify, api-enclosure]
     if: needs.classify.outputs.required == 'true' && needs.classify.outputs.reused != 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 45
+    concurrency:
+      group: trainer-rank-gpu-capacity-${{ inputs.context || vars.TRAINER_RANK_GPU_CONTEXT || 'cks-wb3' }}
+      cancel-in-progress: false
+      queue: max
     environment: trainer-rank-gpu-validation
     env:
+      SKYPILOT_DISABLE_LOCAL_API_SERVER: "1"
       SKY_INFRA: k8s/${{ inputs.context || vars.TRAINER_RANK_GPU_CONTEXT || 'cks-wb3' }}
+      EXPECTED_HEAD_SHA: ${{ needs.classify.outputs.head_sha }}
+      ADMISSION_DEADLINE: ${{ needs.classify.outputs.admission_deadline }}
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ needs.classify.outputs.head_sha }}
 
+      - name: Check admission deadline
+        run: python3 scripts/ci/trainer-rank-gpu.py admit "${RUNNER_TEMP}/trainer-rank-result"
+
       - name: Install launch dependencies
         run: |
           curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -221,16 +253,21 @@ jobs:
           set -euo pipefail
           sky_venv="${RUNNER_TEMP}/skypilot"
           uv venv --python 3.10 "${sky_venv}"
+          # Preserve cancelled requests before retry callbacks can reschedule them.
           uv pip install --python "${sky_venv}/bin/python" \
-            'skypilot[kubernetes,remote]==0.12.0' 'kubernetes==34.1.0'
-          sky="${sky_venv}/bin/sky"
-          cluster="trainer-rank-gpu-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          cleanup() { "${sky}" down -y "${cluster}" || true; }
-          trap cleanup EXIT
-          "${sky}" check kubernetes
+            'skypilot[kubernetes,remote] @ git+https://github.com/OpenPipe/skypilot-modal.git@924100745348cb0d880ba9e1f64b4e54cb1c0898' \
+            'kubernetes==34.1.0'
           "${sky_venv}/bin/python" scripts/ci/trainer-rank-gpu.py \
             "${RUNNER_TEMP}/trainer-rank-result"
 
+      - name: Reconcile owned GPU resources
+        if: always()
+        run: |
+          python="${RUNNER_TEMP}/skypilot/bin/python"
+          if [ ! -x "${python}" ]; then python=python3; fi
+          "${python}" scripts/ci/trainer-rank-gpu.py cleanup \
+            "${RUNNER_TEMP}/trainer-rank-result"
+
       - name: Preserve remote result and diagnostic logs
         if: always()
         uses: actions/upload-artifact@v4
@@ -241,18 +278,23 @@ jobs:
 
   gate:
     name: trainer-rank-gpu-validation
-    needs: [classify, validate]
+    needs: [classify, api-enclosure, validate]
     if: always()
     runs-on: ubuntu-latest
     steps:
       - env:
+          ENCLOSURE_RESULT: ${{ needs.api-enclosure.result }}
           CLASSIFY_RESULT: ${{ needs.classify.result }}
           REQUIRED: ${{ needs.classify.outputs.required }}
           REUSED: ${{ needs.classify.outputs.reused }}
           REUSED_RUN: ${{ needs.classify.outputs.reused_run }}
           RESULT: ${{ needs.validate.result }}
         run: |
           set -euo pipefail
+          if [ "${ENCLOSURE_RESULT}" != "success" ]; then
+            echo "CI API enclosure qualification failed." >&2
+            exit 1
+          fi
           if [ "${CLASSIFY_RESULT}" != "success" ]; then
             echo "TrainerRank GPU classification failed." >&2
             exit 1