From 94615f55da4d593406d3924f27f1cb7600cd96b7 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 09:49:17 +0530 Subject: [PATCH 1/6] FIX: Make SQL Server CI setup retry-safe Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 551 ++++++----------------- eng/scripts/setup_sql_container.py | 513 +++++++++++++++++++++ tests/test_sql_container_pipeline.py | 171 +++++++ tests/test_sql_container_setup.py | 482 ++++++++++++++++++++ 4 files changed, 1303 insertions(+), 414 deletions(-) create mode 100644 eng/scripts/setup_sql_container.py create mode 100644 tests/test_sql_container_pipeline.py create mode 100644 tests/test_sql_container_setup.py diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 66db771d..373f8dd0 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -470,6 +470,9 @@ jobs: vmImage: 'macos-latest' variables: + # Apply to setup, benchmark consumers and final cleanup without changing + # the Docker client's global context. + DOCKER_CONTEXT: colima # pip's on-disk download/wheel cache. Persisted across runs by the Cache@2 # task below so `pip install -r requirements.txt` reuses wheels instead of # re-downloading them on every run. Exposed to script steps as the @@ -517,70 +520,39 @@ jobs: - script: | set -e - # Overlap the ENTIRE container-side setup (Colima VM boot -> image pull -> - # container start -> SQL readiness) with the CPU-bound Python setup (pip - # install + pybind build). The two chains are independent until pytest, so - # we run the container chain in the background and the build in the - # foreground, then wait for the container before continuing. - # - # On the Intel macOS-15 hosted runner the linux/amd64 SQL Server image runs - # natively (no emulation); the dominant container costs are the Colima VM - # boot (~2-4 min) and the ~2 GB image pull, both of which are hidden behind - # the build here. - # - # NOTE: this deliberately merges what used to be separate steps (start - # Colima, pull+start SQL, install Python deps, build pybind). Splitting them - # back out restores the previous per-step timing breakdown. - setup_sql() { - echo "[sql] Starting Colima VM..." - colima start --cpu 4 --memory 8 --disk 50 - docker context use colima >/dev/null || true - docker version - - echo "[sql] Pulling SQL Server image..." - docker pull "$(sqlServerImage)" - - echo "[sql] Starting SQL Server container..." - docker run \ - --name sqlserver \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="${DB_PASSWORD}" \ - -p 1433:1433 \ - -d "$(sqlServerImage)" - - echo "[sql] Waiting for SQL Server to accept connections..." - for i in {1..30}; do - if docker exec sqlserver \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost -U SA -P "$DB_PASSWORD" \ - -C -Q "SELECT 1"; then - echo "[sql] SQL Server is ready." - return 0 + # Only SQL lifecycle is retried. Colima starts once, overlapped with the + # unchanged dependency/native build chain. The helper bounds SQL work + # and redacts output before it reaches this job-private log. + SQL_LOG="$(Agent.TempDirectory)/sql-setup-$(System.JobId).log" + SQL_HELPER="$(Build.SourcesDirectory)/eng/scripts/setup_sql_container.py" + SQL_OWNER="$(Build.BuildId).$(System.JobId)" + SQL_PID= + finish_setup() { + status=$? + trap - EXIT INT TERM + if [ -n "$SQL_PID" ]; then + if kill -0 "$SQL_PID" 2>/dev/null; then + kill -TERM "$SQL_PID" || echo "[sql] Background setup already stopped" fi - sleep 2 - done - - # One last probe: the loop sleeps after its final attempt, so SQL can - # become reachable in that window and would otherwise be missed. - if docker exec sqlserver \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost -U SA -P "$DB_PASSWORD" \ - -C -Q "SELECT 1"; then - echo "[sql] SQL Server is ready." - return 0 + wait "$SQL_PID" || echo "[sql] Background setup stopped or failed" fi - - # sqlcmd only reports that the client could not connect. When the - # container itself exited (bad password policy, EULA, OOM, port clash) - # the reason is in the container's own log, so surface it here. - echo "[sql] SQL Server did not become ready in time." >&2 - echo "[sql] ---- docker logs sqlserver (last 200 lines) ----" >&2 - docker logs --tail 200 sqlserver >&2 || true - return 1 + if [ -f "$SQL_LOG" ]; then + cat "$SQL_LOG" + rm -f "$SQL_LOG" + fi + if [ "$status" -ne 0 ]; then + python3 "$SQL_HELPER" --name sqlserver --owner "$SQL_OWNER" --colima --cleanup \ + || echo "[sql] Cleanup failed; final job cleanup will also run" + fi + exit "$status" } + trap finish_setup EXIT + trap 'exit 130' INT + trap 'exit 143' TERM echo "Starting container setup (Colima + SQL Server) in the background..." - setup_sql > /tmp/sql_setup.log 2>&1 & + python3 "$SQL_HELPER" --name sqlserver --owner "$SQL_OWNER" \ + --image "$(sqlServerImage)" --colima > "$SQL_LOG" 2>&1 & SQL_PID=$! echo "Installing Python dependencies (overlapped with container setup)..." @@ -591,12 +563,12 @@ jobs: ( cd mssql_python/pybind && ./build.sh ) echo "Waiting for container setup (Colima + SQL Server) to finish..." - if ! wait "$SQL_PID"; then - echo "Container setup failed:" - cat /tmp/sql_setup.log - exit 1 + SQL_STATUS=0 + wait "$SQL_PID" || SQL_STATUS=$? + SQL_PID= + if [ "$SQL_STATUS" -ne 0 ]; then + exit "$SQL_STATUS" fi - cat /tmp/sql_setup.log displayName: 'Build + start SQL Server (Colima boot & SQL setup overlapped with build)' env: DB_PASSWORD: $(DB_PASSWORD) @@ -729,6 +701,15 @@ jobs: condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) continueOnError: true + - script: | + python3 eng/scripts/setup_sql_container.py --cleanup --colima \ + --name sqlserver --owner "$(Build.BuildId).$(System.JobId)" + displayName: 'Clean up macOS SQL container' + condition: always() + timeoutInMinutes: 3 + env: + DB_PASSWORD: $(DB_PASSWORD) + - job: PytestOnLinux displayName: 'Linux x86_64' pool: @@ -775,56 +756,12 @@ jobs: displayName: 'Create $(distroName) container' - script: | - # Start SQL Server container - docker run -d --name sqlserver-$(distroName) \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ - -p 1433:1433 \ - $(sqlServerImage) - - # Wait for SQL Server to be ready - echo "Waiting for SQL Server to start..." - sql_ready=false - for i in {1..60}; do - if docker exec sqlserver-$(distroName) \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - sql_ready=true - break - fi - echo "Waiting... ($i/60)" - sleep 2 - done - if [ "$sql_ready" != true ]; then - # the loop sleeps after its final attempt, so probe once more - if docker exec sqlserver-$(distroName) \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - else - echo "SQL Server did not become ready after 60 attempts" - echo "---- docker logs sqlserver-$(distroName) (last 200 lines) ----" - docker logs --tail 200 sqlserver-$(distroName) || true - exit 1 - fi - fi - - # Create test database - docker exec sqlserver-$(distroName) \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "CREATE DATABASE TestDB" + python3 eng/scripts/setup_sql_container.py \ + --name "sqlserver-$(distroName)" --owner "$(Build.BuildId).$(System.JobId)" \ + --image "$(sqlServerImage)" --database TestDB displayName: 'Start SQL Server container for $(distroName)' - condition: eq(variables['useAzureSQL'], 'false') + condition: and(succeeded(), eq(variables['useAzureSQL'], 'false')) + timeoutInMinutes: 22 env: DB_PASSWORD: $(DB_PASSWORD) @@ -1065,13 +1002,17 @@ jobs: condition: always() - script: | - # Clean up containers + python3 eng/scripts/setup_sql_container.py --cleanup \ + --name "sqlserver-$(distroName)" --owner "$(Build.BuildId).$(System.JobId)" + displayName: 'Clean up sqlserver-$(distroName) SQL container' + condition: and(always(), eq(variables['useAzureSQL'], 'false')) + timeoutInMinutes: 3 + env: + DB_PASSWORD: $(DB_PASSWORD) + + - script: | docker stop test-container-$(distroName) || true docker rm test-container-$(distroName) || true - if [ "$(useAzureSQL)" = "false" ]; then - docker stop sqlserver-$(distroName) || true - docker rm sqlserver-$(distroName) || true - fi displayName: 'Clean up $(distroName) containers' condition: always() @@ -1121,56 +1062,11 @@ jobs: displayName: 'Create $(distroName) ARM64 container' - script: | - # Start SQL Server container (x86_64 - SQL Server doesn't support ARM64) - docker run -d --name sqlserver-$(distroName)-$(archName) \ - --platform linux/amd64 \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ - -p 1433:1433 \ - mcr.microsoft.com/mssql/server:2022-latest - - # Wait for SQL Server to be ready - echo "Waiting for SQL Server to start..." - sql_ready=false - for i in {1..60}; do - if docker exec sqlserver-$(distroName)-$(archName) \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - sql_ready=true - break - fi - echo "Waiting... ($i/60)" - sleep 2 - done - if [ "$sql_ready" != true ]; then - # the loop sleeps after its final attempt, so probe once more - if docker exec sqlserver-$(distroName)-$(archName) \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - else - echo "SQL Server did not become ready after 60 attempts" - echo "---- docker logs sqlserver-$(distroName)-$(archName) (last 200 lines) ----" - docker logs --tail 200 sqlserver-$(distroName)-$(archName) || true - exit 1 - fi - fi - - # Create test database - docker exec sqlserver-$(distroName)-$(archName) \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "CREATE DATABASE TestDB" + python3 eng/scripts/setup_sql_container.py \ + --name "sqlserver-$(distroName)-$(archName)" --owner "$(Build.BuildId).$(System.JobId)" \ + --image "mcr.microsoft.com/mssql/server:2022-latest" --database TestDB displayName: 'Start SQL Server container for $(distroName) ARM64' + timeoutInMinutes: 22 env: DB_PASSWORD: $(DB_PASSWORD) @@ -1339,11 +1235,17 @@ jobs: condition: always() - script: | - # Clean up containers + python3 eng/scripts/setup_sql_container.py --cleanup \ + --name "sqlserver-$(distroName)-$(archName)" --owner "$(Build.BuildId).$(System.JobId)" + displayName: 'Clean up sqlserver-$(distroName)-$(archName) SQL container' + condition: always() + timeoutInMinutes: 3 + env: + DB_PASSWORD: $(DB_PASSWORD) + + - script: | docker stop test-container-$(distroName)-$(archName) || true docker rm test-container-$(distroName)-$(archName) || true - docker stop sqlserver-$(distroName)-$(archName) || true - docker rm sqlserver-$(distroName)-$(archName) || true displayName: 'Clean up $(distroName) ARM64 containers' condition: always() @@ -1370,55 +1272,11 @@ jobs: displayName: 'Create RHEL 9 container' - script: | - # Start SQL Server container - docker run -d --name sqlserver-rhel9 \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ - -p 1433:1433 \ - mcr.microsoft.com/mssql/server:2022-latest - - # Wait for SQL Server to be ready - echo "Waiting for SQL Server to start..." - sql_ready=false - for i in {1..60}; do - if docker exec sqlserver-rhel9 \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - sql_ready=true - break - fi - echo "Waiting... ($i/60)" - sleep 2 - done - if [ "$sql_ready" != true ]; then - # the loop sleeps after its final attempt, so probe once more - if docker exec sqlserver-rhel9 \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - else - echo "SQL Server did not become ready after 60 attempts" - echo "---- docker logs sqlserver-rhel9 (last 200 lines) ----" - docker logs --tail 200 sqlserver-rhel9 || true - exit 1 - fi - fi - - # Create test database - docker exec sqlserver-rhel9 \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "CREATE DATABASE TestDB" + python3 eng/scripts/setup_sql_container.py \ + --name "sqlserver-rhel9" --owner "$(Build.BuildId).$(System.JobId)" \ + --image "mcr.microsoft.com/mssql/server:2022-latest" --database TestDB displayName: 'Start SQL Server container for RHEL 9' + timeoutInMinutes: 22 env: DB_PASSWORD: $(DB_PASSWORD) @@ -1575,11 +1433,17 @@ jobs: condition: always() - script: | - # Clean up containers + python3 eng/scripts/setup_sql_container.py --cleanup \ + --name "sqlserver-rhel9" --owner "$(Build.BuildId).$(System.JobId)" + displayName: 'Clean up sqlserver-rhel9 SQL container' + condition: always() + timeoutInMinutes: 3 + env: + DB_PASSWORD: $(DB_PASSWORD) + + - script: | docker stop test-container-rhel9 || true docker rm test-container-rhel9 || true - docker stop sqlserver-rhel9 || true - docker rm sqlserver-rhel9 || true displayName: 'Clean up RHEL 9 containers' condition: always() @@ -1614,56 +1478,11 @@ jobs: displayName: 'Create RHEL 9 ARM64 container' - script: | - # Start SQL Server container (x86_64 - SQL Server doesn't support ARM64) - docker run -d --name sqlserver-rhel9-arm64 \ - --platform linux/amd64 \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ - -p 1433:1433 \ - mcr.microsoft.com/mssql/server:2022-latest - - # Wait for SQL Server to be ready - echo "Waiting for SQL Server to start..." - sql_ready=false - for i in {1..60}; do - if docker exec sqlserver-rhel9-arm64 \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - sql_ready=true - break - fi - echo "Waiting... ($i/60)" - sleep 2 - done - if [ "$sql_ready" != true ]; then - # the loop sleeps after its final attempt, so probe once more - if docker exec sqlserver-rhel9-arm64 \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - else - echo "SQL Server did not become ready after 60 attempts" - echo "---- docker logs sqlserver-rhel9-arm64 (last 200 lines) ----" - docker logs --tail 200 sqlserver-rhel9-arm64 || true - exit 1 - fi - fi - - # Create test database - docker exec sqlserver-rhel9-arm64 \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "CREATE DATABASE TestDB" + python3 eng/scripts/setup_sql_container.py \ + --name "sqlserver-rhel9-arm64" --owner "$(Build.BuildId).$(System.JobId)" \ + --image "mcr.microsoft.com/mssql/server:2022-latest" --database TestDB displayName: 'Start SQL Server container for RHEL 9 ARM64' + timeoutInMinutes: 22 env: DB_PASSWORD: $(DB_PASSWORD) @@ -1824,11 +1643,17 @@ jobs: condition: always() - script: | - # Clean up containers + python3 eng/scripts/setup_sql_container.py --cleanup \ + --name "sqlserver-rhel9-arm64" --owner "$(Build.BuildId).$(System.JobId)" + displayName: 'Clean up sqlserver-rhel9-arm64 SQL container' + condition: always() + timeoutInMinutes: 3 + env: + DB_PASSWORD: $(DB_PASSWORD) + + - script: | docker stop test-container-rhel9-arm64 || true docker rm test-container-rhel9-arm64 || true - docker stop sqlserver-rhel9-arm64 || true - docker rm sqlserver-rhel9-arm64 || true displayName: 'Clean up RHEL 9 ARM64 containers' condition: always() @@ -1863,56 +1688,11 @@ jobs: displayName: 'Create Alpine x86_64 container' - script: | - # Start SQL Server container (x86_64) - docker run -d --name sqlserver-alpine \ - --platform linux/amd64 \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ - -p 1433:1433 \ - mcr.microsoft.com/mssql/server:2022-latest - - # Wait for SQL Server to be ready - echo "Waiting for SQL Server to start..." - sql_ready=false - for i in {1..60}; do - if docker exec sqlserver-alpine \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - sql_ready=true - break - fi - echo "Waiting... ($i/60)" - sleep 2 - done - if [ "$sql_ready" != true ]; then - # the loop sleeps after its final attempt, so probe once more - if docker exec sqlserver-alpine \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - else - echo "SQL Server did not become ready after 60 attempts" - echo "---- docker logs sqlserver-alpine (last 200 lines) ----" - docker logs --tail 200 sqlserver-alpine || true - exit 1 - fi - fi - - # Create test database - docker exec sqlserver-alpine \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "CREATE DATABASE TestDB" + python3 eng/scripts/setup_sql_container.py \ + --name "sqlserver-alpine" --owner "$(Build.BuildId).$(System.JobId)" \ + --image "mcr.microsoft.com/mssql/server:2022-latest" --database TestDB displayName: 'Start SQL Server container for Alpine x86_64' + timeoutInMinutes: 22 env: DB_PASSWORD: $(DB_PASSWORD) @@ -2096,11 +1876,17 @@ jobs: condition: always() - script: | - # Clean up containers + python3 eng/scripts/setup_sql_container.py --cleanup \ + --name "sqlserver-alpine" --owner "$(Build.BuildId).$(System.JobId)" + displayName: 'Clean up sqlserver-alpine SQL container' + condition: always() + timeoutInMinutes: 3 + env: + DB_PASSWORD: $(DB_PASSWORD) + + - script: | docker stop test-container-alpine || true docker rm test-container-alpine || true - docker stop sqlserver-alpine || true - docker rm sqlserver-alpine || true displayName: 'Clean up Alpine x86_64 containers' condition: always() @@ -2137,56 +1923,11 @@ jobs: displayName: 'Create Alpine ARM64 container' - script: | - # Start SQL Server container (x86_64 - SQL Server doesn't support ARM64) - docker run -d --name sqlserver-alpine-arm64 \ - --platform linux/amd64 \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ - -p 1433:1433 \ - mcr.microsoft.com/mssql/server:2022-latest - - # Wait for SQL Server to be ready - echo "Waiting for SQL Server to start..." - sql_ready=false - for i in {1..60}; do - if docker exec sqlserver-alpine-arm64 \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - sql_ready=true - break - fi - echo "Waiting... ($i/60)" - sleep 2 - done - if [ "$sql_ready" != true ]; then - # the loop sleeps after its final attempt, so probe once more - if docker exec sqlserver-alpine-arm64 \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" >/dev/null 2>&1; then - echo "SQL Server is ready!" - else - echo "SQL Server did not become ready after 60 attempts" - echo "---- docker logs sqlserver-alpine-arm64 (last 200 lines) ----" - docker logs --tail 200 sqlserver-alpine-arm64 || true - exit 1 - fi - fi - - # Create test database - docker exec sqlserver-alpine-arm64 \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "CREATE DATABASE TestDB" + python3 eng/scripts/setup_sql_container.py \ + --name "sqlserver-alpine-arm64" --owner "$(Build.BuildId).$(System.JobId)" \ + --image "mcr.microsoft.com/mssql/server:2022-latest" --database TestDB displayName: 'Start SQL Server container for Alpine ARM64' + timeoutInMinutes: 22 env: DB_PASSWORD: $(DB_PASSWORD) @@ -2371,11 +2112,17 @@ jobs: condition: always() - script: | - # Clean up containers + python3 eng/scripts/setup_sql_container.py --cleanup \ + --name "sqlserver-alpine-arm64" --owner "$(Build.BuildId).$(System.JobId)" + displayName: 'Clean up sqlserver-alpine-arm64 SQL container' + condition: always() + timeoutInMinutes: 3 + env: + DB_PASSWORD: $(DB_PASSWORD) + + - script: | docker stop test-container-alpine-arm64 || true docker rm test-container-alpine-arm64 || true - docker stop sqlserver-alpine-arm64 || true - docker rm sqlserver-alpine-arm64 || true displayName: 'Clean up Alpine ARM64 containers' condition: always() @@ -2624,44 +2371,11 @@ jobs: displayName: 'Install build dependencies' - script: | - # Start SQL Server container - docker pull mcr.microsoft.com/mssql/server:2022-latest - docker run \ - --name sqlserver \ - -e ACCEPT_EULA=Y \ - -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ - -p 1433:1433 \ - -d mcr.microsoft.com/mssql/server:2022-latest - - # Wait until SQL Server is ready - sql_ready=false - for i in {1..30}; do - if docker exec sqlserver \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1"; then - sql_ready=true - break - fi - sleep 2 - done - if [ "$sql_ready" != true ]; then - # the loop sleeps after its final attempt, so probe once more - if ! docker exec sqlserver \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1"; then - echo "SQL Server did not become ready after 30 attempts" - echo "---- docker logs sqlserver (last 200 lines) ----" - docker logs --tail 200 sqlserver || true - exit 1 - fi - fi + python3 eng/scripts/setup_sql_container.py \ + --name "sqlserver" --owner "$(Build.BuildId).$(System.JobId)" \ + --image "mcr.microsoft.com/mssql/server:2022-latest" displayName: 'Start SQL Server container' + timeoutInMinutes: 22 env: DB_PASSWORD: $(DB_PASSWORD) @@ -2708,3 +2422,12 @@ jobs: reportDirectory: 'unified-coverage' failIfCoverageEmpty: true displayName: 'Publish unified code coverage results' + + - script: | + python3 eng/scripts/setup_sql_container.py --cleanup \ + --name sqlserver --owner "$(Build.BuildId).$(System.JobId)" + displayName: 'Clean up coverage SQL container' + condition: always() + timeoutInMinutes: 3 + env: + DB_PASSWORD: $(DB_PASSWORD) diff --git a/eng/scripts/setup_sql_container.py b/eng/scripts/setup_sql_container.py new file mode 100644 index 00000000..4ec93203 --- /dev/null +++ b/eng/scripts/setup_sql_container.py @@ -0,0 +1,513 @@ +#!/usr/bin/env python3 +"""Create a job-owned SQL Server container, retrying SQL setup once. + +Invoke on the Docker host with --name, --image and --owner; DB_PASSWORD is +required for setup and is never an argument. Use --database TestDB on Linux +test legs, --colima on macOS, and --cleanup for the always-running final step. +Cleanup refuses containers without the matching owner label. + +Only SQL pull/create/start/readiness/database setup is retried. Colima starts +once. Configuration, ownership, preflight and cleanup failures are terminal. +Linux/macOS attempts are bounded at 600/900 seconds including diagnostics and +cleanup; entire invocations at 1260/2460 seconds including macOS VM startup. +Command budgets include termination/output-drain grace. Cleanup-only uses a +115-second deadline. OS scheduling/host loss can defeat cooperative deadlines; +the pipeline also enforces outer task limits. + +Diagnostics retain redacted beginning/end excerpts, not dumps or full inspect +output. Timeout/cancellation stops only subprocesses created by this helper. +Hard host loss can still prevent cleanup; the pipeline also runs --cleanup. +""" + +import argparse +import codecs +from dataclasses import dataclass +import json +import os +import re +import signal +import subprocess +import sys +import threading +import time + +OWNER_LABEL = "com.microsoft.mssql-python.ci-owner" +STATE_FORMAT = ( + '{{.Id}}|{{index .Config.Labels "' + OWNER_LABEL + '"}}|' + "{{.State.Status}}|{{.State.ExitCode}}|{{.State.OOMKilled}}|{{.Image}}" +) +SQLCMD = "/opt/mssql-tools18/bin/sqlcmd" + + +class SetupFailure(Exception): + def __init__(self, message, *, retryable=True): + super().__init__(message) + self.retryable = retryable + + +class Cancelled(BaseException): + def __init__(self, signum): + self.signum = signum + + +def redact(text, password): + if password: + text = text.replace(password, "[REDACTED]") + return re.sub( + r"(?i)\b(?:DB_PASSWORD|MSSQL_SA_PASSWORD|SQLCMDPASSWORD|PASSWORD|PWD)" + r"\s*[:=]\s*(?:\"[^\"]*\"|'[^']*'|[^;\r\n]*)", + "[credential redacted]", + text, + ) + + +class SafeCapture: + """Redact complete lines before retaining bounded head/tail excerpts.""" + + def __init__(self, password, limit=32768): + self.password = password + self.limit = limit + self.head = "" + self.tail = "" + self.total = 0 + + def add(self, line): + safe = redact(line, self.password) + self.total += len(safe) + remaining = self.limit - len(self.head) + self.head += safe[:remaining] + self.tail = (self.tail + safe[remaining:])[-self.limit :] + + def read(self, stream): + decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") + pending = "" + omitted = False + while True: + chunk = stream.read(4096) + text = decoder.decode(chunk, final=not chunk) + for part in text.splitlines(keepends=True): + pending += part + if len(pending) > 8192: + if not omitted: + self.add("[overlong diagnostic line omitted]\n") + pending = "" + omitted = True + if part.endswith(("\n", "\r")): + if not omitted: + self.add(pending) + pending = "" + omitted = False + if not chunk: + if pending and not omitted: + self.add(pending) + return + + def output(self): + marker = "\n[diagnostic output truncated]\n" if self.total > 2 * self.limit else "" + return self.head + marker + self.tail + + +@dataclass +class Result: + returncode: int + output: str + + +class Commands: + def __init__(self, password): + self.password = password + + @staticmethod + def stop(process, deadline): + # start_new_session gives each command its own group. The launcher may + # already be reaped while a descendant still holds its output pipe. + try: + if os.name == "posix": + os.killpg(process.pid, signal.SIGTERM) + elif process.poll() is None: + process.terminate() + except ProcessLookupError: + pass + try: + process.wait(timeout=max(0, min(1, (deadline - time.monotonic()) / 2))) + except subprocess.TimeoutExpired: + pass + try: + if os.name == "posix": + os.killpg(process.pid, signal.SIGKILL) + elif process.poll() is None: + process.kill() + except ProcessLookupError: + pass + try: + process.wait(timeout=max(0, min(1, deadline - time.monotonic()))) + except subprocess.TimeoutExpired: + return False + return True + + def run(self, args, timeout, *, env=None): + if timeout <= 0: + raise SetupFailure("SQL setup deadline exhausted") + deadline = time.monotonic() + timeout + grace = min(4, timeout / 2) + capture = SafeCapture(self.password) + try: + process = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, + start_new_session=os.name == "posix", + ) + except OSError: + raise SetupFailure("Cannot launch required setup command", retryable=False) from None + reader = threading.Thread(target=capture.read, args=(process.stdout,), daemon=True) + reader.start() + timed_out = False + descendant_output = False + reaped = True + try: + try: + process.wait(timeout=max(0, deadline - time.monotonic() - grace)) + except subprocess.TimeoutExpired: + timed_out = True + finally: + if process.poll() is None: + reaped = self.stop(process, deadline) + else: + reader.join(timeout=max(0, min(0.2, (deadline - time.monotonic()) / 4))) + if reader.is_alive(): + descendant_output = True + reaped = self.stop(process, deadline) + reader.join(timeout=max(0, deadline - time.monotonic())) + if not reader.is_alive(): + process.stdout.close() + if not reaped or reader.is_alive(): + print("[sql] Command teardown incomplete within its deadline", file=sys.stderr) + if not reaped: + raise SetupFailure("Setup command could not be reaped", retryable=False) + if reader.is_alive(): + raise SetupFailure("Setup command output did not close", retryable=False) + if descendant_output: + raise SetupFailure("Setup command left descendants holding output", retryable=False) + if timed_out: + raise SetupFailure("Setup command timed out\n" + capture.output()) + return Result(process.returncode, capture.output()) + + +@dataclass +class Container: + identifier: str + status: str + exit_code: str + oom_killed: str + image: str + + +class SqlSetup: + def __init__(self, args, password, commands=None): + self.args = args + self.password = password + self.commands = commands or Commands(password) + self.deadline = time.monotonic() + (115 if args.cleanup else 2460 if args.colima else 1260) + self.phase_deadline = self.deadline + self.container = None + self.image_id = None + self.docker = ["docker"] + (["--context", "colima"] if args.colima else []) + self.env = os.environ.copy() + for key in ("DB_PASSWORD", "DB_CONNECTION_STRING", "MSSQL_SA_PASSWORD", "SQLCMDPASSWORD"): + self.env.pop(key, None) + self.env.update(MSSQL_SA_PASSWORD=password, SQLCMDPASSWORD=password) + + def log(self, message): + print("[sql] " + redact(message, self.password), flush=True) + + def command(self, args, timeout=15, *, check=True, deadline=None): + end = min(self.deadline, self.phase_deadline if deadline is None else deadline) + remaining = min(timeout, end - time.monotonic()) + if remaining <= 0: + raise SetupFailure("SQL setup deadline exhausted") + result = self.commands.run(args, remaining, env=self.env) + if result.returncode in (-signal.SIGINT, -signal.SIGTERM, 130, 143): + raise Cancelled( + signal.SIGINT if result.returncode in (-signal.SIGINT, 130) else signal.SIGTERM + ) + if check and result.returncode != 0: + raise SetupFailure(f"Setup command failed (exit {result.returncode})\n{result.output}") + return result + + def docker_command(self, *args, **kwargs): + return self.command(self.docker + list(args), **kwargs) + + def find_owned(self, *, deadline=None): + result = self.docker_command( + "container", + "ls", + "--all", + "--no-trunc", + "--filter", + f"name=^/{self.args.name}$", + "--format", + "{{.ID}}", + deadline=deadline, + ) + identifiers = result.output.split() + if not identifiers: + return None + if len(identifiers) != 1 or not re.fullmatch(r"[0-9a-f]{64}", identifiers[0]): + raise SetupFailure("Unexpected exact-name container lookup result", retryable=False) + result = self.docker_command( + "inspect", "--format", STATE_FORMAT, identifiers[0], deadline=deadline + ) + fields = result.output.strip().split("|") + if len(fields) != 6 or fields[0] != identifiers[0]: + raise SetupFailure("Invalid selected container state", retryable=False) + if fields[1] != self.args.owner: + raise SetupFailure( + "Container name is owned by another job or unlabelled", retryable=False + ) + return Container(fields[0], fields[2], fields[3], fields[4], fields[5]) + + def diagnostics(self, container, *, deadline): + self.log( + f"Container {container.identifier}: status={container.status} " + f"exit={container.exit_code} OOMKilled={container.oom_killed} image={container.image}" + ) + try: + result = self.docker_command( + "logs", container.identifier, timeout=20, check=False, deadline=deadline + ) + self.log("Container log excerpts:\n" + result.output) + if result.returncode: + self.log(f"Container logs unavailable (exit {result.returncode})") + except SetupFailure as exc: + self.log(f"Container logs unavailable: {exc}") + + def remove(self, container, *, deadline): + self.docker_command("rm", "--force", container.identifier, timeout=30, deadline=deadline) + remaining = self.find_owned(deadline=deadline) + if remaining is not None: + raise SetupFailure("Owned container still exists after removal", retryable=False) + self.container = None + + def cleanup(self, *, evidence=True): + # Include lookup/ownership checks and removal verification in addition + # to the log and rm deadlines. + self.phase_deadline = min(self.deadline, time.monotonic() + 100) + container = self.find_owned() + if container is None: + self.container = None + return + if evidence: + self.diagnostics( + container, deadline=min(self.phase_deadline - 30, time.monotonic() + 20) + ) + self.remove(container, deadline=self.phase_deadline) + + def preflight(self): + if self.args.colima and not self.args.cleanup: + self.log("Starting Colima once (outside SQL retry)") + self.command(["colima", "start", "--cpu", "4", "--memory", "8", "--disk", "50"], 600) + self.docker_command("info", "--format", "{{.ServerVersion}}") + + def acquire_image(self): + if self.image_id is not None: + return + self.docker_command( + "pull", + "--quiet", + "--platform", + "linux/amd64", + self.args.image, + timeout=600 if self.args.colima else 300, + ) + result = self.docker_command( + "image", "inspect", "--format", "{{.Id}} {{json .RepoDigests}}", self.args.image + ) + parts = result.output.strip().split(" ", 1) + if len(parts) != 2 or not re.fullmatch(r"sha256:[0-9a-f]{64}", parts[0]): + raise SetupFailure("Invalid resolved SQL image", retryable=False) + try: + digests = json.loads(parts[1]) + except json.JSONDecodeError: + raise SetupFailure("Invalid SQL image digest metadata", retryable=False) from None + if not isinstance(digests, list) or not all(isinstance(item, str) for item in digests): + raise SetupFailure("Missing SQL image digest metadata", retryable=False) + self.image_id = parts[0] + self.log(f"SQL image {self.image_id}; repository digests={json.dumps(digests)}") + + def sql(self, query, *, timeout=15, query_timeout=5, deadline=None): + return self.docker_command( + "exec", + "--env", + "SQLCMDPASSWORD", + self.container.identifier, + SQLCMD, + "-S", + "localhost", + "-U", + "SA", + "-C", + "-b", + "-l", + "5", + "-t", + str(query_timeout), + "-Q", + query, + timeout=timeout, + check=False, + deadline=deadline, + ) + + def attempt(self): + stale = self.find_owned() + if stale is not None: + self.log("Removing pre-existing same-job container before fresh setup") + self.diagnostics(stale, deadline=min(self.phase_deadline, time.monotonic() + 20)) + try: + self.remove(stale, deadline=min(self.phase_deadline, time.monotonic() + 30)) + except SetupFailure as exc: + raise SetupFailure(str(exc), retryable=False) from None + self.acquire_image() + self.docker_command( + "create", + "--name", + self.args.name, + "--label", + f"{OWNER_LABEL}={self.args.owner}", + "--platform", + "linux/amd64", + "--env", + "ACCEPT_EULA=Y", + "--env", + "MSSQL_SA_PASSWORD", + "-p", + "1433:1433", + self.image_id, + timeout=30, + ) + self.container = self.find_owned() + if self.container is None: + raise SetupFailure("Created SQL container was not found") + self.docker_command("start", self.container.identifier, timeout=30) + ready_deadline = min( + self.phase_deadline, time.monotonic() + (180 if self.args.colima else 120) + ) + last_output = "" + while time.monotonic() < ready_deadline: + current = self.find_owned(deadline=ready_deadline) + if current is None or current.identifier != self.container.identifier: + raise SetupFailure("SQL container disappeared or was replaced", retryable=False) + if current.status != "running": + raise SetupFailure( + f"SQL container exited before readiness (status={current.status}, " + f"exit={current.exit_code}, OOMKilled={current.oom_killed})" + ) + probe = self.sql("SELECT 1", deadline=ready_deadline) + if probe.returncode == 0: + if self.args.database: + result = self.sql("CREATE DATABASE TestDB", timeout=30, query_timeout=15) + if result.returncode != 0: + raise SetupFailure("TestDB initialization failed\n" + result.output) + return + if probe.returncode in (126, 127): + raise SetupFailure("Required sqlcmd executable is unavailable", retryable=False) + last_output = probe.output + time.sleep(max(0, min(2, ready_deadline - time.monotonic()))) + raise SetupFailure("SQL readiness deadline exhausted\n" + last_output) + + def setup(self): + self.preflight() + if self.args.cleanup: + self.cleanup(evidence=False) + self.log("Owned SQL container cleanup complete (or already absent)") + return + for number in (1, 2): + self.log(f"SQL setup attempt {number}/2") + attempt_end = min( + self.deadline, + time.monotonic() + (900 if self.args.colima else 600), + ) + self.phase_deadline = attempt_end - 100 + try: + self.attempt() + except SetupFailure as exc: + self.log(f"Attempt {number}/2 failed: {exc}") + self.phase_deadline = attempt_end + try: + self.cleanup() + except SetupFailure as cleanup_error: + raise SetupFailure( + f"Cannot safely recover/clean up: {cleanup_error}", retryable=False + ) from None + if not exc.retryable or number == 2: + raise SetupFailure("SQL setup failed; no further attempts", retryable=False) + self.phase_deadline = self.deadline + self.docker_command("info", "--format", "{{.ServerVersion}}") + if self.deadline - time.monotonic() < 5: + raise SetupFailure("SQL setup deadline exhausted before retry", retryable=False) + self.log("Retrying SQL setup only after 5 seconds") + time.sleep(5) + else: + outcome = "ready on first attempt" if number == 1 else "recovered on second attempt" + self.log(f"SQL Server {outcome}") + return + + +def arguments(argv): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--name", required=True) + parser.add_argument("--owner", required=True) + parser.add_argument("--image") + parser.add_argument("--database", choices=("TestDB",)) + parser.add_argument("--colima", action="store_true") + parser.add_argument("--cleanup", action="store_true") + args = parser.parse_args(argv) + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", args.name): + raise SetupFailure("Invalid SQL container name", retryable=False) + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.:-]*", args.owner): + raise SetupFailure("Invalid SQL container owner", retryable=False) + if not args.cleanup and ( + not args.image + or not re.fullmatch(r"mcr\.microsoft\.com/mssql/server:[A-Za-z0-9_.-]+", args.image) + ): + raise SetupFailure( + "An explicit supported SQL Server image tag is required", retryable=False + ) + return args + + +def main(argv=None): + password = os.environ.get("DB_PASSWORD", "") + setup = None + + def cancel(signum, _frame): + raise Cancelled(signum) + + for signum in (signal.SIGINT, signal.SIGTERM): + signal.signal(signum, cancel) + try: + args = arguments(argv) + if not args.cleanup and (not password or "\n" in password or "\r" in password): + raise SetupFailure("DB_PASSWORD must be set to a single-line value", retryable=False) + setup = SqlSetup(args, password) + setup.setup() + return 0 + except Cancelled as exc: + for signum in (signal.SIGINT, signal.SIGTERM): + signal.signal(signum, signal.SIG_IGN) + print("[sql] Setup cancelled; no retry", flush=True) + if setup is not None: + try: + setup.cleanup() + except SetupFailure as cleanup_error: + setup.log(f"Cancellation cleanup failed: {cleanup_error}") + return 128 + exc.signum + except SetupFailure as exc: + print("[sql] " + redact(str(exc), password), file=sys.stderr, flush=True) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_sql_container_pipeline.py b/tests/test_sql_container_pipeline.py new file mode 100644 index 00000000..b78a5bcd --- /dev/null +++ b/tests/test_sql_container_pipeline.py @@ -0,0 +1,171 @@ +import os +from pathlib import Path +import re +import subprocess +import textwrap + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +PIPELINE = ROOT / "eng" / "pipelines" / "pr-validation-pipeline.yml" +if not PIPELINE.is_file(): + pytest.skip("SQL pipeline contracts require a source checkout", allow_module_level=True) + +JOBS = { + "PytestOnMacOS": ("sqlserver", False), + "PytestOnLinux": ("sqlserver-$(distroName)", True), + "PytestOnLinux_ARM64": ("sqlserver-$(distroName)-$(archName)", True), + "PytestOnLinux_RHEL9": ("sqlserver-rhel9", True), + "PytestOnLinux_RHEL9_ARM64": ("sqlserver-rhel9-arm64", True), + "PytestOnLinux_Alpine": ("sqlserver-alpine", True), + "PytestOnLinux_Alpine_ARM64": ("sqlserver-alpine-arm64", True), + "CodeCoverageReport": ("sqlserver", False), +} + + +def section(name): + text = PIPELINE.read_text(encoding="utf-8") + return re.search(r"^- job: " + name + r"\n.*?(?=^- job: |\Z)", text, re.M | re.S)[0] + + +def script_steps(name): + return re.findall(r"^ - script: \|\n.*?(?=^ - |\Z)", section(name), re.M | re.S) + + +@pytest.mark.parametrize("name, configuration", JOBS.items()) +def test_all_sql_setup_and_cleanup_paths_are_wired(name, configuration): + container, database = configuration + steps = script_steps(name) + setup = next(step for step in steps if "setup_sql_container.py" in step and "--image" in step) + assert container in setup + assert ("--database TestDB" in setup) == database + assert "$(Build.BuildId).$(System.JobId)" in setup + assert "DB_PASSWORD: $(DB_PASSWORD)" in setup + assert "continueOnError" not in setup and "retryCountOnTaskFailure" not in setup + assert "MSSQL_SA_PASSWORD=" not in setup and "-P " not in setup + cleanup = [ + step + for step in steps + if "setup_sql_container.py" in step and "--cleanup" in step and "--image" not in step + ] + assert len(cleanup) == 1 + assert "always()" in cleanup[0] + assert "timeoutInMinutes: 3" in cleanup[0] + assert "test-container" not in cleanup[0] + assert container in cleanup[0] + if name != "PytestOnMacOS": + assert "timeoutInMinutes: 22" in setup + + +def test_azuresql_skips_local_sql_and_preserves_matrix(): + linux = section("PytestOnLinux") + assert "Ubuntu_AzureSQL:" in linux + assert "if ne(variables['AZURE_CONNECTION_STRING'], '')" in linux + setup = next(step for step in script_steps("PytestOnLinux") if "--image" in step) + assert "condition: and(succeeded(), eq(variables['useAzureSQL'], 'false'))" in setup + cleanup = next(step for step in script_steps("PytestOnLinux") if "--cleanup" in step) + assert "condition: and(always(), eq(variables['useAzureSQL'], 'false'))" in cleanup + assert "AZURE_CONNECTION_STRING" in linux + + +def test_no_new_build_test_retries_or_sql_architecture_changes(): + pipeline = PIPELINE.read_text(encoding="utf-8") + assert pipeline.count("retryCountOnTaskFailure:") == 3 + for name in ("PytestOnLinux_ARM64", "PytestOnLinux_RHEL9_ARM64", "PytestOnLinux_Alpine_ARM64"): + steps = [step for step in script_steps(name) if "retryCountOnTaskFailure:" in step] + assert len(steps) == 1 + assert "retryCountOnTaskFailure: 2" in steps[0] + assert "build.sh" in steps[0] + assert "setup_sql_container.py" not in steps[0] and "pytest" not in steps[0] + helper = (ROOT / "eng" / "scripts" / "setup_sql_container.py").read_text(encoding="utf-8") + assert '"linux/amd64"' in helper + assert "linux/arm64" not in helper + assert "Config.Env" not in helper and "prune" not in helper + for name in JOBS: + for step in script_steps(name): + if "python -m pytest" in step: + assert "retryCountOnTaskFailure" not in step + assert "continueOnError" not in step + assert "--cleanup" not in step + + +def test_mac_overlap_and_failure_wait_remain_separate_from_tests(): + mac = section("PytestOnMacOS") + assert "DOCKER_CONTEXT: colima" in mac + assert "timeoutInMinutes: 90" in mac + assert "SQL2022:" in mac and "SQL2025:" in mac + setup = next(step for step in script_steps("PytestOnMacOS") if "--image" in step) + assert '> "$SQL_LOG" 2>&1 &' in setup + assert setup.count("./build.sh") == 1 + assert setup.count("pip install -r requirements.txt") == 1 + assert ( + setup.index("SQL_PID=$!") < setup.index("./build.sh") < setup.rindex('wait "$SQL_PID" ||') + ) + assert 'exit "$SQL_STATUS"' in setup + assert "trap finish_setup EXIT" in setup + assert "pytest" not in setup + assert "/tmp/sql_setup.log" not in setup + + +@pytest.mark.skipif(os.name != "posix", reason="Hosted Unix Bash supervision contract") +@pytest.mark.parametrize("sql_status, build_status", [(0, 0), (17, 0), (0, 27)]) +def test_mac_script_builds_once_and_gates_dependents(tmp_path, sql_status, build_status): + step = next(step for step in script_steps("PytestOnMacOS") if "--image" in step) + script = textwrap.dedent(step.split(" displayName:", 1)[0].split("\n", 1)[1]) + replacements = { + "$(Agent.TempDirectory)": str(tmp_path), + "$(Build.SourcesDirectory)": str(tmp_path), + "$(System.JobId)": "test-job", + "$(Build.BuildId)": "1", + "$(sqlServerImage)": "mcr.microsoft.com/mssql/server:2025-latest", + } + for before, after in replacements.items(): + script = script.replace(before, after) + bindir = tmp_path / "bin" + bindir.mkdir() + events = tmp_path / "events" + builddir = tmp_path / "mssql_python" / "pybind" + builddir.mkdir(parents=True) + build = builddir / "build.sh" + build.write_text('#!/bin/sh\necho build >> "$EVENTS"\nexit "$BUILD_STATUS"\n', encoding="utf-8") + build.chmod(0o700) + for name in ("python", "pip"): + file = bindir / name + file.write_text('#!/bin/sh\necho dependency >> "$EVENTS"\n', encoding="utf-8") + file.chmod(0o700) + python3 = bindir / "python3" + python3.write_text( + "#!/bin/sh\n" + 'case "$*" in\n' + ' *--cleanup*) echo cleanup >> "$EVENTS"; exit 0 ;;\n' + "esac\n" + 'echo sql >> "$EVENTS"\n' + 'exit "$SQL_STATUS_TEST"\n', + encoding="utf-8", + ) + python3.chmod(0o700) + env = { + **os.environ, + "PATH": str(bindir) + os.pathsep + os.environ["PATH"], + "EVENTS": str(events), + "SQL_STATUS_TEST": str(sql_status), + "BUILD_STATUS": str(build_status), + } + script_file = tmp_path / "step.sh" + script_file.write_text(script, encoding="utf-8") + result = subprocess.run( + ["bash", str(script_file)], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=15, + ) + actions = events.read_text(encoding="utf-8").splitlines() + assert result.returncode == (build_status or sql_status), result.stdout + result.stderr + assert actions.count("build") == 1 + assert actions.count("dependency") == 2 + assert actions.count("sql") <= 1 + assert actions.count("cleanup") == (1 if result.returncode else 0) + if result.returncode == 0: + assert actions.count("sql") == 1 diff --git a/tests/test_sql_container_setup.py b/tests/test_sql_container_setup.py new file mode 100644 index 00000000..0e70ddae --- /dev/null +++ b/tests/test_sql_container_setup.py @@ -0,0 +1,482 @@ +import importlib.util +import io +import os +from pathlib import Path +import signal +import subprocess +import sys +import time +from types import SimpleNamespace + +import pytest + +HELPER = Path(__file__).resolve().parents[1] / "eng" / "scripts" / "setup_sql_container.py" +if not HELPER.is_file(): + pytest.skip("SQL setup contracts require a source checkout", allow_module_level=True) +spec = importlib.util.spec_from_file_location("sql_container_setup", HELPER) +sql_setup = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = sql_setup +spec.loader.exec_module(sql_setup) + +PASSWORD = "Dummy-Secret-Canary!42" +OWNER = "123.owned-job" +IMAGE_ID = "sha256:" + "a" * 64 + + +class Clock: + def __init__(self): + self.now = 0 + self.sleeps = [] + + def monotonic(self): + return self.now + + def sleep(self, seconds): + self.sleeps.append(seconds) + self.now += seconds + + +class Docker: + def __init__(self, clock, scenario="success"): + self.clock = clock + self.scenario = scenario + self.calls = [] + self.created = 0 + self.identifier = None + self.status = "created" + self.owner = OWNER + + def count(self, command): + return sum(args[0] == command for args, _, _ in self.calls) + + def run(self, args, timeout, *, env): + assert timeout > 0 + args = list(args) + if args[0] == "docker": + args.pop(0) + if args[:2] == ["--context", "colima"]: + args = args[2:] + self.calls.append((args, timeout, env)) + command = args[0] + result = sql_setup.Result + if command == "colima": + return result(0, "") + if command == "info": + return result(1 if self.scenario == "daemon" else 0, "docker status") + if command == "container": + if self.scenario == "lookup-failure": + return result(1, "cannot query daemon") + return result(0, (self.identifier + "\n") if self.identifier else "") + if command == "inspect": + return result(0, f"{self.identifier}|{self.owner}|{self.status}|137|false|{IMAGE_ID}\n") + if command == "pull": + return result(1 if self.scenario == "pull-failure" else 0, "pull result") + if command == "image": + return result(0, f'{IMAGE_ID} ["mcr.microsoft.com/mssql/server@{IMAGE_ID}"]\n') + if command == "create": + self.created += 1 + if self.scenario == "create-once" and self.created == 1: + return result(1, "creation failed") + self.identifier = f"{self.created:064x}" + self.status = "created" + if self.scenario == "create-timeout" and self.created == 1: + self.clock.sleep(timeout) + raise sql_setup.SetupFailure("create timed out after creating the container") + return result(0, self.identifier) + if command == "start": + self.status = "running" + if self.scenario == "start-once" and self.created == 1: + return result(1, "start failed") + if self.scenario == "cancel": + raise sql_setup.Cancelled(signal.SIGTERM) + if self.scenario == "interrupt-exit": + return result(130, "interrupted") + if self.scenario == "dead" or (self.scenario == "dead-once" and self.created == 1): + self.status = "exited" + return result(0, "") + if command == "exec": + if self.scenario == "hung": + self.clock.sleep(timeout) + raise sql_setup.SetupFailure("exec timed out") + if self.scenario == "timeout": + self.clock.sleep(min(5, timeout)) + return result(1, "not ready") + if self.scenario == "no-sqlcmd": + return result(127, "missing sqlcmd") + if self.scenario == "database" and args[-1] == "CREATE DATABASE TestDB": + return result(1, "SQL initialization error") + return result(0, "1") + if command == "logs": + return result(0, f"fatal header\nPASSWORD={PASSWORD}\nend of log\n") + if command == "rm": + assert args[-1] == self.identifier + if self.scenario == "remove-failure": + return result(1, "removal denied") + self.identifier = None + return result(0, "") + raise AssertionError(args) + + +@pytest.fixture +def setup_factory(monkeypatch): + def factory(scenario="success", *, colima=False, database=True): + clock = Clock() + monkeypatch.setattr(sql_setup, "time", clock) + args = SimpleNamespace( + name="sqlserver", + owner=OWNER, + image="mcr.microsoft.com/mssql/server:2025-latest", + colima=colima, + cleanup=False, + database="TestDB" if database else None, + ) + docker = Docker(clock, scenario) + return sql_setup.SqlSetup(args, PASSWORD, docker), docker, clock + + return factory + + +@pytest.mark.parametrize("database", [True, False]) +def test_first_success_and_secret_environment(setup_factory, monkeypatch, capsys, database): + monkeypatch.setenv("MSSQL_SA_PASSWORD", "wrong-inherited-password") + monkeypatch.setenv("SQLCMDPASSWORD", "another-wrong-password") + setup, docker, _ = setup_factory(database=database) + setup.setup() + assert docker.count("create") == docker.count("start") == docker.count("pull") == 1 + assert docker.count("rm") == 0 + assert docker.count("exec") == (2 if database else 1) + for args, timeout, env in docker.calls: + assert PASSWORD not in " ".join(args) + assert "-P" not in args + assert env["MSSQL_SA_PASSWORD"] == env["SQLCMDPASSWORD"] == PASSWORD + assert "DB_CONNECTION_STRING" not in env + if args[0] == "exec": + assert args[1:3] == ["--env", "SQLCMDPASSWORD"] + assert "-b" in args and args[args.index("-l") + 1] == "5" + assert timeout <= 30 + assert "ready on first attempt" in capsys.readouterr().out + + +@pytest.mark.parametrize("scenario", ["create-once", "create-timeout", "start-once", "dead-once"]) +def test_first_failure_recovers_exactly_once(setup_factory, capsys, scenario): + setup, docker, clock = setup_factory(scenario, colima=True) + setup.setup() + assert docker.created == 2 + assert docker.count("pull") == docker.count("colima") == 1 + assert clock.sleeps.count(5) == 1 + assert docker.count("exec") == 2 + assert docker.identifier == f"{2:064x}" + if scenario != "create-once": + actions = [args[0] for args, _, _ in docker.calls] + assert actions.index("logs") < actions.index("rm") < len(actions) - 1 + assert docker.count("rm") == 1 + out = capsys.readouterr().out + assert PASSWORD not in out + assert "attempt 1/2" in out and "attempt 2/2" in out + assert "recovered on second attempt" in out + + +@pytest.mark.parametrize("scenario", ["dead", "database", "pull-failure", "hung", "timeout"]) +def test_permanent_failure_stops_after_two_attempts(setup_factory, scenario, capsys): + setup, docker, clock = setup_factory(scenario) + with pytest.raises(sql_setup.SetupFailure, match="no further attempts"): + setup.setup() + assert docker.count("pull") == (2 if scenario == "pull-failure" else 1) + assert docker.created == (0 if scenario == "pull-failure" else 2) + assert docker.identifier is None + assert docker.count("rm") == docker.created + assert clock.sleeps.count(5) >= 1 + assert clock.now <= 1260 + if scenario == "dead": + assert docker.count("exec") == 0 + assert clock.now == 5 + if scenario == "timeout": + assert 240 <= clock.now <= 246 + assert "recovered" not in capsys.readouterr().out + + +@pytest.mark.parametrize("status", ["running", "exited"]) +def test_preexisting_owned_container_is_not_accepted(setup_factory, status): + setup, docker, _ = setup_factory() + old_id = "e" * 64 + docker.identifier, docker.status = old_id, status + setup.setup() + actions = [args[0] for args, _, _ in docker.calls] + assert actions.index("logs") < actions.index("rm") < actions.index("create") + assert docker.created == 1 + assert docker.identifier != old_id + assert next(args[-1] for args, _, _ in docker.calls if args[0] == "rm") == old_id + + +@pytest.mark.parametrize("scenario", ["daemon", "lookup-failure", "remove-failure"]) +def test_unsafe_cleanup_or_daemon_failure_is_terminal(setup_factory, scenario): + setup, docker, _ = setup_factory(scenario) + if scenario == "remove-failure": + docker.identifier = "e" * 64 + with pytest.raises(sql_setup.SetupFailure): + setup.setup() + assert docker.created == 0 + assert docker.count("pull") == 0 + + +@pytest.mark.parametrize("owner", ["", "another-job"]) +def test_foreign_container_is_never_read_removed_or_reused(setup_factory, owner): + setup, docker, _ = setup_factory() + docker.identifier, docker.owner = "e" * 64, owner + with pytest.raises(sql_setup.SetupFailure): + setup.setup() + assert docker.count("logs") == docker.count("rm") == docker.created == 0 + + +def test_missing_sqlcmd_is_not_retried(setup_factory): + setup, docker, _ = setup_factory("no-sqlcmd") + with pytest.raises(sql_setup.SetupFailure): + setup.setup() + assert docker.created == docker.count("rm") == 1 + + +@pytest.mark.parametrize("scenario, code", [("cancel", 143), ("interrupt-exit", 130)]) +def test_cancellation_propagates_and_cleans_without_retry( + setup_factory, monkeypatch, scenario, code +): + setup, docker, _ = setup_factory(scenario) + monkeypatch.setenv("DB_PASSWORD", PASSWORD) + monkeypatch.setattr(sql_setup, "SqlSetup", lambda *args: setup) + monkeypatch.setattr(sql_setup.signal, "signal", lambda *args: None) + assert ( + sql_setup.main(["--name", "sqlserver", "--owner", OWNER, "--image", setup.args.image]) + == code + ) + assert docker.created == docker.count("rm") == 1 + assert docker.identifier is None + + +@pytest.mark.parametrize("value", ["", "two\nlines"]) +def test_bad_secret_is_terminal_before_docker(monkeypatch, value): + monkeypatch.setenv("DB_PASSWORD", value) + monkeypatch.setattr(sql_setup.signal, "signal", lambda *args: None) + assert ( + sql_setup.main( + [ + "--name", + "sqlserver", + "--owner", + OWNER, + "--image", + "mcr.microsoft.com/mssql/server:2025-latest", + ] + ) + == 1 + ) + + +def test_cleanup_absent_container_is_success(setup_factory): + setup, docker, _ = setup_factory() + setup.args.cleanup = True + setup.setup() + assert docker.count("info") == 1 + assert docker.created == docker.count("rm") == 0 + + +def test_stream_capture_redacts_across_chunks_and_retains_crash_header(): + capture = sql_setup.SafeCapture(PASSWORD, limit=1024) + content = "FATAL HEADER\n" + "a" * 4068 + PASSWORD + "\n" + ("line\n" * 1000) + content += "x" * 9000 + PASSWORD + "\nEND\n" + capture.read(io.BytesIO(content.encode())) + out = capture.output() + assert out.startswith("FATAL HEADER") + assert out.endswith("END\n") + assert PASSWORD not in out + assert "overlong diagnostic line omitted" in out + assert "truncated" in out + assert len(out) < 2200 + + +def test_real_command_timeout_is_bounded_and_redacted(): + start = time.monotonic() + with pytest.raises(sql_setup.SetupFailure, match="timed out") as error: + sql_setup.Commands(PASSWORD).run( + [ + sys.executable, + "-c", + "import os,signal,time; signal.signal(signal.SIGTERM,signal.SIG_IGN); " + "print(os.environ['SQLCMDPASSWORD'], flush=True); time.sleep(60)", + ], + 0.5, + env={**os.environ, "SQLCMDPASSWORD": PASSWORD}, + ) + assert time.monotonic() - start < 1.5 + assert PASSWORD not in str(error.value) + assert "[REDACTED]" in str(error.value) + + +def test_missing_command_is_an_explicit_terminal_failure(tmp_path): + with pytest.raises(sql_setup.SetupFailure, match="Cannot launch") as error: + sql_setup.Commands(PASSWORD).run([str(tmp_path / "missing-docker-command")], 1) + assert error.value.retryable is False + + +def test_cancellation_during_backoff_does_not_start_second_attempt(setup_factory): + setup, docker, clock = setup_factory("dead") + + def interrupt(_seconds): + raise sql_setup.Cancelled(signal.SIGINT) + + clock.sleep = interrupt + with pytest.raises(sql_setup.Cancelled): + setup.setup() + assert docker.created == docker.count("rm") == 1 + assert docker.identifier is None + + +def test_commands_clamp_to_phase_and_total_deadlines(setup_factory): + setup, docker, clock = setup_factory() + setup.phase_deadline = 3 + setup.docker_command("info", timeout=15) + assert docker.calls[-1][1] == 3 + setup.deadline = 2 + setup.docker_command("info", timeout=15) + assert docker.calls[-1][1] == 2 + clock.now = 2 + count = len(docker.calls) + with pytest.raises(sql_setup.SetupFailure, match="deadline exhausted"): + setup.docker_command("info") + assert len(docker.calls) == count + + +def test_cleanup_deadline_exhaustion_does_not_claim_removal(setup_factory, capsys): + setup, docker, clock = setup_factory() + docker.identifier = "e" * 64 + setup.deadline = 10 + original = docker.run + + def slow_lookup(args, timeout, *, env): + result = original(args, timeout, env=env) + clock.sleep(min(6, timeout)) + return result + + docker.run = slow_lookup + with pytest.raises(sql_setup.SetupFailure, match="deadline exhausted"): + setup.cleanup() + assert clock.now == 10 + assert docker.count("rm") == 0 + assert docker.identifier is not None + assert "cleanup complete" not in capsys.readouterr().out + + +def test_cleanup_only_has_its_own_global_deadline(setup_factory): + setup, docker, clock = setup_factory() + setup.args.cleanup = True + cleanup = sql_setup.SqlSetup(setup.args, PASSWORD, docker) + assert cleanup.deadline - clock.now == 115 + + +@pytest.mark.parametrize("colima, cap, total", [(False, 600, 1260), (True, 900, 2460)]) +def test_total_attempt_budget_reserves_cleanup(setup_factory, colima, cap, total): + setup, docker, clock = setup_factory(colima=colima) + assert setup.deadline == total + attempts = [] + cleanups = [] + + def preflight(): + clock.sleep((600 if colima else 0) + 15) + + original = docker.run + + def bounded_info(args, timeout, *, env): + result = original(args, timeout, env=env) + clock.sleep(timeout) + return result + + def exhaust_attempt(): + remaining = setup.phase_deadline - clock.now + attempts.append(remaining) + clock.sleep(remaining) + raise sql_setup.SetupFailure("attempt deadline exhausted") + + def cleanup(): + cleanups.append(setup.phase_deadline - clock.now) + clock.sleep(100) + + setup.preflight = preflight + docker.run = bounded_info + setup.attempt = exhaust_attempt + setup.cleanup = cleanup + with pytest.raises(sql_setup.SetupFailure, match="no further attempts"): + setup.setup() + assert attempts == [cap - 100, cap - 100] + assert cleanups == [100, 100] + assert clock.now == (600 if colima else 0) + 2 * cap + 35 + assert clock.now <= total + assert docker.created == 0 + + +@pytest.mark.skipif(os.name != "posix", reason="Unix descendant process-group contract") +def test_exited_launcher_descendant_is_terminated_without_touching_other_groups(tmp_path): + ready = tmp_path / "descendant" + child = ( + "import os,pathlib,signal,time; " + "signal.signal(signal.SIGTERM,signal.SIG_IGN); " + f"pathlib.Path({str(ready)!r}).write_text(str(os.getpid())); time.sleep(60)" + ) + launcher = ( + "import subprocess,sys,time,pathlib; " + f"subprocess.Popen([sys.executable,'-c',{child!r}]); " + f"ready=pathlib.Path({str(ready)!r}); " + "\nwhile not ready.exists(): time.sleep(0.01)\n" + ) + unrelated = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(60)"], start_new_session=True + ) + try: + start = time.monotonic() + with pytest.raises(sql_setup.SetupFailure, match="descendants holding output"): + sql_setup.Commands("").run([sys.executable, "-c", launcher], 5) + assert time.monotonic() - start < 6 + child_pid = int(ready.read_text()) + state = subprocess.run( + ["ps", "-o", "stat=", "-p", str(child_pid)], + capture_output=True, + text=True, + timeout=2, + ).stdout.strip() + assert not state or state.startswith("Z") + assert unrelated.poll() is None + finally: + unrelated.terminate() + unrelated.wait(timeout=5) + + +@pytest.mark.skipif(os.name != "posix", reason="Unix process-group cancellation contract") +@pytest.mark.parametrize("signum", [signal.SIGINT, signal.SIGTERM]) +def test_real_signal_stops_owned_command(tmp_path, signum): + ready = tmp_path / "ready" + child = ( + "import os, pathlib, time; " + f"pathlib.Path({str(ready)!r}).write_text(str(os.getpid())); time.sleep(60)" + ) + script = ( + "import importlib.util, pathlib, signal, sys\n" + f"spec=importlib.util.spec_from_file_location('helper', {str(HELPER)!r})\n" + "m=importlib.util.module_from_spec(spec); sys.modules['helper']=m; spec.loader.exec_module(m)\n" + "def cancel(sig, frame): raise m.Cancelled(sig)\n" + "signal.signal(signal.SIGINT,cancel); signal.signal(signal.SIGTERM,cancel)\n" + "try:\n" + f" m.Commands('').run([sys.executable,'-c',{child!r}],60)\n" + "except m.Cancelled as exc: sys.exit(128+exc.signum)\n" + ) + process = subprocess.Popen([sys.executable, "-c", script]) + try: + deadline = time.monotonic() + 10 + while not ready.exists() and process.poll() is None and time.monotonic() < deadline: + time.sleep(0.01) + assert ready.exists() + child_pid = int(ready.read_text()) + process.send_signal(signum) + assert process.wait(timeout=8) == 128 + signum + with pytest.raises(ProcessLookupError): + os.kill(child_pid, 0) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) From aa6953785477ab633d2cde853b0571d294966a57 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 10:01:20 +0530 Subject: [PATCH 2/6] FIX: Avoid procps dependency in SQL setup tests Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_sql_container_setup.py | 73 ++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/tests/test_sql_container_setup.py b/tests/test_sql_container_setup.py index 0e70ddae..e970f399 100644 --- a/tests/test_sql_container_setup.py +++ b/tests/test_sql_container_setup.py @@ -411,8 +411,57 @@ def cleanup(): assert docker.created == 0 +def _linux_process_state(pid, proc_root=Path("/proc")): + try: + stat = (proc_root / str(pid) / "stat").read_text(encoding="utf-8", errors="replace") + except (FileNotFoundError, ProcessLookupError): + return None + # comm may contain spaces, newlines and parentheses; state follows its last ')'. + comm, separator, fields = stat.rpartition(")") + fields = fields.split() + assert separator and comm.startswith(f"{pid} (") and fields, "Malformed process stat" + assert len(fields[0]) == 1, "Malformed process state" + return fields[0] + + +@pytest.mark.parametrize( + "comm, state", [("worker", "R"), ("odd) (worker", "Z"), ("worker\nwith ) space", "S")] +) +def test_linux_process_state_without_ps(tmp_path, monkeypatch, comm, state): + proc = tmp_path / "123" + proc.mkdir() + (proc / "stat").write_text(f"123 ({comm}) {state} 1 2 3\n", encoding="utf-8") + monkeypatch.setenv("PATH", "") + assert _linux_process_state(123, tmp_path) == state + + +def test_linux_process_state_when_already_reaped(tmp_path): + assert _linux_process_state(123, tmp_path) is None + + +def test_linux_process_state_handles_reaping_during_read(tmp_path, monkeypatch): + def reaped(*args, **kwargs): + raise ProcessLookupError("Process exited during stat read") + + monkeypatch.setattr(Path, "read_text", reaped) + assert _linux_process_state(123, tmp_path) is None + + +def test_linux_process_state_does_not_mask_permission_errors(tmp_path, monkeypatch): + def denied(*args, **kwargs): + raise PermissionError("Process stat is not readable") + + monkeypatch.setattr(Path, "read_text", denied) + with pytest.raises(PermissionError): + _linux_process_state(123, tmp_path) + + @pytest.mark.skipif(os.name != "posix", reason="Unix descendant process-group contract") -def test_exited_launcher_descendant_is_terminated_without_touching_other_groups(tmp_path): +def test_exited_launcher_descendant_is_terminated_without_touching_other_groups( + tmp_path, monkeypatch +): + if sys.platform.startswith("linux"): + monkeypatch.setenv("PATH", "") ready = tmp_path / "descendant" child = ( "import os,pathlib,signal,time; " @@ -434,12 +483,22 @@ def test_exited_launcher_descendant_is_terminated_without_touching_other_groups( sql_setup.Commands("").run([sys.executable, "-c", launcher], 5) assert time.monotonic() - start < 6 child_pid = int(ready.read_text()) - state = subprocess.run( - ["ps", "-o", "stat=", "-p", str(child_pid)], - capture_output=True, - text=True, - timeout=2, - ).stdout.strip() + reaped_deadline = time.monotonic() + 2 + while True: + if sys.platform.startswith("linux"): + state = _linux_process_state(child_pid) + else: + result = subprocess.run( + ["/bin/ps", "-o", "stat=", "-p", str(child_pid)], + capture_output=True, + text=True, + timeout=2, + ) + assert result.returncode in (0, 1) and not result.stderr, result.stderr + state = result.stdout.strip() + if not state or state.startswith("Z") or time.monotonic() >= reaped_deadline: + break + time.sleep(0.01) assert not state or state.startswith("Z") assert unrelated.poll() is None finally: From dc67c26fcd2939a8aedee00426be8207972740e4 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 10:09:00 +0530 Subject: [PATCH 3/6] CHORE: Remove pipeline test files from the PR Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_sql_container_pipeline.py | 171 --------- tests/test_sql_container_setup.py | 541 --------------------------- 2 files changed, 712 deletions(-) delete mode 100644 tests/test_sql_container_pipeline.py delete mode 100644 tests/test_sql_container_setup.py diff --git a/tests/test_sql_container_pipeline.py b/tests/test_sql_container_pipeline.py deleted file mode 100644 index b78a5bcd..00000000 --- a/tests/test_sql_container_pipeline.py +++ /dev/null @@ -1,171 +0,0 @@ -import os -from pathlib import Path -import re -import subprocess -import textwrap - -import pytest - -ROOT = Path(__file__).resolve().parents[1] -PIPELINE = ROOT / "eng" / "pipelines" / "pr-validation-pipeline.yml" -if not PIPELINE.is_file(): - pytest.skip("SQL pipeline contracts require a source checkout", allow_module_level=True) - -JOBS = { - "PytestOnMacOS": ("sqlserver", False), - "PytestOnLinux": ("sqlserver-$(distroName)", True), - "PytestOnLinux_ARM64": ("sqlserver-$(distroName)-$(archName)", True), - "PytestOnLinux_RHEL9": ("sqlserver-rhel9", True), - "PytestOnLinux_RHEL9_ARM64": ("sqlserver-rhel9-arm64", True), - "PytestOnLinux_Alpine": ("sqlserver-alpine", True), - "PytestOnLinux_Alpine_ARM64": ("sqlserver-alpine-arm64", True), - "CodeCoverageReport": ("sqlserver", False), -} - - -def section(name): - text = PIPELINE.read_text(encoding="utf-8") - return re.search(r"^- job: " + name + r"\n.*?(?=^- job: |\Z)", text, re.M | re.S)[0] - - -def script_steps(name): - return re.findall(r"^ - script: \|\n.*?(?=^ - |\Z)", section(name), re.M | re.S) - - -@pytest.mark.parametrize("name, configuration", JOBS.items()) -def test_all_sql_setup_and_cleanup_paths_are_wired(name, configuration): - container, database = configuration - steps = script_steps(name) - setup = next(step for step in steps if "setup_sql_container.py" in step and "--image" in step) - assert container in setup - assert ("--database TestDB" in setup) == database - assert "$(Build.BuildId).$(System.JobId)" in setup - assert "DB_PASSWORD: $(DB_PASSWORD)" in setup - assert "continueOnError" not in setup and "retryCountOnTaskFailure" not in setup - assert "MSSQL_SA_PASSWORD=" not in setup and "-P " not in setup - cleanup = [ - step - for step in steps - if "setup_sql_container.py" in step and "--cleanup" in step and "--image" not in step - ] - assert len(cleanup) == 1 - assert "always()" in cleanup[0] - assert "timeoutInMinutes: 3" in cleanup[0] - assert "test-container" not in cleanup[0] - assert container in cleanup[0] - if name != "PytestOnMacOS": - assert "timeoutInMinutes: 22" in setup - - -def test_azuresql_skips_local_sql_and_preserves_matrix(): - linux = section("PytestOnLinux") - assert "Ubuntu_AzureSQL:" in linux - assert "if ne(variables['AZURE_CONNECTION_STRING'], '')" in linux - setup = next(step for step in script_steps("PytestOnLinux") if "--image" in step) - assert "condition: and(succeeded(), eq(variables['useAzureSQL'], 'false'))" in setup - cleanup = next(step for step in script_steps("PytestOnLinux") if "--cleanup" in step) - assert "condition: and(always(), eq(variables['useAzureSQL'], 'false'))" in cleanup - assert "AZURE_CONNECTION_STRING" in linux - - -def test_no_new_build_test_retries_or_sql_architecture_changes(): - pipeline = PIPELINE.read_text(encoding="utf-8") - assert pipeline.count("retryCountOnTaskFailure:") == 3 - for name in ("PytestOnLinux_ARM64", "PytestOnLinux_RHEL9_ARM64", "PytestOnLinux_Alpine_ARM64"): - steps = [step for step in script_steps(name) if "retryCountOnTaskFailure:" in step] - assert len(steps) == 1 - assert "retryCountOnTaskFailure: 2" in steps[0] - assert "build.sh" in steps[0] - assert "setup_sql_container.py" not in steps[0] and "pytest" not in steps[0] - helper = (ROOT / "eng" / "scripts" / "setup_sql_container.py").read_text(encoding="utf-8") - assert '"linux/amd64"' in helper - assert "linux/arm64" not in helper - assert "Config.Env" not in helper and "prune" not in helper - for name in JOBS: - for step in script_steps(name): - if "python -m pytest" in step: - assert "retryCountOnTaskFailure" not in step - assert "continueOnError" not in step - assert "--cleanup" not in step - - -def test_mac_overlap_and_failure_wait_remain_separate_from_tests(): - mac = section("PytestOnMacOS") - assert "DOCKER_CONTEXT: colima" in mac - assert "timeoutInMinutes: 90" in mac - assert "SQL2022:" in mac and "SQL2025:" in mac - setup = next(step for step in script_steps("PytestOnMacOS") if "--image" in step) - assert '> "$SQL_LOG" 2>&1 &' in setup - assert setup.count("./build.sh") == 1 - assert setup.count("pip install -r requirements.txt") == 1 - assert ( - setup.index("SQL_PID=$!") < setup.index("./build.sh") < setup.rindex('wait "$SQL_PID" ||') - ) - assert 'exit "$SQL_STATUS"' in setup - assert "trap finish_setup EXIT" in setup - assert "pytest" not in setup - assert "/tmp/sql_setup.log" not in setup - - -@pytest.mark.skipif(os.name != "posix", reason="Hosted Unix Bash supervision contract") -@pytest.mark.parametrize("sql_status, build_status", [(0, 0), (17, 0), (0, 27)]) -def test_mac_script_builds_once_and_gates_dependents(tmp_path, sql_status, build_status): - step = next(step for step in script_steps("PytestOnMacOS") if "--image" in step) - script = textwrap.dedent(step.split(" displayName:", 1)[0].split("\n", 1)[1]) - replacements = { - "$(Agent.TempDirectory)": str(tmp_path), - "$(Build.SourcesDirectory)": str(tmp_path), - "$(System.JobId)": "test-job", - "$(Build.BuildId)": "1", - "$(sqlServerImage)": "mcr.microsoft.com/mssql/server:2025-latest", - } - for before, after in replacements.items(): - script = script.replace(before, after) - bindir = tmp_path / "bin" - bindir.mkdir() - events = tmp_path / "events" - builddir = tmp_path / "mssql_python" / "pybind" - builddir.mkdir(parents=True) - build = builddir / "build.sh" - build.write_text('#!/bin/sh\necho build >> "$EVENTS"\nexit "$BUILD_STATUS"\n', encoding="utf-8") - build.chmod(0o700) - for name in ("python", "pip"): - file = bindir / name - file.write_text('#!/bin/sh\necho dependency >> "$EVENTS"\n', encoding="utf-8") - file.chmod(0o700) - python3 = bindir / "python3" - python3.write_text( - "#!/bin/sh\n" - 'case "$*" in\n' - ' *--cleanup*) echo cleanup >> "$EVENTS"; exit 0 ;;\n' - "esac\n" - 'echo sql >> "$EVENTS"\n' - 'exit "$SQL_STATUS_TEST"\n', - encoding="utf-8", - ) - python3.chmod(0o700) - env = { - **os.environ, - "PATH": str(bindir) + os.pathsep + os.environ["PATH"], - "EVENTS": str(events), - "SQL_STATUS_TEST": str(sql_status), - "BUILD_STATUS": str(build_status), - } - script_file = tmp_path / "step.sh" - script_file.write_text(script, encoding="utf-8") - result = subprocess.run( - ["bash", str(script_file)], - cwd=tmp_path, - env=env, - capture_output=True, - text=True, - timeout=15, - ) - actions = events.read_text(encoding="utf-8").splitlines() - assert result.returncode == (build_status or sql_status), result.stdout + result.stderr - assert actions.count("build") == 1 - assert actions.count("dependency") == 2 - assert actions.count("sql") <= 1 - assert actions.count("cleanup") == (1 if result.returncode else 0) - if result.returncode == 0: - assert actions.count("sql") == 1 diff --git a/tests/test_sql_container_setup.py b/tests/test_sql_container_setup.py deleted file mode 100644 index e970f399..00000000 --- a/tests/test_sql_container_setup.py +++ /dev/null @@ -1,541 +0,0 @@ -import importlib.util -import io -import os -from pathlib import Path -import signal -import subprocess -import sys -import time -from types import SimpleNamespace - -import pytest - -HELPER = Path(__file__).resolve().parents[1] / "eng" / "scripts" / "setup_sql_container.py" -if not HELPER.is_file(): - pytest.skip("SQL setup contracts require a source checkout", allow_module_level=True) -spec = importlib.util.spec_from_file_location("sql_container_setup", HELPER) -sql_setup = importlib.util.module_from_spec(spec) -sys.modules[spec.name] = sql_setup -spec.loader.exec_module(sql_setup) - -PASSWORD = "Dummy-Secret-Canary!42" -OWNER = "123.owned-job" -IMAGE_ID = "sha256:" + "a" * 64 - - -class Clock: - def __init__(self): - self.now = 0 - self.sleeps = [] - - def monotonic(self): - return self.now - - def sleep(self, seconds): - self.sleeps.append(seconds) - self.now += seconds - - -class Docker: - def __init__(self, clock, scenario="success"): - self.clock = clock - self.scenario = scenario - self.calls = [] - self.created = 0 - self.identifier = None - self.status = "created" - self.owner = OWNER - - def count(self, command): - return sum(args[0] == command for args, _, _ in self.calls) - - def run(self, args, timeout, *, env): - assert timeout > 0 - args = list(args) - if args[0] == "docker": - args.pop(0) - if args[:2] == ["--context", "colima"]: - args = args[2:] - self.calls.append((args, timeout, env)) - command = args[0] - result = sql_setup.Result - if command == "colima": - return result(0, "") - if command == "info": - return result(1 if self.scenario == "daemon" else 0, "docker status") - if command == "container": - if self.scenario == "lookup-failure": - return result(1, "cannot query daemon") - return result(0, (self.identifier + "\n") if self.identifier else "") - if command == "inspect": - return result(0, f"{self.identifier}|{self.owner}|{self.status}|137|false|{IMAGE_ID}\n") - if command == "pull": - return result(1 if self.scenario == "pull-failure" else 0, "pull result") - if command == "image": - return result(0, f'{IMAGE_ID} ["mcr.microsoft.com/mssql/server@{IMAGE_ID}"]\n') - if command == "create": - self.created += 1 - if self.scenario == "create-once" and self.created == 1: - return result(1, "creation failed") - self.identifier = f"{self.created:064x}" - self.status = "created" - if self.scenario == "create-timeout" and self.created == 1: - self.clock.sleep(timeout) - raise sql_setup.SetupFailure("create timed out after creating the container") - return result(0, self.identifier) - if command == "start": - self.status = "running" - if self.scenario == "start-once" and self.created == 1: - return result(1, "start failed") - if self.scenario == "cancel": - raise sql_setup.Cancelled(signal.SIGTERM) - if self.scenario == "interrupt-exit": - return result(130, "interrupted") - if self.scenario == "dead" or (self.scenario == "dead-once" and self.created == 1): - self.status = "exited" - return result(0, "") - if command == "exec": - if self.scenario == "hung": - self.clock.sleep(timeout) - raise sql_setup.SetupFailure("exec timed out") - if self.scenario == "timeout": - self.clock.sleep(min(5, timeout)) - return result(1, "not ready") - if self.scenario == "no-sqlcmd": - return result(127, "missing sqlcmd") - if self.scenario == "database" and args[-1] == "CREATE DATABASE TestDB": - return result(1, "SQL initialization error") - return result(0, "1") - if command == "logs": - return result(0, f"fatal header\nPASSWORD={PASSWORD}\nend of log\n") - if command == "rm": - assert args[-1] == self.identifier - if self.scenario == "remove-failure": - return result(1, "removal denied") - self.identifier = None - return result(0, "") - raise AssertionError(args) - - -@pytest.fixture -def setup_factory(monkeypatch): - def factory(scenario="success", *, colima=False, database=True): - clock = Clock() - monkeypatch.setattr(sql_setup, "time", clock) - args = SimpleNamespace( - name="sqlserver", - owner=OWNER, - image="mcr.microsoft.com/mssql/server:2025-latest", - colima=colima, - cleanup=False, - database="TestDB" if database else None, - ) - docker = Docker(clock, scenario) - return sql_setup.SqlSetup(args, PASSWORD, docker), docker, clock - - return factory - - -@pytest.mark.parametrize("database", [True, False]) -def test_first_success_and_secret_environment(setup_factory, monkeypatch, capsys, database): - monkeypatch.setenv("MSSQL_SA_PASSWORD", "wrong-inherited-password") - monkeypatch.setenv("SQLCMDPASSWORD", "another-wrong-password") - setup, docker, _ = setup_factory(database=database) - setup.setup() - assert docker.count("create") == docker.count("start") == docker.count("pull") == 1 - assert docker.count("rm") == 0 - assert docker.count("exec") == (2 if database else 1) - for args, timeout, env in docker.calls: - assert PASSWORD not in " ".join(args) - assert "-P" not in args - assert env["MSSQL_SA_PASSWORD"] == env["SQLCMDPASSWORD"] == PASSWORD - assert "DB_CONNECTION_STRING" not in env - if args[0] == "exec": - assert args[1:3] == ["--env", "SQLCMDPASSWORD"] - assert "-b" in args and args[args.index("-l") + 1] == "5" - assert timeout <= 30 - assert "ready on first attempt" in capsys.readouterr().out - - -@pytest.mark.parametrize("scenario", ["create-once", "create-timeout", "start-once", "dead-once"]) -def test_first_failure_recovers_exactly_once(setup_factory, capsys, scenario): - setup, docker, clock = setup_factory(scenario, colima=True) - setup.setup() - assert docker.created == 2 - assert docker.count("pull") == docker.count("colima") == 1 - assert clock.sleeps.count(5) == 1 - assert docker.count("exec") == 2 - assert docker.identifier == f"{2:064x}" - if scenario != "create-once": - actions = [args[0] for args, _, _ in docker.calls] - assert actions.index("logs") < actions.index("rm") < len(actions) - 1 - assert docker.count("rm") == 1 - out = capsys.readouterr().out - assert PASSWORD not in out - assert "attempt 1/2" in out and "attempt 2/2" in out - assert "recovered on second attempt" in out - - -@pytest.mark.parametrize("scenario", ["dead", "database", "pull-failure", "hung", "timeout"]) -def test_permanent_failure_stops_after_two_attempts(setup_factory, scenario, capsys): - setup, docker, clock = setup_factory(scenario) - with pytest.raises(sql_setup.SetupFailure, match="no further attempts"): - setup.setup() - assert docker.count("pull") == (2 if scenario == "pull-failure" else 1) - assert docker.created == (0 if scenario == "pull-failure" else 2) - assert docker.identifier is None - assert docker.count("rm") == docker.created - assert clock.sleeps.count(5) >= 1 - assert clock.now <= 1260 - if scenario == "dead": - assert docker.count("exec") == 0 - assert clock.now == 5 - if scenario == "timeout": - assert 240 <= clock.now <= 246 - assert "recovered" not in capsys.readouterr().out - - -@pytest.mark.parametrize("status", ["running", "exited"]) -def test_preexisting_owned_container_is_not_accepted(setup_factory, status): - setup, docker, _ = setup_factory() - old_id = "e" * 64 - docker.identifier, docker.status = old_id, status - setup.setup() - actions = [args[0] for args, _, _ in docker.calls] - assert actions.index("logs") < actions.index("rm") < actions.index("create") - assert docker.created == 1 - assert docker.identifier != old_id - assert next(args[-1] for args, _, _ in docker.calls if args[0] == "rm") == old_id - - -@pytest.mark.parametrize("scenario", ["daemon", "lookup-failure", "remove-failure"]) -def test_unsafe_cleanup_or_daemon_failure_is_terminal(setup_factory, scenario): - setup, docker, _ = setup_factory(scenario) - if scenario == "remove-failure": - docker.identifier = "e" * 64 - with pytest.raises(sql_setup.SetupFailure): - setup.setup() - assert docker.created == 0 - assert docker.count("pull") == 0 - - -@pytest.mark.parametrize("owner", ["", "another-job"]) -def test_foreign_container_is_never_read_removed_or_reused(setup_factory, owner): - setup, docker, _ = setup_factory() - docker.identifier, docker.owner = "e" * 64, owner - with pytest.raises(sql_setup.SetupFailure): - setup.setup() - assert docker.count("logs") == docker.count("rm") == docker.created == 0 - - -def test_missing_sqlcmd_is_not_retried(setup_factory): - setup, docker, _ = setup_factory("no-sqlcmd") - with pytest.raises(sql_setup.SetupFailure): - setup.setup() - assert docker.created == docker.count("rm") == 1 - - -@pytest.mark.parametrize("scenario, code", [("cancel", 143), ("interrupt-exit", 130)]) -def test_cancellation_propagates_and_cleans_without_retry( - setup_factory, monkeypatch, scenario, code -): - setup, docker, _ = setup_factory(scenario) - monkeypatch.setenv("DB_PASSWORD", PASSWORD) - monkeypatch.setattr(sql_setup, "SqlSetup", lambda *args: setup) - monkeypatch.setattr(sql_setup.signal, "signal", lambda *args: None) - assert ( - sql_setup.main(["--name", "sqlserver", "--owner", OWNER, "--image", setup.args.image]) - == code - ) - assert docker.created == docker.count("rm") == 1 - assert docker.identifier is None - - -@pytest.mark.parametrize("value", ["", "two\nlines"]) -def test_bad_secret_is_terminal_before_docker(monkeypatch, value): - monkeypatch.setenv("DB_PASSWORD", value) - monkeypatch.setattr(sql_setup.signal, "signal", lambda *args: None) - assert ( - sql_setup.main( - [ - "--name", - "sqlserver", - "--owner", - OWNER, - "--image", - "mcr.microsoft.com/mssql/server:2025-latest", - ] - ) - == 1 - ) - - -def test_cleanup_absent_container_is_success(setup_factory): - setup, docker, _ = setup_factory() - setup.args.cleanup = True - setup.setup() - assert docker.count("info") == 1 - assert docker.created == docker.count("rm") == 0 - - -def test_stream_capture_redacts_across_chunks_and_retains_crash_header(): - capture = sql_setup.SafeCapture(PASSWORD, limit=1024) - content = "FATAL HEADER\n" + "a" * 4068 + PASSWORD + "\n" + ("line\n" * 1000) - content += "x" * 9000 + PASSWORD + "\nEND\n" - capture.read(io.BytesIO(content.encode())) - out = capture.output() - assert out.startswith("FATAL HEADER") - assert out.endswith("END\n") - assert PASSWORD not in out - assert "overlong diagnostic line omitted" in out - assert "truncated" in out - assert len(out) < 2200 - - -def test_real_command_timeout_is_bounded_and_redacted(): - start = time.monotonic() - with pytest.raises(sql_setup.SetupFailure, match="timed out") as error: - sql_setup.Commands(PASSWORD).run( - [ - sys.executable, - "-c", - "import os,signal,time; signal.signal(signal.SIGTERM,signal.SIG_IGN); " - "print(os.environ['SQLCMDPASSWORD'], flush=True); time.sleep(60)", - ], - 0.5, - env={**os.environ, "SQLCMDPASSWORD": PASSWORD}, - ) - assert time.monotonic() - start < 1.5 - assert PASSWORD not in str(error.value) - assert "[REDACTED]" in str(error.value) - - -def test_missing_command_is_an_explicit_terminal_failure(tmp_path): - with pytest.raises(sql_setup.SetupFailure, match="Cannot launch") as error: - sql_setup.Commands(PASSWORD).run([str(tmp_path / "missing-docker-command")], 1) - assert error.value.retryable is False - - -def test_cancellation_during_backoff_does_not_start_second_attempt(setup_factory): - setup, docker, clock = setup_factory("dead") - - def interrupt(_seconds): - raise sql_setup.Cancelled(signal.SIGINT) - - clock.sleep = interrupt - with pytest.raises(sql_setup.Cancelled): - setup.setup() - assert docker.created == docker.count("rm") == 1 - assert docker.identifier is None - - -def test_commands_clamp_to_phase_and_total_deadlines(setup_factory): - setup, docker, clock = setup_factory() - setup.phase_deadline = 3 - setup.docker_command("info", timeout=15) - assert docker.calls[-1][1] == 3 - setup.deadline = 2 - setup.docker_command("info", timeout=15) - assert docker.calls[-1][1] == 2 - clock.now = 2 - count = len(docker.calls) - with pytest.raises(sql_setup.SetupFailure, match="deadline exhausted"): - setup.docker_command("info") - assert len(docker.calls) == count - - -def test_cleanup_deadline_exhaustion_does_not_claim_removal(setup_factory, capsys): - setup, docker, clock = setup_factory() - docker.identifier = "e" * 64 - setup.deadline = 10 - original = docker.run - - def slow_lookup(args, timeout, *, env): - result = original(args, timeout, env=env) - clock.sleep(min(6, timeout)) - return result - - docker.run = slow_lookup - with pytest.raises(sql_setup.SetupFailure, match="deadline exhausted"): - setup.cleanup() - assert clock.now == 10 - assert docker.count("rm") == 0 - assert docker.identifier is not None - assert "cleanup complete" not in capsys.readouterr().out - - -def test_cleanup_only_has_its_own_global_deadline(setup_factory): - setup, docker, clock = setup_factory() - setup.args.cleanup = True - cleanup = sql_setup.SqlSetup(setup.args, PASSWORD, docker) - assert cleanup.deadline - clock.now == 115 - - -@pytest.mark.parametrize("colima, cap, total", [(False, 600, 1260), (True, 900, 2460)]) -def test_total_attempt_budget_reserves_cleanup(setup_factory, colima, cap, total): - setup, docker, clock = setup_factory(colima=colima) - assert setup.deadline == total - attempts = [] - cleanups = [] - - def preflight(): - clock.sleep((600 if colima else 0) + 15) - - original = docker.run - - def bounded_info(args, timeout, *, env): - result = original(args, timeout, env=env) - clock.sleep(timeout) - return result - - def exhaust_attempt(): - remaining = setup.phase_deadline - clock.now - attempts.append(remaining) - clock.sleep(remaining) - raise sql_setup.SetupFailure("attempt deadline exhausted") - - def cleanup(): - cleanups.append(setup.phase_deadline - clock.now) - clock.sleep(100) - - setup.preflight = preflight - docker.run = bounded_info - setup.attempt = exhaust_attempt - setup.cleanup = cleanup - with pytest.raises(sql_setup.SetupFailure, match="no further attempts"): - setup.setup() - assert attempts == [cap - 100, cap - 100] - assert cleanups == [100, 100] - assert clock.now == (600 if colima else 0) + 2 * cap + 35 - assert clock.now <= total - assert docker.created == 0 - - -def _linux_process_state(pid, proc_root=Path("/proc")): - try: - stat = (proc_root / str(pid) / "stat").read_text(encoding="utf-8", errors="replace") - except (FileNotFoundError, ProcessLookupError): - return None - # comm may contain spaces, newlines and parentheses; state follows its last ')'. - comm, separator, fields = stat.rpartition(")") - fields = fields.split() - assert separator and comm.startswith(f"{pid} (") and fields, "Malformed process stat" - assert len(fields[0]) == 1, "Malformed process state" - return fields[0] - - -@pytest.mark.parametrize( - "comm, state", [("worker", "R"), ("odd) (worker", "Z"), ("worker\nwith ) space", "S")] -) -def test_linux_process_state_without_ps(tmp_path, monkeypatch, comm, state): - proc = tmp_path / "123" - proc.mkdir() - (proc / "stat").write_text(f"123 ({comm}) {state} 1 2 3\n", encoding="utf-8") - monkeypatch.setenv("PATH", "") - assert _linux_process_state(123, tmp_path) == state - - -def test_linux_process_state_when_already_reaped(tmp_path): - assert _linux_process_state(123, tmp_path) is None - - -def test_linux_process_state_handles_reaping_during_read(tmp_path, monkeypatch): - def reaped(*args, **kwargs): - raise ProcessLookupError("Process exited during stat read") - - monkeypatch.setattr(Path, "read_text", reaped) - assert _linux_process_state(123, tmp_path) is None - - -def test_linux_process_state_does_not_mask_permission_errors(tmp_path, monkeypatch): - def denied(*args, **kwargs): - raise PermissionError("Process stat is not readable") - - monkeypatch.setattr(Path, "read_text", denied) - with pytest.raises(PermissionError): - _linux_process_state(123, tmp_path) - - -@pytest.mark.skipif(os.name != "posix", reason="Unix descendant process-group contract") -def test_exited_launcher_descendant_is_terminated_without_touching_other_groups( - tmp_path, monkeypatch -): - if sys.platform.startswith("linux"): - monkeypatch.setenv("PATH", "") - ready = tmp_path / "descendant" - child = ( - "import os,pathlib,signal,time; " - "signal.signal(signal.SIGTERM,signal.SIG_IGN); " - f"pathlib.Path({str(ready)!r}).write_text(str(os.getpid())); time.sleep(60)" - ) - launcher = ( - "import subprocess,sys,time,pathlib; " - f"subprocess.Popen([sys.executable,'-c',{child!r}]); " - f"ready=pathlib.Path({str(ready)!r}); " - "\nwhile not ready.exists(): time.sleep(0.01)\n" - ) - unrelated = subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(60)"], start_new_session=True - ) - try: - start = time.monotonic() - with pytest.raises(sql_setup.SetupFailure, match="descendants holding output"): - sql_setup.Commands("").run([sys.executable, "-c", launcher], 5) - assert time.monotonic() - start < 6 - child_pid = int(ready.read_text()) - reaped_deadline = time.monotonic() + 2 - while True: - if sys.platform.startswith("linux"): - state = _linux_process_state(child_pid) - else: - result = subprocess.run( - ["/bin/ps", "-o", "stat=", "-p", str(child_pid)], - capture_output=True, - text=True, - timeout=2, - ) - assert result.returncode in (0, 1) and not result.stderr, result.stderr - state = result.stdout.strip() - if not state or state.startswith("Z") or time.monotonic() >= reaped_deadline: - break - time.sleep(0.01) - assert not state or state.startswith("Z") - assert unrelated.poll() is None - finally: - unrelated.terminate() - unrelated.wait(timeout=5) - - -@pytest.mark.skipif(os.name != "posix", reason="Unix process-group cancellation contract") -@pytest.mark.parametrize("signum", [signal.SIGINT, signal.SIGTERM]) -def test_real_signal_stops_owned_command(tmp_path, signum): - ready = tmp_path / "ready" - child = ( - "import os, pathlib, time; " - f"pathlib.Path({str(ready)!r}).write_text(str(os.getpid())); time.sleep(60)" - ) - script = ( - "import importlib.util, pathlib, signal, sys\n" - f"spec=importlib.util.spec_from_file_location('helper', {str(HELPER)!r})\n" - "m=importlib.util.module_from_spec(spec); sys.modules['helper']=m; spec.loader.exec_module(m)\n" - "def cancel(sig, frame): raise m.Cancelled(sig)\n" - "signal.signal(signal.SIGINT,cancel); signal.signal(signal.SIGTERM,cancel)\n" - "try:\n" - f" m.Commands('').run([sys.executable,'-c',{child!r}],60)\n" - "except m.Cancelled as exc: sys.exit(128+exc.signum)\n" - ) - process = subprocess.Popen([sys.executable, "-c", script]) - try: - deadline = time.monotonic() + 10 - while not ready.exists() and process.poll() is None and time.monotonic() < deadline: - time.sleep(0.01) - assert ready.exists() - child_pid = int(ready.read_text()) - process.send_signal(signum) - assert process.wait(timeout=8) == 128 + signum - with pytest.raises(ProcessLookupError): - os.kill(child_pid, 0) - finally: - if process.poll() is None: - process.kill() - process.wait(timeout=5) From 84f924838f23644e8ee2af376310cb0896aac2fa Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 10:32:46 +0530 Subject: [PATCH 4/6] FIX: Harden bounded SQL setup recovery and diagnostics Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/scripts/setup_sql_container.py | 216 ++++++++++++++++++++--------- 1 file changed, 149 insertions(+), 67 deletions(-) diff --git a/eng/scripts/setup_sql_container.py b/eng/scripts/setup_sql_container.py index 4ec93203..f8c59ef3 100644 --- a/eng/scripts/setup_sql_container.py +++ b/eng/scripts/setup_sql_container.py @@ -6,17 +6,28 @@ test legs, --colima on macOS, and --cleanup for the always-running final step. Cleanup refuses containers without the matching owner label. -Only SQL pull/create/start/readiness/database setup is retried. Colima starts -once. Configuration, ownership, preflight and cleanup failures are terminal. +Only SQL lookup/pull/create/start/readiness/database setup is retried. Colima +starts once. The owned-container lookup also checks Docker availability, so +transient read-only failures can consume the same two attempts without creating +or removing anything. Missing tools, invalid configuration/lookup results, +ownership conflicts and unsafe cleanup are terminal. Linux/macOS attempts are bounded at 600/900 seconds including diagnostics and cleanup; entire invocations at 1260/2460 seconds including macOS VM startup. Command budgets include termination/output-drain grace. Cleanup-only uses a 115-second deadline. OS scheduling/host loss can defeat cooperative deadlines; the pipeline also enforces outer task limits. -Diagnostics retain redacted beginning/end excerpts, not dumps or full inspect -output. Timeout/cancellation stops only subprocesses created by this helper. -Hard host loss can still prevent cleanup; the pipeline also runs --cleanup. +Readiness polls for 120/180 seconds (Linux/macOS), then performs one final +state/query check bounded at 45 seconds, for at most 165/225 seconds total. +Both are clamped to the remaining attempt/global budget, preserving cleanup. + +Docker diagnostics request only the last 30 minutes and at most 5000 lines, +within 20 seconds. Redacted output retains 32768-character head/tail excerpts +plus at most 8192 characters of fatal/Reason context, not complete history. +No dumps or full inspect output are collected. Timeout/cancellation stops only +subprocess groups created by this helper, with at most four seconds of teardown +within the original command budget. Hard host loss can still prevent cleanup; +the pipeline also runs --cleanup. """ import argparse @@ -45,6 +56,10 @@ def __init__(self, message, *, retryable=True): self.retryable = retryable +class SetupTimeout(SetupFailure): + pass + + class Cancelled(BaseException): def __init__(self, signum): self.signum = signum @@ -70,9 +85,17 @@ def __init__(self, password, limit=32768): self.head = "" self.tail = "" self.total = 0 + self.fatal_context = "" + self.context_remaining = 0 def add(self, line): safe = redact(line, self.password) + if re.search(r"\bfatal\b|\bReason:", safe, re.IGNORECASE): + self.context_remaining = 12 + if self.context_remaining: + remaining = min(8192, self.limit // 4) - len(self.fatal_context) + self.fatal_context += safe[:remaining] + self.context_remaining -= 1 self.total += len(safe) remaining = self.limit - len(self.head) self.head += safe[:remaining] @@ -104,7 +127,12 @@ def read(self, stream): def output(self): marker = "\n[diagnostic output truncated]\n" if self.total > 2 * self.limit else "" - return self.head + marker + self.tail + context = ( + "\n[fatal/Reason context from retrieved output]\n" + self.fatal_context + if marker and self.fatal_context + else "" + ) + return self.head + marker + context + self.tail @dataclass @@ -147,9 +175,10 @@ def stop(process, deadline): def run(self, args, timeout, *, env=None): if timeout <= 0: - raise SetupFailure("SQL setup deadline exhausted") + raise SetupTimeout("SQL setup deadline exhausted") deadline = time.monotonic() + timeout grace = min(4, timeout / 2) + work_deadline = deadline - grace capture = SafeCapture(self.password) try: process = subprocess.Popen( @@ -161,37 +190,58 @@ def run(self, args, timeout, *, env=None): ) except OSError: raise SetupFailure("Cannot launch required setup command", retryable=False) from None - reader = threading.Thread(target=capture.read, args=(process.stdout,), daemon=True) - reader.start() + output_done = threading.Event() + read_errors = ["Setup output reader did not complete"] + + def read_output(): + try: + capture.read(process.stdout) + except OSError: + read_errors[:] = ["Cannot read setup command output"] + else: + read_errors.clear() + finally: + output_done.set() + + reader = threading.Thread(target=read_output, daemon=True) + reader_started = False timed_out = False - descendant_output = False + drain_timed_out = False reaped = True try: try: - process.wait(timeout=max(0, deadline - time.monotonic() - grace)) + reader.start() + reader_started = True + except RuntimeError: + raise SetupFailure("Cannot start setup output reader", retryable=False) from None + try: + process.wait(timeout=max(0, work_deadline - time.monotonic())) except subprocess.TimeoutExpired: timed_out = True + if not timed_out: + drain_timed_out = not output_done.wait( + timeout=max(0, work_deadline - time.monotonic()) + ) finally: - if process.poll() is None: - reaped = self.stop(process, deadline) - else: - reader.join(timeout=max(0, min(0.2, (deadline - time.monotonic()) / 4))) - if reader.is_alive(): - descendant_output = True - reaped = self.stop(process, deadline) - reader.join(timeout=max(0, deadline - time.monotonic())) - if not reader.is_alive(): + teardown_deadline = min(deadline, time.monotonic() + 4) + if process.poll() is None or not output_done.is_set() or read_errors: + reaped = self.stop(process, teardown_deadline) + if reader_started: + output_done.wait(timeout=max(0, teardown_deadline - time.monotonic())) + if not reader_started or output_done.is_set(): process.stdout.close() - if not reaped or reader.is_alive(): + if not reaped or (reader_started and not output_done.is_set()): print("[sql] Command teardown incomplete within its deadline", file=sys.stderr) if not reaped: raise SetupFailure("Setup command could not be reaped", retryable=False) - if reader.is_alive(): - raise SetupFailure("Setup command output did not close", retryable=False) - if descendant_output: - raise SetupFailure("Setup command left descendants holding output", retryable=False) + if read_errors: + raise SetupFailure(read_errors[0], retryable=False) + if drain_timed_out or not output_done.is_set(): + raise SetupFailure( + "Setup command output drain timed out\n" + capture.output(), retryable=False + ) if timed_out: - raise SetupFailure("Setup command timed out\n" + capture.output()) + raise SetupTimeout("Setup command timed out\n" + capture.output()) return Result(process.returncode, capture.output()) @@ -212,6 +262,7 @@ def __init__(self, args, password, commands=None): self.deadline = time.monotonic() + (115 if args.cleanup else 2460 if args.colima else 1260) self.phase_deadline = self.deadline self.container = None + self.creation_requested = False self.image_id = None self.docker = ["docker"] + (["--context", "colima"] if args.colima else []) self.env = os.environ.copy() @@ -226,7 +277,7 @@ def command(self, args, timeout=15, *, check=True, deadline=None): end = min(self.deadline, self.phase_deadline if deadline is None else deadline) remaining = min(timeout, end - time.monotonic()) if remaining <= 0: - raise SetupFailure("SQL setup deadline exhausted") + raise SetupTimeout("SQL setup deadline exhausted") result = self.commands.run(args, remaining, env=self.env) if result.returncode in (-signal.SIGINT, -signal.SIGTERM, 130, 143): raise Cancelled( @@ -246,7 +297,7 @@ def find_owned(self, *, deadline=None): "--all", "--no-trunc", "--filter", - f"name=^/{self.args.name}$", + f"name=^/{re.escape(self.args.name)}$", "--format", "{{.ID}}", deadline=deadline, @@ -273,11 +324,23 @@ def diagnostics(self, container, *, deadline): f"Container {container.identifier}: status={container.status} " f"exit={container.exit_code} OOMKilled={container.oom_killed} image={container.image}" ) + self.log( + "Container log excerpts (last 30m, max 5000 lines; may omit older history; " + "retained output may be truncated):" + ) try: result = self.docker_command( - "logs", container.identifier, timeout=20, check=False, deadline=deadline + "logs", + "--since", + "30m", + "--tail", + "5000", + container.identifier, + timeout=20, + check=False, + deadline=deadline, ) - self.log("Container log excerpts:\n" + result.output) + self.log(result.output) if result.returncode: self.log(f"Container logs unavailable (exit {result.returncode})") except SetupFailure as exc: @@ -304,11 +367,10 @@ def cleanup(self, *, evidence=True): ) self.remove(container, deadline=self.phase_deadline) - def preflight(self): + def prepare_runtime(self): if self.args.colima and not self.args.cleanup: self.log("Starting Colima once (outside SQL retry)") self.command(["colima", "start", "--cpu", "4", "--memory", "8", "--disk", "50"], 600) - self.docker_command("info", "--format", "{{.ServerVersion}}") def acquire_image(self): if self.image_id is not None: @@ -360,16 +422,55 @@ def sql(self, query, *, timeout=15, query_timeout=5, deadline=None): deadline=deadline, ) + def readiness_probe(self, deadline): + current = self.find_owned(deadline=deadline) + if current is None or current.identifier != self.container.identifier: + raise SetupFailure("SQL container disappeared or was replaced", retryable=False) + if current.status != "running": + raise SetupFailure( + f"SQL container exited before readiness (status={current.status}, " + f"exit={current.exit_code}, OOMKilled={current.oom_killed})" + ) + probe = self.sql("SELECT 1", deadline=deadline) + if probe.returncode in (126, 127): + raise SetupFailure("Required sqlcmd executable is unavailable", retryable=False) + return probe + + def wait_ready(self): + polling_deadline = min( + self.deadline, + self.phase_deadline, + time.monotonic() + (180 if self.args.colima else 120), + ) + while time.monotonic() < polling_deadline: + try: + probe = self.readiness_probe(polling_deadline) + except SetupTimeout as exc: + self.log(f"Readiness probe timed out: {exc}") + else: + if probe.returncode == 0: + return + time.sleep(max(0, min(2, polling_deadline - time.monotonic()))) + final_deadline = min(self.deadline, self.phase_deadline, polling_deadline + 45) + if time.monotonic() >= final_deadline: + raise SetupTimeout("SQL readiness budget exhausted before final probe") + self.log("Polling window ended; performing one final bounded SQL readiness check") + probe = self.readiness_probe(final_deadline) + if probe.returncode != 0: + raise SetupFailure("SQL final readiness check failed\n" + probe.output) + def attempt(self): stale = self.find_owned() if stale is not None: self.log("Removing pre-existing same-job container before fresh setup") self.diagnostics(stale, deadline=min(self.phase_deadline, time.monotonic() + 20)) try: - self.remove(stale, deadline=min(self.phase_deadline, time.monotonic() + 30)) + self.remove(stale, deadline=self.phase_deadline) except SetupFailure as exc: raise SetupFailure(str(exc), retryable=False) from None self.acquire_image() + # The daemon may create the container even when the CLI times out. + self.creation_requested = True self.docker_command( "create", "--name", @@ -391,39 +492,20 @@ def attempt(self): if self.container is None: raise SetupFailure("Created SQL container was not found") self.docker_command("start", self.container.identifier, timeout=30) - ready_deadline = min( - self.phase_deadline, time.monotonic() + (180 if self.args.colima else 120) - ) - last_output = "" - while time.monotonic() < ready_deadline: - current = self.find_owned(deadline=ready_deadline) - if current is None or current.identifier != self.container.identifier: - raise SetupFailure("SQL container disappeared or was replaced", retryable=False) - if current.status != "running": - raise SetupFailure( - f"SQL container exited before readiness (status={current.status}, " - f"exit={current.exit_code}, OOMKilled={current.oom_killed})" - ) - probe = self.sql("SELECT 1", deadline=ready_deadline) - if probe.returncode == 0: - if self.args.database: - result = self.sql("CREATE DATABASE TestDB", timeout=30, query_timeout=15) - if result.returncode != 0: - raise SetupFailure("TestDB initialization failed\n" + result.output) - return - if probe.returncode in (126, 127): - raise SetupFailure("Required sqlcmd executable is unavailable", retryable=False) - last_output = probe.output - time.sleep(max(0, min(2, ready_deadline - time.monotonic()))) - raise SetupFailure("SQL readiness deadline exhausted\n" + last_output) + self.wait_ready() + if self.args.database: + result = self.sql("CREATE DATABASE TestDB", timeout=30, query_timeout=15) + if result.returncode != 0: + raise SetupFailure("TestDB initialization failed\n" + result.output) def setup(self): - self.preflight() + self.prepare_runtime() if self.args.cleanup: self.cleanup(evidence=False) self.log("Owned SQL container cleanup complete (or already absent)") return for number in (1, 2): + self.creation_requested = False self.log(f"SQL setup attempt {number}/2") attempt_end = min( self.deadline, @@ -435,16 +517,16 @@ def setup(self): except SetupFailure as exc: self.log(f"Attempt {number}/2 failed: {exc}") self.phase_deadline = attempt_end - try: - self.cleanup() - except SetupFailure as cleanup_error: - raise SetupFailure( - f"Cannot safely recover/clean up: {cleanup_error}", retryable=False - ) from None + if self.creation_requested: + try: + self.cleanup() + except SetupFailure as cleanup_error: + raise SetupFailure( + f"Cannot safely recover/clean up: {cleanup_error}", retryable=False + ) from None if not exc.retryable or number == 2: raise SetupFailure("SQL setup failed; no further attempts", retryable=False) self.phase_deadline = self.deadline - self.docker_command("info", "--format", "{{.ServerVersion}}") if self.deadline - time.monotonic() < 5: raise SetupFailure("SQL setup deadline exhausted before retry", retryable=False) self.log("Retrying SQL setup only after 5 seconds") @@ -500,7 +582,7 @@ def cancel(signum, _frame): print("[sql] Setup cancelled; no retry", flush=True) if setup is not None: try: - setup.cleanup() + setup.cleanup(evidence=not setup.args.cleanup) except SetupFailure as cleanup_error: setup.log(f"Cancellation cleanup failed: {cleanup_error}") return 128 + exc.signum From ac1ab5856282f6861e75bea4757a51c20b964e32 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 12:52:24 +0530 Subject: [PATCH 5/6] FIX: Preserve Colima daemon launcher completion Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/scripts/setup_sql_container.py | 340 ++++++++++++++++++++++++----- 1 file changed, 285 insertions(+), 55 deletions(-) diff --git a/eng/scripts/setup_sql_container.py b/eng/scripts/setup_sql_container.py index f8c59ef3..2be190e6 100644 --- a/eng/scripts/setup_sql_container.py +++ b/eng/scripts/setup_sql_container.py @@ -24,21 +24,37 @@ Docker diagnostics request only the last 30 minutes and at most 5000 lines, within 20 seconds. Redacted output retains 32768-character head/tail excerpts plus at most 8192 characters of fatal/Reason context, not complete history. -No dumps or full inspect output are collected. Timeout/cancellation stops only -subprocess groups created by this helper, with at most four seconds of teardown -within the original command budget. Hard host loss can still prevent cleanup; -the pipeline also runs --cleanup. +No dumps or full inspect output are collected. Timeout/cancellation targets only +each command's original group and still-owned unreaped child, with at most four +seconds of teardown within the original command budget. Hard host loss can +still prevent cleanup; the pipeline also runs --cleanup. +Permission or identity failures are reported as incomplete, nonretryable +teardown without replacing the original failure or cancellation. Group checks +are not atomic with signalling: departed groups are not followed, and a reaped +PID observed again is not targeted. A missing leader alone does not establish +that its original group is gone. + +Colima is a daemon launcher, not a finite-output command. Its direct exit is +captured through a private anonymous temporary file, without requiring EOF or +stopping successful background processes. Only a fixed-size startup snapshot +is read/redacted; Docker/SQL readiness is checked separately. The parent closes +its descriptor on every outcome. Daemons can retain the unlinked backing inode +until their descriptors close or the hosted job ends: raw backing storage is +not hard-capped. This contract is intended for job-scoped hosted Colima startup, +not a general-purpose persistent daemon logging service. """ import argparse import codecs from dataclasses import dataclass +import io import json import os import re import signal import subprocess import sys +import tempfile import threading import time @@ -105,8 +121,9 @@ def read(self, stream): decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") pending = "" omitted = False + read = getattr(stream, "read1", stream.read) while True: - chunk = stream.read(4096) + chunk = read(4096) text = decoder.decode(chunk, final=not chunk) for part in text.splitlines(keepends=True): pending += part @@ -141,39 +158,150 @@ class Result: output: str +@dataclass +class StopResult: + reaped: bool + errors: list[str] + + class Commands: def __init__(self, password): self.password = password @staticmethod def stop(process, deadline): - # start_new_session gives each command its own group. The launcher may - # already be reaped while a descendant still holds its output pipe. - try: - if os.name == "posix": - os.killpg(process.pid, signal.SIGTERM) - elif process.poll() is None: - process.terminate() - except ProcessLookupError: - pass + errors = [] + group_blocked = False + + def signal_child(name): + # Popen rechecks its own unreaped child before signalling its PID. + if process.poll() is not None: + return + try: + if name == "TERM": + process.terminate() + else: + process.kill() + except ProcessLookupError: + pass + except PermissionError as exc: + errors.append( + f"Permission denied sending SIG{name} to owned child " + f"{process.pid} (errno={exc.errno})" + ) + + def signal_owned(name): + nonlocal group_blocked + if os.name != "posix": + signal_child(name) + return + if group_blocked: + signal_child(name) + return + reaped = process.poll() is not None + try: + group = os.getpgid(process.pid) + except ProcessLookupError: + # Descendants may still hold the original group and stdout. + pass + except PermissionError as exc: + errors.append( + f"Permission denied checking original group {process.pid} (errno={exc.errno})" + ) + group_blocked = True + else: + if reaped: + errors.append(f"Reaped PID {process.pid} is present again; group not signalled") + group_blocked = True + elif group != process.pid: + errors.append( + f"Owned child {process.pid} left its original group; " + f"group {group} not followed" + ) + group_blocked = True + if group_blocked: + signal_child(name) + return + try: + os.killpg(process.pid, getattr(signal, "SIG" + name)) + except ProcessLookupError: + signal_child(name) + except PermissionError as exc: + errors.append( + f"Permission denied sending SIG{name} to original group " + f"{process.pid} (errno={exc.errno})" + ) + group_blocked = True + signal_child(name) + + signal_owned("TERM") try: process.wait(timeout=max(0, min(1, (deadline - time.monotonic()) / 2))) except subprocess.TimeoutExpired: pass - try: - if os.name == "posix": - os.killpg(process.pid, signal.SIGKILL) - elif process.poll() is None: - process.kill() - except ProcessLookupError: - pass + signal_owned("KILL") try: process.wait(timeout=max(0, min(1, deadline - time.monotonic()))) except subprocess.TimeoutExpired: - return False - return True + return StopResult(False, errors) + return StopResult(True, errors) + + @staticmethod + def snapshot(output, capture): + output.flush() + size = os.fstat(output.fileno()).st_size + window = 65536 + ranges = [(0, size)] if size <= 2 * window else [(0, window), (size - window, window)] + capture.add( + "[Colima startup snapshot: at most 64KiB head and 64KiB tail; " + "partial boundary lines and later background output omitted]\n" + ) + for index, (offset, length) in enumerate(ranges): + if index: + capture.context_remaining = 0 + capture.add("[Colima startup snapshot truncated; middle output omitted]\n") + if hasattr(os, "pread"): + data = os.pread(output.fileno(), length, offset) + else: + output.seek(offset) + data = output.read(length) + if offset: + data = data.partition(b"\n")[2] + if data and not data.endswith(b"\n"): + data = data[: data.rfind(b"\n") + 1] + capture.read(io.BytesIO(data)) + + def run_launcher(self, args, timeout, *, env=None): + deadline = time.monotonic() + timeout + try: + output = tempfile.TemporaryFile(mode="a+b") + except OSError as exc: + raise SetupFailure( + f"Cannot create private Colima output capture (errno={exc.errno})", retryable=False + ) from None + primary = None + try: + return self._run(args, deadline - time.monotonic(), env=env, output_file=output) + except (SetupFailure, Cancelled) as exc: + primary = exc + raise + finally: + try: + output.close() + except OSError as exc: + message = f"Cannot close private Colima output capture (errno={exc.errno})" + if primary is None: + raise SetupFailure(message, retryable=False) from None + if isinstance(primary, SetupFailure): + primary.retryable = False + primary.args = (str(primary) + "\n" + message,) + else: + print("[sql] " + message, file=sys.stderr, flush=True) def run(self, args, timeout, *, env=None): + return self._run(args, timeout, env=env) + + def _run(self, args, timeout, *, env=None, output_file=None): if timeout <= 0: raise SetupTimeout("SQL setup deadline exhausted") deadline = time.monotonic() + timeout @@ -183,7 +311,7 @@ def run(self, args, timeout, *, env=None): try: process = subprocess.Popen( args, - stdout=subprocess.PIPE, + stdout=subprocess.PIPE if output_file is None else output_file, stderr=subprocess.STDOUT, env=env, start_new_session=os.name == "posix", @@ -191,7 +319,7 @@ def run(self, args, timeout, *, env=None): except OSError: raise SetupFailure("Cannot launch required setup command", retryable=False) from None output_done = threading.Event() - read_errors = ["Setup output reader did not complete"] + read_errors = ["Setup output reader did not complete"] if output_file is None else [] def read_output(): try: @@ -203,45 +331,98 @@ def read_output(): finally: output_done.set() - reader = threading.Thread(target=read_output, daemon=True) + reader = threading.Thread(target=read_output, daemon=True) if output_file is None else None reader_started = False timed_out = False drain_timed_out = False - reaped = True + stopped = StopResult(True, []) + pending_error = None + problems = [] + + def failure_message(reason): + message = reason + if problems: + message += "\nCommand teardown incomplete:\n" + "\n".join(problems) + message += "\nCaptured command output (may be incomplete):" + message += "\n" + (capture.output() or "[no completed output lines captured]") + return redact(message, self.password) + try: - try: - reader.start() - reader_started = True - except RuntimeError: - raise SetupFailure("Cannot start setup output reader", retryable=False) from None + if reader is not None: + try: + reader.start() + reader_started = True + except RuntimeError: + raise SetupFailure( + "Cannot start setup output reader", retryable=False + ) from None try: process.wait(timeout=max(0, work_deadline - time.monotonic())) except subprocess.TimeoutExpired: timed_out = True - if not timed_out: + if not timed_out and output_file is None: drain_timed_out = not output_done.wait( timeout=max(0, work_deadline - time.monotonic()) ) + except (Cancelled, SetupFailure) as exc: + pending_error = exc + raise finally: teardown_deadline = min(deadline, time.monotonic() + 4) - if process.poll() is None or not output_done.is_set() or read_errors: - reaped = self.stop(process, teardown_deadline) + if ( + process.poll() is None + or (output_file is None and (not output_done.is_set() or read_errors)) + or ( + output_file is not None + and (pending_error is not None or process.returncode != 0) + ) + ): + stopped = self.stop(process, teardown_deadline) + problems.extend(stopped.errors) if reader_started: output_done.wait(timeout=max(0, teardown_deadline - time.monotonic())) - if not reader_started or output_done.is_set(): - process.stdout.close() - if not reaped or (reader_started and not output_done.is_set()): - print("[sql] Command teardown incomplete within its deadline", file=sys.stderr) - if not reaped: - raise SetupFailure("Setup command could not be reaped", retryable=False) - if read_errors: - raise SetupFailure(read_errors[0], retryable=False) + if output_file is not None: + try: + self.snapshot(output_file, capture) + except OSError as exc: + problems.append(f"Cannot read Colima startup snapshot (errno={exc.errno})") + output_done.set() + elif not reader_started or output_done.is_set(): + try: + process.stdout.close() + except OSError as exc: + problems.append(f"Cannot close command output (errno={exc.errno})") + if not stopped.reaped: + problems.append("Owned child could not be reaped within the teardown deadline") + if reader_started and not output_done.is_set(): + problems.append("Output reader did not finish within the teardown deadline") + elif reader_started and read_errors: + problems.append("Output capture error: " + read_errors[0]) + if pending_error is not None: + reason = ( + f"Command cancelled (signal {pending_error.signum})" + if isinstance(pending_error, Cancelled) + else str(pending_error) + ) + print("[sql] " + failure_message(reason), file=sys.stderr, flush=True) + if timed_out: + raise SetupTimeout( + failure_message("Setup command timed out"), + retryable=not problems and not read_errors, + ) if drain_timed_out or not output_done.is_set(): raise SetupFailure( - "Setup command output drain timed out\n" + capture.output(), retryable=False + failure_message("Setup command output drain timed out"), retryable=False ) - if timed_out: - raise SetupTimeout("Setup command timed out\n" + capture.output()) + if read_errors: + raise SetupFailure(failure_message(read_errors[0]), retryable=False) + if problems: + reason = ( + f"Setup command failed (exit {process.returncode})" + if process.returncode + else "Setup command teardown failed" + ) + raise SetupFailure(failure_message(reason), retryable=False) return Result(process.returncode, capture.output()) @@ -273,12 +454,13 @@ def __init__(self, args, password, commands=None): def log(self, message): print("[sql] " + redact(message, self.password), flush=True) - def command(self, args, timeout=15, *, check=True, deadline=None): + def command(self, args, timeout=15, *, check=True, deadline=None, launcher=False): end = min(self.deadline, self.phase_deadline if deadline is None else deadline) remaining = min(timeout, end - time.monotonic()) if remaining <= 0: raise SetupTimeout("SQL setup deadline exhausted") - result = self.commands.run(args, remaining, env=self.env) + run = self.commands.run_launcher if launcher else self.commands.run + result = run(args, remaining, env=self.env) if result.returncode in (-signal.SIGINT, -signal.SIGTERM, 130, 143): raise Cancelled( signal.SIGINT if result.returncode in (-signal.SIGINT, 130) else signal.SIGTERM @@ -345,6 +527,8 @@ def diagnostics(self, container, *, deadline): self.log(f"Container logs unavailable (exit {result.returncode})") except SetupFailure as exc: self.log(f"Container logs unavailable: {exc}") + if not exc.retryable: + raise def remove(self, container, *, deadline): self.docker_command("rm", "--force", container.identifier, timeout=30, deadline=deadline) @@ -353,6 +537,37 @@ def remove(self, container, *, deadline): raise SetupFailure("Owned container still exists after removal", retryable=False) self.container = None + def remove_with_evidence(self, container, *, deadline, diagnostic_deadline): + diagnostic_error = None + try: + self.diagnostics(container, deadline=diagnostic_deadline) + except (SetupFailure, Cancelled) as exc: + diagnostic_error = exc + try: + self.remove(container, deadline=deadline) + except (SetupFailure, Cancelled) as removal_error: + if diagnostic_error is None: + raise + if isinstance(diagnostic_error, Cancelled): + self.log( + "Owned removal also failed during diagnostic cancellation: " + + ( + f"signal {removal_error.signum}" + if isinstance(removal_error, Cancelled) + else str(removal_error) + ) + ) + raise diagnostic_error from None + if isinstance(removal_error, Cancelled): + self.log(f"Diagnostics also failed before removal cancellation: {diagnostic_error}") + raise + raise SetupFailure( + f"Diagnostics failed: {diagnostic_error}\nOwned removal failed: {removal_error}", + retryable=False, + ) from None + if diagnostic_error is not None: + raise diagnostic_error + def cleanup(self, *, evidence=True): # Include lookup/ownership checks and removal verification in addition # to the log and rm deadlines. @@ -362,15 +577,24 @@ def cleanup(self, *, evidence=True): self.container = None return if evidence: - self.diagnostics( - container, deadline=min(self.phase_deadline - 30, time.monotonic() + 20) + self.remove_with_evidence( + container, + deadline=self.phase_deadline, + diagnostic_deadline=min(self.phase_deadline - 30, time.monotonic() + 20), ) - self.remove(container, deadline=self.phase_deadline) + else: + self.remove(container, deadline=self.phase_deadline) def prepare_runtime(self): if self.args.colima and not self.args.cleanup: self.log("Starting Colima once (outside SQL retry)") - self.command(["colima", "start", "--cpu", "4", "--memory", "8", "--disk", "50"], 600) + result = self.command( + ["colima", "start", "--cpu", "4", "--memory", "8", "--disk", "50"], + 600, + launcher=True, + ) + self.log("Colima launcher completed; Docker and SQL readiness still require checks") + self.log(result.output) def acquire_image(self): if self.image_id is not None: @@ -446,6 +670,8 @@ def wait_ready(self): try: probe = self.readiness_probe(polling_deadline) except SetupTimeout as exc: + if not exc.retryable: + raise self.log(f"Readiness probe timed out: {exc}") else: if probe.returncode == 0: @@ -463,9 +689,12 @@ def attempt(self): stale = self.find_owned() if stale is not None: self.log("Removing pre-existing same-job container before fresh setup") - self.diagnostics(stale, deadline=min(self.phase_deadline, time.monotonic() + 20)) try: - self.remove(stale, deadline=self.phase_deadline) + self.remove_with_evidence( + stale, + deadline=self.phase_deadline, + diagnostic_deadline=min(self.phase_deadline, time.monotonic() + 20), + ) except SetupFailure as exc: raise SetupFailure(str(exc), retryable=False) from None self.acquire_image() @@ -522,7 +751,8 @@ def setup(self): self.cleanup() except SetupFailure as cleanup_error: raise SetupFailure( - f"Cannot safely recover/clean up: {cleanup_error}", retryable=False + f"SQL setup failed: {exc}\nCannot safely recover/clean up: {cleanup_error}", + retryable=False, ) from None if not exc.retryable or number == 2: raise SetupFailure("SQL setup failed; no further attempts", retryable=False) From 912968ed20c770a452ec97631272fd624439c1ec Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 16:10:24 +0530 Subject: [PATCH 6/6] FIX: Use the shared setup deadline for Colima Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/scripts/setup_sql_container.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/scripts/setup_sql_container.py b/eng/scripts/setup_sql_container.py index 2be190e6..7beda1af 100644 --- a/eng/scripts/setup_sql_container.py +++ b/eng/scripts/setup_sql_container.py @@ -7,7 +7,8 @@ Cleanup refuses containers without the matching owner label. Only SQL lookup/pull/create/start/readiness/database setup is retried. Colima -starts once. The owned-container lookup also checks Docker availability, so +starts once within the remaining overall setup budget, without a separate +startup timeout. The owned-container lookup also checks Docker availability, so transient read-only failures can consume the same two attempts without creating or removing anything. Missing tools, invalid configuration/lookup results, ownership conflicts and unsafe cleanup are terminal. @@ -590,7 +591,7 @@ def prepare_runtime(self): self.log("Starting Colima once (outside SQL retry)") result = self.command( ["colima", "start", "--cpu", "4", "--memory", "8", "--disk", "50"], - 600, + self.deadline - time.monotonic(), launcher=True, ) self.log("Colima launcher completed; Docker and SQL readiness still require checks")