diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index 9fbc1f8500..df7393de4c 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -102,7 +102,7 @@ functions: - SSL - ORCHESTRATION_FILE - UV_PYTHON - - TOOLCHAIN_VERSION + - UV_PYTHON_PREFERENCE - STORAGE_ENGINE - REQUIRE_API_VERSION - DRIVERS_TOOLS @@ -111,6 +111,9 @@ functions: - LOAD_BALANCER - LOCAL_ATLAS - NO_EXT + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN type: test - command: expansions.update params: @@ -135,9 +138,9 @@ functions: - AWS_SESSION_TOKEN - COVERAGE - UV_PYTHON + - UV_PYTHON_PREFERENCE - LIBMONGOCRYPT_URL - MONGODB_URI - - TOOLCHAIN_VERSION - DISABLE_TEST_COMMANDS - GREEN_FRAMEWORK - NO_EXT @@ -249,7 +252,7 @@ functions: - test-numpy working_dir: src include_expansions_in_env: - - TOOLCHAIN_VERSION + - UV_PYTHON - COVERAGE type: test diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index e2677e7399..169f8ce85f 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -20,7 +20,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: regular - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_MIN_DEPS: "1" tags: [auth-aws, auth-aws-regular] - name: test-auth-aws-5.0-assume-role-python3.11 @@ -34,7 +34,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: assume-role - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" tags: [auth-aws, auth-aws-assume-role] - name: test-auth-aws-6.0-ec2-python3.12 commands: @@ -47,7 +47,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: ec2 - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" tags: [auth-aws, auth-aws-ec2] - name: test-auth-aws-7.0-env-creds-python3.13 commands: @@ -60,7 +60,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: env-creds - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" tags: [auth-aws, auth-aws-env-creds] - name: test-auth-aws-8.0-session-creds-python3.14t commands: @@ -73,7 +73,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: session-creds - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t tags: [auth-aws, auth-aws-session-creds, free-threaded] - name: test-auth-aws-9.0-web-identity-python3.14 commands: @@ -86,7 +86,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: web-identity - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: [auth-aws, auth-aws-web-identity] - name: test-auth-aws-9.0-web-identity-session-name-python3.14 commands: @@ -100,7 +100,7 @@ tasks: TEST_NAME: auth_aws SUB_TEST_NAME: web-identity AWS_ROLE_SESSION_NAME: test - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: [auth-aws, auth-aws-web-identity] - name: test-auth-aws-rapid-regular-python3.15t commands: @@ -113,7 +113,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: regular - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t tags: [auth-aws, auth-aws-regular, free-threaded] - name: test-auth-aws-latest-assume-role-python3.15-cov commands: @@ -126,7 +126,7 @@ tasks: vars: TEST_NAME: auth_aws SUB_TEST_NAME: assume-role - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" COVERAGE: "1" tags: [auth-aws, auth-aws-assume-role, pr] - name: test-auth-aws-ecs @@ -322,7 +322,7 @@ tasks: - func: assume ec2 role - func: run tests vars: - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_MIN_DEPS: "1" tags: [test-no-orchestration, python-3.10] - name: test-no-orchestration-python3.15 @@ -330,14 +330,14 @@ tasks: - func: assume ec2 role - func: run tests vars: - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: [test-no-orchestration, python-3.15] - name: test-no-orchestration-pypy3.11 commands: - func: assume ec2 role - func: run tests vars: - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: [test-no-orchestration, python-pypy3.11] # No toolchain tests @@ -392,7 +392,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -403,7 +403,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -414,7 +414,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -425,7 +425,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -436,7 +436,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -447,7 +447,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -458,7 +458,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -469,7 +469,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-invalid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -479,7 +479,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -490,7 +490,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -501,7 +501,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -512,7 +512,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -523,7 +523,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -534,7 +534,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -545,7 +545,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -556,7 +556,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-delegate-valid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -566,7 +566,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -577,7 +577,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -588,7 +588,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -599,7 +599,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -610,7 +610,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -621,7 +621,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -632,7 +632,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -643,7 +643,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-delegate-invalid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -653,7 +653,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -664,7 +664,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -675,7 +675,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -686,7 +686,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -697,7 +697,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -708,7 +708,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -719,7 +719,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -730,7 +730,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-soft-fail-v4.4-python3.10-min-deps @@ -740,7 +740,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -751,7 +751,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -762,7 +762,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -773,7 +773,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -784,7 +784,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -795,7 +795,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -806,7 +806,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -817,7 +817,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-valid-cert-server-staples-v4.4-python3.10-min-deps @@ -827,7 +827,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -843,7 +843,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -858,7 +858,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -873,7 +873,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -888,7 +888,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -903,7 +903,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -918,7 +918,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -933,7 +933,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest COVERAGE: "1" tags: @@ -949,7 +949,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -964,7 +964,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -979,7 +979,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -994,7 +994,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -1009,7 +1009,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -1024,7 +1024,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -1039,7 +1039,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -1054,7 +1054,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -1068,7 +1068,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -1083,7 +1083,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -1098,7 +1098,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -1113,7 +1113,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -1128,7 +1128,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -1143,7 +1143,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -1158,7 +1158,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -1173,7 +1173,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -1187,7 +1187,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -1202,7 +1202,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -1217,7 +1217,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -1232,7 +1232,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -1247,7 +1247,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -1262,7 +1262,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -1277,7 +1277,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -1292,7 +1292,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -1306,7 +1306,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -1317,7 +1317,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -1328,7 +1328,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -1339,7 +1339,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -1350,7 +1350,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -1361,7 +1361,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -1372,7 +1372,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -1383,7 +1383,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-v4.4-python3.10-min-deps @@ -1393,7 +1393,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -1404,7 +1404,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -1415,7 +1415,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -1426,7 +1426,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -1437,7 +1437,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -1448,7 +1448,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -1459,7 +1459,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -1470,7 +1470,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-ecdsa-malicious-no-responder-muststaple-server-does-not-staple-v4.4-python3.10-min-deps @@ -1480,7 +1480,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "4.4"] @@ -1491,7 +1491,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "5.0"] @@ -1502,7 +1502,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "6.0"] @@ -1513,7 +1513,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "7.0"] @@ -1524,7 +1524,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "8.0"] @@ -1535,7 +1535,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, "9.0"] @@ -1546,7 +1546,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-ecdsa, rapid] @@ -1557,7 +1557,7 @@ tasks: ORCHESTRATION_FILE: ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-ecdsa, latest] - name: test-ocsp-rsa-valid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -1567,7 +1567,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -1578,7 +1578,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -1589,7 +1589,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -1600,7 +1600,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -1611,7 +1611,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -1622,7 +1622,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -1633,7 +1633,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -1644,7 +1644,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-invalid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -1654,7 +1654,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -1665,7 +1665,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -1676,7 +1676,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -1687,7 +1687,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -1698,7 +1698,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -1709,7 +1709,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -1720,7 +1720,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -1731,7 +1731,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-delegate-valid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -1741,7 +1741,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -1752,7 +1752,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -1763,7 +1763,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -1774,7 +1774,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -1785,7 +1785,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -1796,7 +1796,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -1807,7 +1807,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -1818,7 +1818,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-delegate-invalid-cert-server-does-not-staple-v4.4-python3.10-min-deps @@ -1828,7 +1828,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -1839,7 +1839,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -1850,7 +1850,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -1861,7 +1861,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -1872,7 +1872,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -1883,7 +1883,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -1894,7 +1894,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -1905,7 +1905,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-soft-fail-v4.4-python3.10-min-deps @@ -1915,7 +1915,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -1926,7 +1926,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -1937,7 +1937,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -1948,7 +1948,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -1959,7 +1959,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -1970,7 +1970,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -1981,7 +1981,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -1992,7 +1992,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-valid-cert-server-staples-v4.4-python3.10-min-deps @@ -2002,7 +2002,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -2017,7 +2017,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -2032,7 +2032,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -2047,7 +2047,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -2062,7 +2062,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -2077,7 +2077,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -2092,7 +2092,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -2107,7 +2107,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -2121,7 +2121,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -2136,7 +2136,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -2151,7 +2151,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -2166,7 +2166,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -2181,7 +2181,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -2196,7 +2196,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -2211,7 +2211,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -2226,7 +2226,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -2240,7 +2240,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -2255,7 +2255,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -2270,7 +2270,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -2285,7 +2285,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -2300,7 +2300,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -2315,7 +2315,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -2330,7 +2330,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -2345,7 +2345,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: valid-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -2359,7 +2359,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: @@ -2374,7 +2374,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: @@ -2389,7 +2389,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: @@ -2404,7 +2404,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: @@ -2419,7 +2419,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: @@ -2434,7 +2434,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: @@ -2449,7 +2449,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: @@ -2464,7 +2464,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: - ocsp @@ -2478,7 +2478,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -2489,7 +2489,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -2500,7 +2500,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -2511,7 +2511,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -2522,7 +2522,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -2533,7 +2533,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -2544,7 +2544,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -2555,7 +2555,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-delegate-malicious-invalid-cert-muststaple-server-does-not-staple-v4.4-python3.10-min-deps @@ -2565,7 +2565,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -2576,7 +2576,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -2587,7 +2587,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -2598,7 +2598,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -2609,7 +2609,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -2620,7 +2620,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -2631,7 +2631,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -2642,7 +2642,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: revoked-delegate TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] - name: test-ocsp-rsa-malicious-no-responder-muststaple-server-does-not-staple-v4.4-python3.10-min-deps @@ -2652,7 +2652,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "4.4" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "4.4"] @@ -2663,7 +2663,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "5.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "5.0"] @@ -2674,7 +2674,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "6.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "6.0"] @@ -2685,7 +2685,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "7.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "7.0"] @@ -2696,7 +2696,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "8.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "8.0"] @@ -2707,7 +2707,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: "9.0" TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, "9.0"] @@ -2718,7 +2718,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" VERSION: rapid TEST_MIN_DEPS: "1" tags: [ocsp, ocsp-rsa, rapid] @@ -2729,7 +2729,7 @@ tasks: ORCHESTRATION_FILE: rsa-basic-tls-ocsp-mustStaple-disableStapling.json OCSP_SERVER_TYPE: no-responder TEST_NAME: ocsp - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" VERSION: latest tags: [ocsp, ocsp-rsa, latest] @@ -2784,10 +2784,14 @@ tasks: vars: VERSION: v8.0-perf SSL: ssl + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: run tests vars: TEST_NAME: perf SUB_TEST_NAME: sync + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: attach benchmark test results - func: send dashboard data tags: [perf] @@ -2797,10 +2801,14 @@ tasks: vars: VERSION: v8.0-perf SSL: ssl + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: run tests vars: TEST_NAME: perf SUB_TEST_NAME: async + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: attach benchmark test results - func: send dashboard data tags: [perf] @@ -2810,10 +2818,14 @@ tasks: vars: VERSION: v8.0-perf SSL: nossl + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: run tests vars: TEST_NAME: perf SUB_TEST_NAME: sync + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: attach benchmark test results - func: send dashboard data tags: [perf] @@ -2823,10 +2835,14 @@ tasks: vars: VERSION: v8.0-perf SSL: nossl + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: run tests vars: TEST_NAME: perf SUB_TEST_NAME: async + UV_PYTHON: 3.10.11 + UV_PYTHON_PREFERENCE: only-managed - func: attach benchmark test results - func: send dashboard data tags: [perf] @@ -2841,7 +2857,7 @@ tasks: - func: run tests vars: TEST_NAME: search_index - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: [search_index] # Server version tests @@ -2857,7 +2873,7 @@ tasks: AUTH: auth SSL: nossl TOPOLOGY: replica_set - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_async tags: - server-version @@ -2876,7 +2892,7 @@ tasks: AUTH: auth SSL: nossl TOPOLOGY: replica_set - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - server-version @@ -2897,7 +2913,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_async tags: - server-version @@ -2916,7 +2932,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: replica_set - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_sync tags: - server-version @@ -2937,7 +2953,7 @@ tasks: SSL: nossl TOPOLOGY: replica_set COVERAGE: "1" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_async tags: - server-version @@ -2960,7 +2976,7 @@ tasks: SSL: nossl TOPOLOGY: replica_set COVERAGE: "1" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_sync tags: - server-version @@ -2980,7 +2996,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: replica_set - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - server-version @@ -3000,7 +3016,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: replica_set - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_sync tags: - server-version @@ -3019,7 +3035,7 @@ tasks: AUTH: auth SSL: nossl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - server-version @@ -3040,7 +3056,7 @@ tasks: SSL: nossl TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - server-version @@ -3063,7 +3079,7 @@ tasks: TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" COVERAGE: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_async tags: - server-version @@ -3083,7 +3099,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - server-version @@ -3102,7 +3118,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_async tags: - server-version @@ -3121,7 +3137,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - server-version @@ -3140,7 +3156,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - server-version @@ -3159,7 +3175,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - server-version @@ -3179,7 +3195,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - server-version @@ -3198,7 +3214,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_async tags: - server-version @@ -3218,7 +3234,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_async tags: - server-version @@ -3241,7 +3257,7 @@ tasks: TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" COVERAGE: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - server-version @@ -3261,7 +3277,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - server-version @@ -3280,7 +3296,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - server-version @@ -3299,7 +3315,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_sync tags: - server-version @@ -3318,7 +3334,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_sync tags: - server-version @@ -3337,7 +3353,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - server-version @@ -3357,7 +3373,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_sync tags: - server-version @@ -3376,7 +3392,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - server-version @@ -3396,7 +3412,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_sync tags: - server-version @@ -3415,7 +3431,7 @@ tasks: AUTH: noauth SSL: nossl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - server-version @@ -3434,7 +3450,7 @@ tasks: AUTH: noauth SSL: nossl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - server-version @@ -3454,7 +3470,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - server-version @@ -3473,7 +3489,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: sharded_cluster - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - server-version @@ -3492,7 +3508,7 @@ tasks: AUTH: auth SSL: nossl TOPOLOGY: standalone - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - server-version @@ -3511,7 +3527,7 @@ tasks: AUTH: auth SSL: nossl TOPOLOGY: standalone - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - server-version @@ -3530,7 +3546,7 @@ tasks: AUTH: auth SSL: ssl TOPOLOGY: standalone - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - server-version @@ -3551,7 +3567,7 @@ tasks: SSL: ssl TOPOLOGY: standalone TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - server-version @@ -3572,7 +3588,7 @@ tasks: SSL: nossl TOPOLOGY: standalone COVERAGE: "1" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - server-version @@ -3594,7 +3610,7 @@ tasks: SSL: nossl TOPOLOGY: standalone COVERAGE: "1" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - server-version @@ -3615,7 +3631,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: standalone - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - server-version @@ -3634,7 +3650,7 @@ tasks: AUTH: noauth SSL: ssl TOPOLOGY: standalone - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - server-version @@ -3658,7 +3674,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "4.4" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - test-standard @@ -3680,7 +3696,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "4.4" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_sync tags: - test-standard @@ -3702,7 +3718,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "4.4" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - test-standard @@ -3724,7 +3740,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "4.4" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - test-standard @@ -3749,7 +3765,7 @@ tasks: TOPOLOGY: standalone VERSION: "4.4" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - test-standard @@ -3771,7 +3787,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "4.4" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - test-standard @@ -3794,7 +3810,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "4.4" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_sync tags: - test-standard @@ -3817,7 +3833,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "5.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - test-standard @@ -3839,7 +3855,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "5.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - test-standard @@ -3861,7 +3877,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "5.0" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_async tags: - test-standard @@ -3883,7 +3899,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "5.0" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_async tags: - test-standard @@ -3908,7 +3924,7 @@ tasks: TOPOLOGY: standalone VERSION: "5.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_async tags: - test-standard @@ -3930,7 +3946,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "5.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - test-standard @@ -3953,7 +3969,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "5.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_async tags: - test-standard @@ -3978,7 +3994,7 @@ tasks: TOPOLOGY: replica_set VERSION: "6.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - test-standard @@ -4000,7 +4016,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "6.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - test-standard @@ -4023,7 +4039,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "6.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_sync tags: - test-standard @@ -4046,7 +4062,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "6.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - test-standard @@ -4068,7 +4084,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "6.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_sync tags: - test-standard @@ -4090,7 +4106,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "6.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_sync tags: - test-standard @@ -4112,7 +4128,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "6.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_sync tags: - test-standard @@ -4136,7 +4152,7 @@ tasks: TOPOLOGY: replica_set VERSION: "7.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_async tags: - test-standard @@ -4158,7 +4174,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "7.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - test-standard @@ -4181,7 +4197,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "7.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_async tags: - test-standard @@ -4204,7 +4220,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "7.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - test-standard @@ -4226,7 +4242,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "7.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - test-standard @@ -4248,7 +4264,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "7.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - test-standard @@ -4270,7 +4286,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "7.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - test-standard @@ -4292,7 +4308,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "8.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_sync tags: - test-standard @@ -4314,7 +4330,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "8.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_sync tags: - test-standard @@ -4338,7 +4354,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: "8.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_sync tags: - test-standard @@ -4360,7 +4376,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "8.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_sync tags: - test-standard @@ -4383,7 +4399,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "8.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_sync tags: - test-standard @@ -4406,7 +4422,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "8.0" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - test-standard @@ -4428,7 +4444,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "8.0" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - test-standard @@ -4451,7 +4467,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "9.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - test-standard @@ -4473,7 +4489,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "9.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - test-standard @@ -4497,7 +4513,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: "9.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" TEST_NAME: default_async tags: - test-standard @@ -4519,7 +4535,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t TEST_NAME: default_async tags: - test-standard @@ -4542,7 +4558,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 TEST_NAME: default_async tags: - test-standard @@ -4565,7 +4581,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "9.0" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_async tags: - test-standard @@ -4587,7 +4603,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "9.0" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_async tags: - test-standard @@ -4612,7 +4628,7 @@ tasks: TOPOLOGY: replica_set VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_async tags: - test-standard @@ -4637,7 +4653,7 @@ tasks: TOPOLOGY: replica_set VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_async tags: - test-standard @@ -4663,7 +4679,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_async tags: - test-standard @@ -4688,7 +4704,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_async tags: - test-standard @@ -4713,7 +4729,7 @@ tasks: TOPOLOGY: standalone VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_async tags: - test-standard @@ -4738,7 +4754,7 @@ tasks: TOPOLOGY: standalone VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_async tags: - test-standard @@ -4761,7 +4777,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: rapid - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" TEST_NAME: default_sync tags: - test-standard @@ -4783,7 +4799,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: rapid - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t TEST_NAME: default_sync tags: - test-standard @@ -4806,7 +4822,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: rapid - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" TEST_NAME: default_sync tags: - test-standard @@ -4828,7 +4844,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: rapid - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" TEST_NAME: default_sync tags: - test-standard @@ -4850,7 +4866,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: rapid - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" TEST_NAME: default_sync tags: - test-standard @@ -4872,7 +4888,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: rapid - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" TEST_NAME: default_sync tags: - test-standard @@ -4896,7 +4912,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "8.2" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-string-query-preview - server-8.2 @@ -4919,7 +4935,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "4.4" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" tags: - test-non-standard - server-4.4 @@ -4940,7 +4956,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "4.4" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" tags: - test-non-standard - server-4.4 @@ -4963,7 +4979,7 @@ tasks: TOPOLOGY: standalone VERSION: "4.4" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: - test-non-standard - server-4.4 @@ -4984,7 +5000,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "4.4" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-4.4 @@ -5006,7 +5022,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "5.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t tags: - test-non-standard - server-5.0 @@ -5028,7 +5044,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "5.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-5.0 @@ -5050,7 +5066,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "5.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: - test-non-standard - server-5.0 @@ -5071,7 +5087,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "5.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" tags: - test-non-standard - server-5.0 @@ -5092,7 +5108,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "6.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-non-standard - server-6.0 @@ -5115,7 +5131,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: "6.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: - test-non-standard - server-6.0 @@ -5136,7 +5152,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "6.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-6.0 @@ -5158,7 +5174,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "6.0" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t tags: - test-non-standard - server-6.0 @@ -5180,7 +5196,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "7.0" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" tags: - test-non-standard - server-7.0 @@ -5201,7 +5217,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "7.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" tags: - test-non-standard - server-7.0 @@ -5222,7 +5238,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "7.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" tags: - test-non-standard - server-7.0 @@ -5243,7 +5259,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "7.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-7.0 @@ -5265,7 +5281,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "8.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: - test-non-standard - server-8.0 @@ -5286,7 +5302,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: "8.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-8.0 @@ -5308,7 +5324,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "8.0" - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t tags: - test-non-standard - server-8.0 @@ -5330,7 +5346,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "8.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t tags: - test-non-standard - server-8.0 @@ -5354,7 +5370,7 @@ tasks: TOPOLOGY: replica_set VERSION: "9.0" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: - test-non-standard - server-9.0 @@ -5375,7 +5391,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" tags: - test-non-standard - server-9.0 @@ -5396,7 +5412,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-9.0 @@ -5418,7 +5434,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: "9.0" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-non-standard - server-9.0 @@ -5439,7 +5455,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: latest - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t tags: - test-non-standard - server-latest @@ -5461,7 +5477,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: latest - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-latest @@ -5485,7 +5501,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-non-standard - server-latest @@ -5508,7 +5524,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-non-standard-no-cov - server-latest @@ -5530,7 +5546,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: latest - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: - test-non-standard - server-latest @@ -5551,7 +5567,7 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: rapid - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" tags: - test-non-standard - server-rapid @@ -5572,7 +5588,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: rapid - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t tags: - test-non-standard - server-rapid @@ -5594,7 +5610,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: rapid - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" tags: - test-non-standard - server-rapid @@ -5615,7 +5631,7 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: rapid - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-non-standard - server-rapid @@ -5629,7 +5645,7 @@ tasks: commands: - func: test numpy vars: - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: - binary - vector @@ -5639,7 +5655,7 @@ tasks: commands: - func: test numpy vars: - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" COVERAGE: "1" tags: - binary @@ -5665,7 +5681,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: "4.4" TEST_MIN_DEPS: "1" - TOOLCHAIN_VERSION: "3.10" + UV_PYTHON: "3.10" tags: - test-standard-auth - server-4.4 @@ -5686,7 +5702,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "4.4" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-4.4 @@ -5708,7 +5724,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "5.0" - TOOLCHAIN_VERSION: "3.11" + UV_PYTHON: "3.11" tags: - test-standard-auth - server-5.0 @@ -5729,7 +5745,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "5.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-5.0 @@ -5751,7 +5767,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "6.0" - TOOLCHAIN_VERSION: "3.12" + UV_PYTHON: "3.12" tags: - test-standard-auth - server-6.0 @@ -5772,7 +5788,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "6.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-6.0 @@ -5794,7 +5810,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "7.0" - TOOLCHAIN_VERSION: "3.13" + UV_PYTHON: "3.13" tags: - test-standard-auth - server-7.0 @@ -5815,7 +5831,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "7.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-7.0 @@ -5837,7 +5853,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "8.0" - TOOLCHAIN_VERSION: 3.14t + UV_PYTHON: 3.14t tags: - test-standard-auth - server-8.0 @@ -5859,7 +5875,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "8.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-8.0 @@ -5881,7 +5897,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: "3.14" + UV_PYTHON: "3.14" tags: - test-standard-auth - server-9.0 @@ -5902,7 +5918,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: "9.0" - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-9.0 @@ -5926,7 +5942,7 @@ tasks: TOPOLOGY: sharded_cluster VERSION: latest COVERAGE: "1" - TOOLCHAIN_VERSION: "3.15" + UV_PYTHON: "3.15" tags: - test-standard-auth - server-latest @@ -5948,7 +5964,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-latest @@ -5970,7 +5986,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: rapid - TOOLCHAIN_VERSION: 3.15t + UV_PYTHON: 3.15t tags: - test-standard-auth - server-rapid @@ -5992,7 +6008,7 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: rapid - TOOLCHAIN_VERSION: pypy3.11 + UV_PYTHON: pypy3.11 tags: - test-standard-auth - server-rapid diff --git a/.evergreen/resync-specs.sh b/.evergreen/resync-specs.sh index 4feca0aa5b..840d05bf1e 100755 --- a/.evergreen/resync-specs.sh +++ b/.evergreen/resync-specs.sh @@ -85,28 +85,59 @@ cp_psl () { echo "Could not find the public suffix list at $src" >&2 return 1 fi - python - "$src" "$PYMONGO"/pymongo/public_suffix_list.dat <<'EOF' + # Use the newest mongodbtoolchain python on CI hosts; python3 fallback locally. + PY=$(ls -1d /opt/mongodbtoolchain/v*/bin/python3 2>/dev/null | sort -V | tail -1) + [ -n "$PY" ] || PY=python3 + [ "$("$PY" -c 'import sys; print(sys.version_info >= (3, 7))' 2>/dev/null)" = True ] \ + || { echo "Error: $PY is not Python 3.7+, which this conversion requires." >&2; return 1; } + "$PY" - "$src" "$PYMONGO"/pymongo/public_suffix_list.dat <<'EOF' +import os import sys src, dst = sys.argv[1], sys.argv[2] -with open(src, encoding="utf-8") as f, open(dst, "w", encoding="utf-8") as out: - for line in f: - rule = line.strip() - if rule and not rule.startswith("//") and not rule.isascii(): - # "!" and "*." prefixes are not valid IDNA, so encode only the - # domain part and re-attach the prefix afterwards. - prefix = "" - for p in ("!", "*."): - if rule.startswith(p): - prefix, rule = p, rule[len(p) :] - break - try: - rule = rule.encode("idna").decode("ascii") - except UnicodeError: - print(f"Could not convert rule to Punycode: {prefix}{rule}", file=sys.stderr) - sys.exit(1) - line = f"{prefix}{rule.lower()}\n" - out.write(line) + +# Convert in memory and write atomically so failures can't truncate the list. +with open(src, encoding="utf-8") as f: + lines = f.readlines() +if not lines: + print(f"Source public suffix list is empty: {src}", file=sys.stderr) + sys.exit(1) + +converted = [] +for line in lines: + rule = line.strip() + if rule and not rule.startswith("//") and not rule.isascii(): + # "!" and "*." prefixes are not valid IDNA, so encode only the + # domain part and re-attach the prefix afterwards. + prefix = "" + for p in ("!", "*."): + if rule.startswith(p): + prefix, rule = p, rule[len(p) :] + break + try: + rule = rule.encode("idna").decode("ascii") + except UnicodeError: + print(f"Could not convert rule to Punycode: {prefix}{rule}", file=sys.stderr) + sys.exit(1) + line = f"{prefix}{rule.lower()}\n" + converted.append(line) + +if len(converted) != len(lines): + print( + f"Conversion lost lines: {len(lines)} in, {len(converted)} out", + file=sys.stderr, + ) + sys.exit(1) + +# A blank or comment-only source must not replace the real list. +if not any(ln.strip() and not ln.strip().startswith("//") for ln in converted): + print(f"Source public suffix list has no rules: {src}", file=sys.stderr) + sys.exit(1) + +tmp = dst + ".tmp" +with open(tmp, "w", encoding="utf-8") as out: + out.writelines(converted) +os.replace(tmp, dst) EOF } diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 4ec527c941..aa1fe97610 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -26,10 +26,13 @@ else exit 1 fi +# No-op on Evergreen (just.sh already ran this); needed for direct/local runs (PYTHON-6135). +. $SCRIPT_DIR/scripts/setup-uv-python.sh + # Start the test runner. -echo "Running tests with UV_PYTHON=${UV_PYTHON:-}..." +echo "Running tests with UV_PYTHON=${UV_PYTHON:-} UV_PYTHON_SEARCH_PATH=${UV_PYTHON_SEARCH_PATH:-}..." echo "UV_ARGS=${UV_ARGS}" uv run ${UV_ARGS} --reinstall-package pymongo .evergreen/scripts/run_tests.py "$@" -echo "Running tests with UV_PYTHON=${UV_PYTHON:-}... done." +echo "Running tests with UV_PYTHON=${UV_PYTHON:-} UV_PYTHON_SEARCH_PATH=${UV_PYTHON_SEARCH_PATH:-}... done." cd $PREV_DIR diff --git a/.evergreen/scripts/cleanup.sh b/.evergreen/scripts/cleanup.sh index f04a936fd2..44b36d3569 100755 --- a/.evergreen/scripts/cleanup.sh +++ b/.evergreen/scripts/cleanup.sh @@ -10,5 +10,10 @@ if [ -f $HERE/env.sh ]; then source $HERE/env.sh fi -rm -rf "${DRIVERS_TOOLS}" || true +# Don't delete the in-tree submodule; clean the ignored credential and state +# files (secrets-export.sh, AWS creds, token files) that `git submodule update` +# leaves behind, so they can't carry into later tasks on a reused host. A +# caller-provided DRIVERS_TOOLS wins, so the checkout actually used is cleaned. +: "${DRIVERS_TOOLS:=$HERE/../../drivers-evergreen-tools}" rm -f $HERE/../../secrets-export.sh || true +git -C "$DRIVERS_TOOLS" clean -fdx 2>/dev/null || true diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index fec3cdadae..5e36125838 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -12,7 +12,8 @@ else fi PROJECT_DIRECTORY="$(pwd)" -DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools" +# Default to the submodule; an env var override wins. +DRIVERS_TOOLS="${DRIVERS_TOOLS:-$PROJECT_DIRECTORY/drivers-evergreen-tools}" CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin" MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" @@ -93,12 +94,40 @@ export PROJECT="${project:-mongo-python-driver}" export PIP_QUIET=1 EOT -# Write the .env file for drivers-tools. -rm -rf $DRIVERS_TOOLS -BRANCH=master -ORG=mongodb-labs -git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS +# Only touch the in-tree submodule when it is the checkout actually in use; +# an overridden DRIVERS_TOOLS is a checkout we do not own. +if [ "$DRIVERS_TOOLS" = "$PROJECT_DIRECTORY/drivers-evergreen-tools" ]; then + # Initialize the submodule (Evergreen's git.get_project does not); tolerate + # non-git hosts with a warning. + if ! git -C "$PROJECT_DIRECTORY" submodule update --init --recursive; then + echo "WARNING: could not initialize the drivers-evergreen-tools submodule;" \ + "using the existing checkout contents instead." + fi + # Write a uv config boundary into the submodule: it is vendored inside this + # project, so uv run by the tools' own scripts would otherwise walk up to + # pyproject.toml and enforce our required-version pin. Write-if-absent, so an + # upstream uv.toml fails the submodule update loudly instead of being clobbered. + if [ -d "${DRIVERS_TOOLS}" ] && [ ! -f "${DRIVERS_TOOLS}/uv.toml" ]; then + cat < "${DRIVERS_TOOLS}/uv.toml" +# Written by mongo-python-driver to stop uv's config discovery here; see +# .evergreen/scripts/configure-env.sh. +EOT + fi + + # Keep the boundary out of git status via the submodule's local exclude; + # no-op without git. + if _git_dir=$(git -C "${DRIVERS_TOOLS}" rev-parse --absolute-git-dir 2>/dev/null); then + mkdir -p "${_git_dir}/info" + grep -qxF "uv.toml" "${_git_dir}/info/exclude" 2>/dev/null || + printf "uv.toml\n" >> "${_git_dir}/info/exclude" + fi +fi + +# Write the .env file for drivers-tools. Create the checkout if it is missing so +# a failed submodule init does not stop this script before setup-tests.py can +# reach check_drivers_tools() and report the problem actionably. +mkdir -p "${DRIVERS_TOOLS}" cat < ${DRIVERS_TOOLS}/.env SKIP_LEGACY_SHELL=1 DRIVERS_TOOLS="$DRIVERS_TOOLS" diff --git a/.evergreen/scripts/create-spec-pr.sh b/.evergreen/scripts/create-spec-pr.sh index 1de1a7cae1..daa68d0db6 100755 --- a/.evergreen/scripts/create-spec-pr.sh +++ b/.evergreen/scripts/create-spec-pr.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash +set -eu -tools="$(realpath -s "../drivers-tools")" +# Default to the drivers-evergreen-tools submodule when unset; a +# caller-provided DRIVERS_TOOLS wins, as in the other consumers. +tools="$(realpath -s "${DRIVERS_TOOLS:-./drivers-evergreen-tools}")" pushd $tools/.evergreen/github_app || exit owner="mongodb" @@ -31,20 +34,87 @@ echo "Creating the git checkout..." branch="spec-resync-"$(date '+%m-%d-%Y') git remote set-url origin https://x-access-token:${token}@github.com/$owner/$repo.git -git checkout -b $branch "origin/main" +# -B resets the branch if a reused checkout already has it, keeping reruns idempotent. +git checkout -B "$branch" "origin/main" + +# Attribute the commit to the bot user; fall back to the host identity. +bot_login="mongodb-drivers-pr-bot[bot]" +bot_id=$(curl -gsm 10 "https://api.github.com/users/${bot_login}" | jq -r '.id // empty' || true) +if [ -n "$bot_id" ]; then + git config user.name "$bot_login" + git config user.email "${bot_id}+${bot_login}@users.noreply.github.com" +fi + git add ./test git commit -am "resyncing specs $(date '+%m-%d-%Y')" echo "Creating the git checkout... done." -git push origin $branch +# Date-stamped branch: force-push makes same-day reruns idempotent; retry +# absorbs GitHub's intermittent 403s on the receive-pack endpoint. +push_ok=false +for _attempt in 1 2 3; do + if git push --force origin $branch; then + push_ok=true + break + fi + if [ "$_attempt" -lt 3 ]; then + echo "Push attempt ${_attempt} failed; retrying in 30s..." + sleep 30 + fi +done +if [ "$push_ok" != true ]; then + echo "Failed to push $branch after 3 attempts!" >&2 + exit 1 +fi + +# Build the payload as a file so the body content is always properly +# JSON-escaped, rather than interpolated into the command line as text. +payload_file=$(mktemp) +trap 'rm -f "$payload_file"' EXIT +jq -n \ + --arg title "[Spec Resync] $(date '+%m-%d-%Y')" \ + --arg head "${branch}" \ + --rawfile body "$1" \ + '{title: $title, body: $body, head: $head, base: "main"}' > "$payload_file" + +# Reuse an open PR for this branch (same-day rerun) instead of duplicating it. +existing_pr_json=$(curl -gs \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $token" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + --url "https://api.github.com/repos/$owner/$repo/pulls?head=$owner:$branch&state=open") +existing_pr_url=$(echo "$existing_pr_json" | jq -r '.[0].html_url // empty') +if [ -n "$existing_pr_url" ]; then + existing_pr_number=$(echo "$existing_pr_json" | jq -r '.[0].number // empty') + if ! curl -sfX PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $token" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + -d "{\"title\": \"[Spec Resync] $(date '+%m-%d-%Y')\", \"body\": $(jq -Rs . < "$1")}" \ + --url "https://api.github.com/repos/$owner/$repo/pulls/${existing_pr_number}" > /dev/null + then + echo "Failed to update PR summary!" >&2 + exit 1 + fi + echo "$existing_pr_url" + echo "Creating the PR... done. (PR already existed; branch and summary were updated)" + rm -rf $tools + exit 0 +fi + resp=$(curl -L \ -X POST \ -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer $token" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - -d "{\"title\":\"[Spec Resync] $(date '+%m-%d-%Y')\",\"body\":\"$(cat "$1")\",\"head\":\"${branch}\",\"base\":\"main\"}" \ + -d "@${payload_file}" \ --url https://api.github.com/repos/$owner/$repo/pulls) -echo $resp | jq '.html_url' -echo "Creating the PR... done." -rm -rf $tools +pr_url=$(echo "$resp" | jq -r '.html_url // empty') +if [ -z "$pr_url" ]; then + echo "Failed to create PR! API response:" + echo "$resp" | jq . + exit 1 +fi +echo "$pr_url" +echo "Creating the PR... done." diff --git a/.evergreen/scripts/dispatch.sh b/.evergreen/scripts/dispatch.sh new file mode 100755 index 0000000000..372cf6535c --- /dev/null +++ b/.evergreen/scripts/dispatch.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Route the just test recipes to the mod_wsgi tests or the Evergreen test +# runner. The mod_wsgi setup writes TEST_NAME to the test env file. +set -eu + +SCRIPT_DIR=$(dirname "${BASH_SOURCE:-$0}") +RECIPE="${1:-}" +if [ "$#" -gt 0 ]; then + shift +fi + +mod_wsgi_env() { + [ -f "$SCRIPT_DIR/test-env.sh" ] \ + && grep -q 'TEST_NAME="mod_wsgi"' "$SCRIPT_DIR/test-env.sh" \ + && bash "$SCRIPT_DIR/mod_wsgi.sh" active +} + +case "$RECIPE" in + setup-tests) + if [ "${1:-}" = "mod_wsgi" ]; then + shift + exec bash "$SCRIPT_DIR/mod_wsgi.sh" setup "${1:-}" + fi + exec bash "$SCRIPT_DIR/setup-tests.sh" "$@" + ;; + run-tests) + if mod_wsgi_env; then + if [ "$#" -gt 0 ]; then + echo "Ignoring arguments for the mod_wsgi tests: $*" >&2 + fi + exec bash "$SCRIPT_DIR/mod_wsgi.sh" test + fi + exec bash "$(dirname "$SCRIPT_DIR")/run-tests.sh" "$@" + ;; + teardown-tests) + if [ -f "$SCRIPT_DIR/test-env.sh" ] \ + && grep -q 'TEST_NAME="mod_wsgi"' "$SCRIPT_DIR/test-env.sh"; then + exec bash "$SCRIPT_DIR/mod_wsgi.sh" teardown + fi + exec bash "$SCRIPT_DIR/teardown-tests.sh" "$@" + ;; + *) + echo "Unknown recipe: $RECIPE" + exit 1 + ;; +esac diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 97fd9b0176..9ec13d5037 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -43,6 +43,7 @@ git_get_project, ) from shrub.v3.evg_task import EvgTask, EvgTaskDependency, EvgTaskRef +from utils import PERF_PYTHON_VERSION ############## # Variants @@ -412,7 +413,7 @@ def create_test_numpy_tasks(): tasks = [] for python in MIN_MAX_PYTHON: tags = ["binary", "vector", f"python-{python}", "test-numpy"] - vars = dict(TOOLCHAIN_VERSION=python) + vars = dict(UV_PYTHON=python) if python == MIN_MAX_PYTHON[-1]: tags.append("pr") vars["COVERAGE"] = "1" @@ -692,7 +693,7 @@ def create_server_version_tasks(): ) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() - test_vars["TOOLCHAIN_VERSION"] = python + test_vars["UV_PYTHON"] = python test_vars["TEST_NAME"] = f"default_{sync}" test_func = FunctionCall(func="run tests", vars=test_vars) tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) @@ -754,7 +755,7 @@ def create_test_non_standard_tasks(): name = get_task_name("test-non-standard", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() - test_vars["TOOLCHAIN_VERSION"] = python + test_vars["UV_PYTHON"] = python test_func = FunctionCall(func="run tests", vars=test_vars) tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) # For each coverage task, also emit a non-coverage companion so that @@ -772,7 +773,7 @@ def create_test_non_standard_tasks(): nc_name = get_task_name("test-non-standard", python=python, **nc_expansions) nc_server_func = FunctionCall(func="run server", vars=nc_expansions) nc_test_vars = nc_expansions.copy() - nc_test_vars["TOOLCHAIN_VERSION"] = python + nc_test_vars["UV_PYTHON"] = python nc_test_func = FunctionCall(func="run tests", vars=nc_test_vars) tasks.append( EvgTask(name=nc_name, tags=nc_tags, commands=[nc_server_func, nc_test_func]) @@ -803,7 +804,7 @@ def create_string_query_preview_tasks(): name = get_task_name("test-string-query-preview", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() - test_vars["TOOLCHAIN_VERSION"] = python + test_vars["UV_PYTHON"] = python test_func = FunctionCall(func="run tests", vars=test_vars) return [EvgTask(name=name, tags=tags, commands=[server_func, test_func])] @@ -846,7 +847,7 @@ def create_test_standard_auth_tasks(): name = get_task_name("test-standard-auth", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() - test_vars["TOOLCHAIN_VERSION"] = python + test_vars["UV_PYTHON"] = python test_func = FunctionCall(func="run tests", vars=test_vars) tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) return tasks @@ -886,7 +887,7 @@ def create_standard_tasks(): name = get_task_name("test-standard", python=python, sync=sync, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() - test_vars["TOOLCHAIN_VERSION"] = python + test_vars["UV_PYTHON"] = python test_vars["TEST_NAME"] = f"default_{sync}" test_func = FunctionCall(func="run tests", vars=test_vars) tasks.append(EvgTask(name=name, tags=tags, commands=[server_func, test_func])) @@ -901,7 +902,7 @@ def create_no_orchestration_tasks(): f"python-{python}", ] assume_func = FunctionCall(func="assume ec2 role") - test_vars = dict(TOOLCHAIN_VERSION=python) + test_vars = dict(UV_PYTHON=python) if python == ALL_PYTHONS[0]: test_vars["TEST_MIN_DEPS"] = "1" name = get_task_name("test-no-orchestration", **test_vars) @@ -951,7 +952,7 @@ def create_aws_tasks(): tags = [*base_tags, f"auth-aws-{test_type}"] if "t" in python: tags.append("free-threaded") - test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, TOOLCHAIN_VERSION=python) + test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, UV_PYTHON=python) if python == MIN_MAX_PYTHON[0]: test_vars["TEST_MIN_DEPS"] = "1" elif python == MIN_MAX_PYTHON[-1]: @@ -969,7 +970,7 @@ def create_aws_tasks(): TEST_NAME="auth_aws", SUB_TEST_NAME="web-identity", AWS_ROLE_SESSION_NAME="test", - TOOLCHAIN_VERSION=python, + UV_PYTHON=python, ) if "t" in python: tags.append("free-threaded") @@ -1018,7 +1019,7 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name): ORCHESTRATION_FILE=file_name, OCSP_SERVER_TYPE=server_type, TEST_NAME="ocsp", - TOOLCHAIN_VERSION=python, + UV_PYTHON=python, VERSION=version, ) if python == ALL_PYTHONS[0]: @@ -1081,7 +1082,7 @@ def create_aws_lambda_tasks(): def create_search_index_tasks(): assume_func = FunctionCall(func="assume ec2 role") server_func = FunctionCall(func="run server", vars=dict(TEST_NAME="search_index")) - vars = dict(TEST_NAME="search_index", TOOLCHAIN_VERSION=CPYTHONS[0]) + vars = dict(TEST_NAME="search_index", UV_PYTHON=CPYTHONS[0]) test_func = FunctionCall(func="run tests", vars=vars) task_name = "test-search-index-helpers" tags = ["search_index"] @@ -1092,9 +1093,12 @@ def create_search_index_tasks(): def create_perf_tasks(): tasks = [] for version, ssl, sync in product(["8.0"], ["ssl", "nossl"], ["sync", "async"]): - vars = dict(VERSION=f"v{version}-perf", SSL=ssl) + # The toolchain build may be this same patch version but is not + # optimized, which regressed CPU-bound benchmarks (PYTHON-6135). + perf_vars = dict(UV_PYTHON=PERF_PYTHON_VERSION, UV_PYTHON_PREFERENCE="only-managed") + vars = dict(VERSION=f"v{version}-perf", SSL=ssl, **perf_vars) server_func = FunctionCall(func="run server", vars=vars) - vars = dict(TEST_NAME="perf", SUB_TEST_NAME=sync) + vars = dict(TEST_NAME="perf", SUB_TEST_NAME=sync, **perf_vars) test_func = FunctionCall(func="run tests", vars=vars) attach_func = FunctionCall(func="attach benchmark test results") send_func = FunctionCall(func="send dashboard data") @@ -1313,7 +1317,7 @@ def create_run_server_func(): "SSL", "ORCHESTRATION_FILE", "UV_PYTHON", - "TOOLCHAIN_VERSION", + "UV_PYTHON_PREFERENCE", "STORAGE_ENGINE", "REQUIRE_API_VERSION", "DRIVERS_TOOLS", @@ -1322,6 +1326,11 @@ def create_run_server_func(): "LOAD_BALANCER", "LOCAL_ATLAS", "NO_EXT", + # The AWS credentials are needed to download the "latest" nightly + # build from the private S3 bucket, see DRIVERS-3628. + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", ] args = [".evergreen/just.sh", "run-server", "${TEST_NAME}"] sub_cmd = get_subprocess_exec(include_expansions_in_env=includes, args=args) @@ -1338,9 +1347,9 @@ def create_run_tests_func(): "AWS_SESSION_TOKEN", "COVERAGE", "UV_PYTHON", + "UV_PYTHON_PREFERENCE", "LIBMONGOCRYPT_URL", "MONGODB_URI", - "TOOLCHAIN_VERSION", "DISABLE_TEST_COMMANDS", "GREEN_FRAMEWORK", "NO_EXT", @@ -1363,7 +1372,7 @@ def create_run_tests_func(): def create_test_numpy_func(): - includes = ["TOOLCHAIN_VERSION", "COVERAGE"] + includes = ["UV_PYTHON", "COVERAGE"] test_cmd = get_subprocess_exec( include_expansions_in_env=includes, args=[".evergreen/just.sh", "test-numpy"] ) diff --git a/.evergreen/scripts/generate_config_utils.py b/.evergreen/scripts/generate_config_utils.py index efb8d6a34b..67be0dbec6 100644 --- a/.evergreen/scripts/generate_config_utils.py +++ b/.evergreen/scripts/generate_config_utils.py @@ -178,7 +178,7 @@ def get_common_name(base: str, sep: str, **kwargs) -> str: display_name = f"{display_name}{sep}{version}" for key, value in kwargs.items(): name = value - if key.lower() in ["python", "toolchain_version"]: + if key.lower() in ["python", "uv_python"]: if not value.startswith("pypy"): name = f"Python{value}" else: diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index 36e7866735..93b62ce6b6 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -2,6 +2,13 @@ # Install the necessary dependencies. set -euo pipefail +# The uv tool installs below (pinning uv, rust-just) just need a working +# interpreter, so drop the task's Python selection: it may not exist on this +# host (e.g. 3.15 with a 3.14 toolchain), and uv would fail on a too-old +# interpreter found in the search path (e.g. /usr/bin/python3 on RHEL) rather +# than skip it. Later steps re-source env.sh, restoring the selection. +unset UV_PYTHON UV_PYTHON_SEARCH_PATH UV_PYTHON_PREFERENCE + HERE=$(dirname ${BASH_SOURCE:-$0}) HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" pushd "$(dirname "$(dirname $HERE)")" > /dev/null @@ -60,8 +67,9 @@ fi # Set up uv if needed. if [ "$_need_setup" = "1" ]; then - # ensure-uv.sh (drivers-evergreen-tools) finds or installs uv and scopes its env. - if [ -n "${DRIVERS_TOOLS:-}" ] && [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then + # Default DRIVERS_TOOLS to the submodule; an env var override wins. + : "${DRIVERS_TOOLS:=$(dirname "$(dirname "$HERE")")/drivers-evergreen-tools}" + if [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then . "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ensure_uv || exit 1 fi diff --git a/.evergreen/scripts/mod_wsgi.sh b/.evergreen/scripts/mod_wsgi.sh new file mode 100755 index 0000000000..78233a4ff2 --- /dev/null +++ b/.evergreen/scripts/mod_wsgi.sh @@ -0,0 +1,263 @@ +#!/bin/bash +# Set up, run, or tear down the mod_wsgi tests. On Linux the test runs against +# the host's Apache; on other hosts it runs in an ubuntu:24.04 container with +# its own MongoDB replica set. +# +# Usage: mod_wsgi.sh setup | test | teardown | active +set -euo pipefail + +SCRIPT_DIR=$(dirname "${BASH_SOURCE:-$0}") +ROOT=$(dirname "$(dirname "$SCRIPT_DIR")") +CMD="${1:-}" +MODE="${2:-}" +CONTAINER=pymongo-mod-wsgi +HOME_SRC=/home/smoke/src +# The venv, MongoDB install, and the recorded version selection live outside +# the source copy, so refreshing the checkout never reinstalls them. +VENV=/home/smoke/venv +VERSIONS=/home/smoke/work/versions + +cd "$ROOT" + +case "$CMD" in + setup) + if [ "$MODE" != "standalone" ] && [ "$MODE" != "embedded" ]; then + echo "Usage: mod_wsgi.sh setup " + exit 1 + fi + # The harness recipes (run-tests, teardown-tests) dispatch on this. + echo "export TEST_NAME=\"mod_wsgi\"" > .evergreen/scripts/test-env.sh + echo "export SUB_TEST_NAME=\"$MODE\"" >> .evergreen/scripts/test-env.sh + # A failed setup leaves a partially started session behind. While the + # marker still routes to this session, tear it down; then remove the + # marker, so later just recipes do not route to the half-set-up state. + # The teardown is what stops the server or container setup may have + # started, and the next setup attempt stops any leftovers as well. + cleanup_marker() { + status=$? + if [ "$status" -ne 0 ]; then + bash "$SCRIPT_DIR/mod_wsgi.sh" teardown || true + rm -f .evergreen/scripts/test-env.sh + fi + } + trap cleanup_marker EXIT + ;; + test|teardown|active) ;; + *) + echo "Usage: mod_wsgi.sh setup | test | teardown | active" + exit 1 + ;; +esac + +if [ "$CMD" = "active" ]; then + # Exit 0 when a mod_wsgi session is active in this environment, so the + # dispatch script can route to us without a stale marker. Uses the system + # python and the tester's own STATE_FILE constant, so a missing or + # mismatched uv cannot break the dispatch and the path cannot drift. + if [ "$(uname -s)" = "Linux" ] && [ -z "${MOD_WSGI_DOCKER:-}" ]; then + if python3 -c " +import sys +sys.path.insert(0, '.evergreen/scripts') +from mod_wsgi_tester import STATE_FILE +sys.exit(0 if STATE_FILE.exists() else 1) +"; then + exit 0 + fi + exit 1 + fi + # The docker branch below handles the container case. +fi + +if [ "$(uname -s)" = "Linux" ] && [ -z "${MOD_WSGI_DOCKER:-}" ]; then + # Native: Apache runs on the host; MongoDB is already running, started by + # "just run-server" or the CI workflow. Set MOD_WSGI_DOCKER to force the + # container path on Linux. + if [ "$CMD" != "teardown" ]; then + # Other recipes' exact uv syncs prune the on-demand mod_wsgi install; + # restore it when missing. Checking first avoids reconciling a caller's + # differently-resolved dependencies. + if ! uv run --no-sync python -c "import mod_wsgi" 2>/dev/null; then + uv pip install -r requirements/mod_wsgi.txt --quiet + fi + fi + if [ "$CMD" = "setup" ]; then + uv run --no-sync python .evergreen/scripts/mod_wsgi_tester.py setup "$MODE" + else + uv run --no-sync python .evergreen/scripts/mod_wsgi_tester.py "$CMD" + fi + # Clear the dispatch marker so later just recipes use the harness. + if [ "$CMD" = "teardown" ]; then + rm -f .evergreen/scripts/test-env.sh + fi + exit 0 +fi + +# Docker path for hosts that cannot run Apache and mongod natively. +container_exists() { docker container inspect "$CONTAINER" >/dev/null 2>&1; } +container_running() { + [ "$(docker container inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null)" = "true" ] +} + +if [ "$CMD" = "active" ]; then + if container_exists; then + exit 0 + fi + exit 1 +fi + +# Copy the checkout into the container's home directory, so root-owned test +# artifacts never pollute it. +sync_source() { + docker exec "$CONTAINER" rm -rf "$HOME_SRC" + docker exec -u smoke "$CONTAINER" mkdir -p "$HOME_SRC" + tar -C "$ROOT" -cf - --exclude=.git --exclude=.venv \ + --exclude=access_log --exclude=error_log . | docker exec -i -u smoke "$CONTAINER" tar -C "$HOME_SRC" -xf - +} + +# Start a single-node replica set, if it is not already running. +ensure_mongod() { + docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + mkdir -p "$HOME/work/db" + if ! pgrep -x mongod >/dev/null; then + "$HOME/work/bin/mongod" --replSet rs0 --bind_ip 127.0.0.1 --port 27017 --dbpath "$HOME/work/db" --fork --logpath "$HOME/work/mongod.log" + fi + ' +} + +# Create the container and install the environment (idempotent). +bootstrap() { + if ! container_exists; then + docker run --name "$CONTAINER" -d ubuntu:24.04 sleep infinity + # A transient failure during initialization (for example in apt-get) + # would leave the half-created container behind, and later bootstraps + # would skip initialization and fail when sync_source expects the + # missing smoke user. Remove the container on failure, so the next + # attempt initializes a fresh one. + initialize() { + docker exec "$CONTAINER" apt-get update -qq + docker exec "$CONTAINER" apt-get install -y -qq apache2 apache2-dev build-essential curl jq git tar gzip ca-certificates + docker exec "$CONTAINER" useradd -m smoke + } + if ! initialize; then + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + exit 1 + fi + fi + if ! container_running; then + docker start "$CONTAINER" + fi + # Always refresh the checkout copy, so reused containers test the current + # source. The venv and versions file live outside it and survive the + # refresh. + sync_source + # Re-evaluate the newest Python and MongoDB on every bootstrap, so a reused + # container tracks matrix and server releases, but only reinstall when the + # recorded selection differs, since the downloads are slow. + docker exec --user smoke -w "$HOME_SRC" \ + -e VERSIONS="$VERSIONS" -e VENV="$VENV" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_PYTHON_INSTALL_DIR="$HOME/uv-python" + export UV_PROJECT_ENVIRONMENT="$VENV" + if ! command -v uv >/dev/null; then + curl -LsSf https://astral.sh/uv/install.sh | sh >/dev/null + fi + # Mirror the CI jobs and test the newest supported CPython. + LATEST_PYTHON=$(uv run --no-project --with "shrub.py>=3.10.0" python .evergreen/scripts/mod_wsgi_matrix.py | jq -r ".[-1].\"python-version\"") + case "$(uname -m)" in + aarch64|arm64) MARCH=aarch64 ;; + x86_64|amd64) MARCH=x86_64 ;; + *) echo "Unsupported architecture: $(uname -m)"; exit 1 ;; + esac + # MongoDB builds are per-distro; the container is ubuntu:24.04. + MONGODB_URL=$(curl -fsSL https://downloads.mongodb.org/current.json | jq -r " + .versions[] | select(.current and .production_release) | .downloads[] | + select(.target==\"ubuntu2404\" and .arch==\"$MARCH\") | .archive.url" | grep -v enterprise | head -1) + DESIRED="python=$LATEST_PYTHON mongodb=$MONGODB_URL" + if [ "$(cat "$VERSIONS" 2>/dev/null || true)" != "$DESIRED" ]; then + uv python install "$LATEST_PYTHON" >/dev/null + mkdir -p "$HOME/work/bin" + curl -fsSL "$MONGODB_URL" -o "$HOME/work/mongo.tgz" + MEMBER=$(tar -tzf "$HOME/work/mongo.tgz" | grep "bin/mongod$") + # Strip the two leading path components, so ensure_mongod finds the + # binary at "$HOME/work/bin/mongod". + tar -xz -C "$HOME/work/bin" --strip-components=2 -f "$HOME/work/mongo.tgz" "$MEMBER" + # A new interpreter needs a fresh venv; the sync below repopulates it. + rm -rf "$VENV" + printf "%s\n" "$DESIRED" > "$VERSIONS" + fi + # Rebuild the in-place C extensions on every bootstrap; the source + # refresh may carry artifacts built by a different interpreter. + export PYMONGO_C_EXT_MUST_BUILD=1 + uv sync --refresh-package pymongo + # The sync above prunes mod_wsgi, so install it after. + VIRTUAL_ENV="$VENV" uv pip install -r requirements/mod_wsgi.txt --quiet + ' + ensure_mongod + docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_PROJECT_ENVIRONMENT="'"$VENV"'" + # Probe replSetGetStatus over a direct connection: it returns immediately + # for an uninitialized replica set, while the default hello-based server + # selection would block for the full server selection timeout first. + if ! uv run --no-sync python -c " +from pymongo import MongoClient +client = MongoClient(\"127.0.0.1:27017\", directConnection=True) +try: + client.admin.command(\"replSetGetStatus\") +except Exception: + # NotYetInitialized (or a transient failure): initiation is still needed. + raise SystemExit(1) +" 2>/dev/null; then + uv run --no-sync python -c " +from pymongo import MongoClient +client = MongoClient(\"127.0.0.1:27017\", directConnection=True) +client.admin.command(\"replSetInitiate\", {\"_id\": \"rs0\", \"members\": [{\"_id\": 0, \"host\": \"127.0.0.1:27017\"}]}) +" + fi + ' +} + +# Bring a running session to readiness for a test run without refreshing the +# checkout copy: Apache may be live in it, and sync_source would unlink its +# pid file and error log. Only a missing, stopped, or never-bootstrapped +# container needs the full bootstrap, since then nothing is live to disturb. +ensure_ready() { + if container_exists && container_running \ + && docker exec "$CONTAINER" test -e "$VERSIONS"; then + ensure_mongod + else + bootstrap + fi +} + +case "$CMD" in + setup) + bootstrap + docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_PYTHON_INSTALL_DIR="$HOME/uv-python" + export UV_PROJECT_ENVIRONMENT="'"$VENV"'" + uv run --no-sync python .evergreen/scripts/mod_wsgi_tester.py setup '"$MODE"' + ' + ;; + test) + ensure_ready + docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_PROJECT_ENVIRONMENT="'"$VENV"'" + uv run --no-sync python .evergreen/scripts/mod_wsgi_tester.py test + ' + ;; + teardown) + if container_running; then + docker exec --user smoke -w "$HOME_SRC" "$CONTAINER" bash -c 'set -euo pipefail + export PATH="$HOME/.local/bin:$PATH" + export UV_PROJECT_ENVIRONMENT="'"$VENV"'" + uv run --no-sync python .evergreen/scripts/mod_wsgi_tester.py teardown + ' || true + docker stop -t 1 "$CONTAINER" + fi + rm -f .evergreen/scripts/test-env.sh + ;; +esac diff --git a/.evergreen/scripts/mod_wsgi_matrix.py b/.evergreen/scripts/mod_wsgi_matrix.py new file mode 100644 index 0000000000..2b8bb6611b --- /dev/null +++ b/.evergreen/scripts/mod_wsgi_matrix.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import json +import sys + +from generate_config_utils import ALL_VERSIONS, CPYTHONS + +# MongoDB 6.0 with the oldest supported CPython and minimum dependencies, and +# the latest MongoDB with the newest supported CPython. MongoDB 6.0 is the +# oldest version that installs on the GitHub runners. +VERSIONS = [ + { + "python-version": CPYTHONS[0], + "mongodb-version": "6.0", + }, + { + "python-version": CPYTHONS[-1], + "mongodb-version": ALL_VERSIONS[-1], + }, +] + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--index": + # Emit one version pair in GITHUB_OUTPUT format. + entry = VERSIONS[int(sys.argv[2])] + print(f"python-version={entry['python-version']}") + print(f"mongodb-version={entry['mongodb-version']}") + else: + print(json.dumps(VERSIONS)) diff --git a/.evergreen/scripts/mod_wsgi_tester.py b/.evergreen/scripts/mod_wsgi_tester.py new file mode 100644 index 0000000000..c326f6b476 --- /dev/null +++ b/.evergreen/scripts/mod_wsgi_tester.py @@ -0,0 +1,312 @@ +"""Setup, run, and tear down the mod_wsgi tests. + +Usage: + python .evergreen/scripts/mod_wsgi_tester.py setup + python .evergreen/scripts/mod_wsgi_tester.py test + python .evergreen/scripts/mod_wsgi_tester.py teardown + +The commands run as separate workflow steps, so state is passed between them +in a 0600 file in a user-owned cache directory. +""" + +from __future__ import annotations + +import json +import logging +import os +import signal +import socket +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") +LOGGER = logging.getLogger("mod_wsgi_tester") + +# Resolve __file__, so ROOT is correct when the script is invoked with a +# relative path, as mod_wsgi.sh does. +ROOT = Path(__file__).resolve().parents[2] +# State lives in a user-owned cache directory rather than shared /tmp: another +# local user must not be able to read or plant it, since teardown feeds its +# values into the commands that stop Apache. +STATE_DIR = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "pymongo-mod-wsgi" +STATE_DIR.mkdir(mode=0o700, parents=True, exist_ok=True) +STATE_FILE = STATE_DIR / "state.json" +PORT = 8080 + + +def run_command( + cmd: str | list[str], env: dict[str, str] | None = None, check: bool = True +) -> None: + LOGGER.info(f"Running command: {cmd}") + try: + result = subprocess.run( # noqa: S603 + cmd, + shell=isinstance(cmd, str), + env={**os.environ, **(env or {})}, + cwd=str(ROOT), + capture_output=True, + text=True, + check=False, + ) + except FileNotFoundError: + # A missing binary is a failed command like any other: raised when + # required, logged and swallowed for best-effort calls. The shell + # path used to behave the same way via the exit code, and hosts like + # the Fedora job's container do not ship pkill. + if check: + raise + LOGGER.warning(f"Command not found: {cmd if isinstance(cmd, str) else cmd[0]}") + return + if result.stdout: + LOGGER.info(result.stdout) + if result.returncode != 0: + LOGGER.error(result.stderr) + if check: + raise RuntimeError(f"Command failed with code {result.returncode}: {cmd}") + + +def make_request(url: str, timeout: int = 10) -> None: + for _ in range(timeout): + try: + # Per-request timeout, so an Apache that accepts the connection + # but never answers fails after roughly timeout seconds instead + # of hanging forever. + urllib.request.urlopen(url, timeout=1) # noqa: S310 + return + except OSError: + # URLError (and its HTTPError subclass) plus a socket timeout. + pass + time.sleep(1) + raise TimeoutError(f"Failed to access {url}") + + +def find_mod_wsgi_so() -> str: + # The .so is built against the installing interpreter and lives in the + # installed package. + import mod_wsgi + + so_files = list((Path(mod_wsgi.__file__).parent / "server").glob("mod_wsgi*.so")) + if len(so_files) != 1: + raise ValueError(f"Expected one mod_wsgi.so in {mod_wsgi.__file__}, found {so_files}") + return str(so_files[0]) + + +def find_apache() -> tuple[str, str]: + """Return the Apache binary and the config file matching its distribution.""" + from shutil import which + + apache = which("apache2") + if not apache: + # apache2 lives in sbin, which is not always on PATH for non-root users. + for candidate in ("/usr/lib/apache2/mpm-prefork/apache2", "/usr/sbin/apache2"): + if Path(candidate).exists(): + apache = candidate + break + if apache: + return apache, "apache24ubuntu.conf" + apache = which("httpd") + if not apache: + for candidate in ("/usr/sbin/httpd", "/usr/local/apache2/bin/httpd"): + if Path(candidate).exists(): + apache = candidate + break + if not apache: + raise ValueError("Could not find apache2 or httpd") + return apache, "httpd24fedora.conf" + + +def port_is_open() -> bool: + try: + socket.create_connection(("127.0.0.1", PORT), timeout=1).close() + return True + except OSError: + return False + + +def pid_file(apache_config: str) -> Path: + # The test configs write the master process's pid to the checkout root. + return ROOT / ("httpd.pid" if "httpd" in apache_config else "apache2.pid") + + +def pid_is_our_apache(pid: int, apache_config: str) -> bool: + # A stale pid file may name a pid the OS has reused for an unrelated + # process, so only signal an Apache whose command line carries this + # checkout's config argument. + try: + cmdline = (Path("/proc") / str(pid) / "cmdline").read_bytes() + except OSError: + return False + argv = cmdline.decode("utf-8", "replace").split("\0") + return f"{ROOT}/test/mod_wsgi_test/{apache_config}" in argv + + +def stop_apache(apache: str, apache_config: str, env: dict[str, str] | None = None) -> None: + # Idempotent: safe to call when Apache is not running. + # Prefer the pid file: -k stop re-parses the config, which fails when the + # mod_wsgi .so or the config's env vars are unavailable. Fall back to a + # validated pgrep for a server whose pid file was removed under it. The + # commands run as argument lists, so state-derived values never reach a + # shell. + from shutil import which + + pid = pid_file(apache_config) + if pid.exists(): + try: + victim = int(pid.read_text().strip()) + except ValueError: + victim = 0 + if victim and pid_is_our_apache(victim, apache_config): + try: + os.kill(victim, signal.SIGTERM) + except (ProcessLookupError, PermissionError): + LOGGER.warning("Could not stop Apache via the pid file") + else: + # Stale, reused, or unreadable pid: remove the file and stop + # through the config-validated path instead. + LOGGER.warning(f"Removing stale pid file {pid}") + pid.unlink() + if not pid.exists(): + run_command( + [apache, "-k", "stop", "-f", f"{ROOT}/test/mod_wsgi_test/{apache_config}"], + env=env, + check=False, + ) + if port_is_open(): + # Last resort: find pids whose full command line mentions this + # checkout's absolute config path, then signal only the ones whose + # command line carries it as an exact argument, so a server from + # another checkout (or an unrelated process) is never killed. + config_path = f"{ROOT}/test/mod_wsgi_test/{apache_config}" + pgrep = which("pgrep") + if pgrep: + result = subprocess.run( # noqa: S603 + [pgrep, "-f", config_path], capture_output=True, text=True, check=False + ) + for line in result.stdout.split(): + try: + victim = int(line) + except ValueError: + continue + if pid_is_our_apache(victim, apache_config): + try: + os.kill(victim, signal.SIGTERM) + except (ProcessLookupError, PermissionError): + LOGGER.warning(f"Could not stop Apache pid {victim}") + + # Stopping returns before the port is released, which the next mode needs + # to rebind. + deadline = time.time() + 10 + while time.time() < deadline: + if not port_is_open(): + return + time.sleep(0.5) + raise ValueError(f"Apache did not release port {PORT}") + + +def write_state(state: dict) -> None: + # The cache directory is user-only (0700) and the file is created 0600, so + # no other local user can read or plant this state. + fd = os.open(STATE_FILE, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + json.dump(state, f) + + +def read_state() -> dict: + # Lost or corrupt state falls back to discovery in teardown. + try: + return json.loads(STATE_FILE.read_text()) + except (OSError, ValueError): + return {} + + +def setup(sub_test_name: str) -> None: + if sub_test_name == "embedded": + conf = "mod_wsgi_test_embedded.conf" + elif sub_test_name == "standalone": + conf = "mod_wsgi_test.conf" + else: + raise ValueError("mod_wsgi sub test must be either 'standalone' or 'embedded'") + apache, apache_config = find_apache() + # On Fedora/RHEL, httpd embeds the interpreter mod_wsgi was built with and + # needs to be told where the Python installation lives. + env = { + "MOD_WSGI_CONF": conf, + "MOD_WSGI_SO": find_mod_wsgi_so(), + "MOD_WSGI_PYTHON_HOME": sys.base_prefix, + "PROJECT_DIRECTORY": str(ROOT), + } + # Stop any server left behind by a previous failed run so the port is free. + stop_apache(apache, apache_config, env) + run_command( + [apache, "-k", "start", "-f", f"{ROOT}/test/mod_wsgi_test/{apache_config}"], + env=env, + ) + write_state({"apache": apache, "config": apache_config, "env": env}) + + try: + for interp in ("interpreter1", "interpreter2"): + make_request(f"http://localhost:{PORT}/{interp}{ROOT}", 10) + except Exception: + # The Apache error log holds the actual failure. Stop the server so a + # failed setup does not leave it bound to the port. + stop_apache(apache, apache_config, env) + error_log = Path("error_log") + if error_log.exists(): + LOGGER.error(error_log.read_text()) + raise + + +def test() -> None: + sys.path.insert(0, str(ROOT)) + from test.mod_wsgi_test.test_client import main, parse_args + + uri1 = f"http://localhost:{PORT}/interpreter1{ROOT}" + uri2 = f"http://localhost:{PORT}/interpreter2{ROOT}" + try: + main(*parse_args(f"-n 25000 -t 100 parallel {uri1} {uri2}".split())) + main(*parse_args(f"-n 25000 serial {uri1} {uri2}".split())) + except BaseException: + # The test client raises KeyboardInterrupt in the failing path. + error_log = Path("error_log") + if error_log.exists(): + LOGGER.error(error_log.read_text()) + raise + + +def teardown() -> None: + state = read_state() + apache = state.get("apache") + apache_config = state.get("config") + env = state.get("env") + if not apache or not apache_config: + # State lost; fall back to discovery to stop a leftover server. The + # pid file in stop_apache usually makes this unnecessary. + try: + apache, apache_config = find_apache() + except ValueError: + return + env = { + "MOD_WSGI_CONF": "mod_wsgi_test.conf", + "MOD_WSGI_PYTHON_HOME": sys.base_prefix, + "PROJECT_DIRECTORY": str(ROOT), + } + try: + env["MOD_WSGI_SO"] = find_mod_wsgi_so() + except (ImportError, ValueError): + pass + stop_apache(apache, apache_config, env) + + +if __name__ == "__main__": + command = sys.argv[1] if len(sys.argv) > 1 else "" + if command == "setup": + setup(sys.argv[2]) + elif command == "test": + test() + elif command == "teardown": + teardown() + else: + raise SystemExit(f"Usage: {sys.argv[0]} setup |test|teardown") diff --git a/.evergreen/scripts/resync-all-specs.py b/.evergreen/scripts/resync-all-specs.py index 78afe8a4d8..52c276b8b8 100644 --- a/.evergreen/scripts/resync-all-specs.py +++ b/.evergreen/scripts/resync-all-specs.py @@ -28,6 +28,8 @@ def resync_specs(directory: pathlib.Path, errored: dict[str, str]) -> None: ) except CalledProcessError as exc: errored[spec.name] = exc.stderr + # The PR-body summary is lost if PR creation fails. + print(f"Spec sync failed for {spec.name}:\n{exc.stderr}", flush=True) print("Done syncing specs") @@ -57,9 +59,11 @@ def apply_patches(errored): ], check=True, stderr=subprocess.PIPE, + text=True, ) except CalledProcessError as exc: errored[f"{patch}"] = exc.stderr + print(f"Patch application failed for {patch}:\n{exc.stderr}", flush=True) def check_new_spec_directories(directory: pathlib.Path) -> list[str]: @@ -127,9 +131,10 @@ def write_summary(errored: dict[str, str], new: list[str], filename: str | None) if filename is None: print(f"\n{pr_body}") else: + # Plain text: the consumer embeds this file via jq --rawfile, + # which handles the JSON escaping itself. with open(filename, "w") as f: - # replacements made for proper json - f.write(pr_body.replace("\n", "\\n").replace("\t", "\\t")) + f.write(pr_body) def main(args: Namespace): diff --git a/.evergreen/scripts/resync-all-specs.sh b/.evergreen/scripts/resync-all-specs.sh index 41e4a2bc73..11dd2e65ba 100755 --- a/.evergreen/scripts/resync-all-specs.sh +++ b/.evergreen/scripts/resync-all-specs.sh @@ -34,7 +34,12 @@ then # we're running locally python3 ./.evergreen/scripts/resync-all-specs.py else - /opt/devtools/bin/python3.11 ./.evergreen/scripts/resync-all-specs.py --filename "$PR_DESC" + # Use the newest mongodbtoolchain python; the default python3 may be too old. + PY=$(ls -1d /opt/mongodbtoolchain/v*/bin/python3 2>/dev/null | sort -V | tail -1) + [ -n "$PY" ] || PY=python3 + [ "$("$PY" -c 'import sys; print(sys.version_info >= (3, 7))' 2>/dev/null)" = True ] \ + || { echo "Error: $PY is not Python 3.7+, which resync-all-specs.py requires." >&2; exit 1; } + "$PY" ./.evergreen/scripts/resync-all-specs.py --filename "$PR_DESC" if [[ -f $PR_DESC ]]; then # changes were made -> call scrypt to create PR for us .evergreen/scripts/create-spec-pr.sh "$PR_DESC" diff --git a/.evergreen/scripts/run-getdata.sh b/.evergreen/scripts/run-getdata.sh index 9435a5fcc3..416744ac03 100755 --- a/.evergreen/scripts/run-getdata.sh +++ b/.evergreen/scripts/run-getdata.sh @@ -2,6 +2,13 @@ # Get the debug data for an evergreen task. set -eu +HERE=$(dirname ${BASH_SOURCE:-$0}) +HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" +ROOT=$(dirname "$(dirname $HERE)") + +# Default to the drivers-evergreen-tools submodule when unset. +: "${DRIVERS_TOOLS:=$ROOT/drivers-evergreen-tools}" + . ${DRIVERS_TOOLS}/.evergreen/get-distro.sh || true get_distro || true echo $DISTRO diff --git a/.evergreen/scripts/run_server.py b/.evergreen/scripts/run_server.py index f81964f367..38f817c15d 100644 --- a/.evergreen/scripts/run_server.py +++ b/.evergreen/scripts/run_server.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -from utils import DRIVERS_TOOLS, ROOT, get_test_options, run_command +from utils import DRIVERS_TOOLS, ROOT, check_drivers_tools, get_test_options, run_command def set_env(name: str, value: Any = "1") -> None: @@ -14,11 +14,9 @@ def set_env(name: str, value: Any = "1") -> None: def start_server(): - run_mongodb_script = ( - Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh" if DRIVERS_TOOLS else None - ) + run_mongodb_script = Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh" want_help = bool({"-h", "--help"} & set(sys.argv[1:])) - if want_help and run_mongodb_script and run_mongodb_script.is_file(): + if want_help and run_mongodb_script.is_file(): # Forward straight to run-mongodb.sh's own help, without run_command's # "Running command..." logging noise. subprocess.run( # noqa: S603 @@ -28,13 +26,10 @@ def start_server(): ) return - # DRIVERS_TOOLS is only needed to actually start a server. When it's unset and - # -h/--help was requested, fall through to get_test_options' own argparse help below. - if not want_help and not DRIVERS_TOOLS: - raise ValueError( - "DRIVERS_TOOLS is not set; run `just run-server` from an Evergreen task " - "or set DRIVERS_TOOLS to a drivers-evergreen-tools checkout." - ) + # DRIVERS_TOOLS is only needed to start a server; allow -h/--help to fall + # through to argparse when the submodule is missing. + if not want_help: + check_drivers_tools() opts, extra_opts = get_test_options( "Run a MongoDB server. All given flags will be passed to run-mongodb.sh in DRIVERS_TOOLS.", diff --git a/.evergreen/scripts/setup-dev-env.sh b/.evergreen/scripts/setup-dev-env.sh index 508eb099bc..d9fbef2b04 100755 --- a/.evergreen/scripts/setup-dev-env.sh +++ b/.evergreen/scripts/setup-dev-env.sh @@ -53,6 +53,29 @@ if [ "${CI:-}" != "true" ] && [ "${GITHUB_ACTIONS:-}" != "true" ]; then printf 'export PATH="%s:$PATH"\n' "$PYMONGO_BIN_DIR_POSIX" >> "$_rc" fi +# Initialize the submodule (Evergreen's git.get_project does not); tolerate +# non-git hosts with a warning. +if ! git -C "$ROOT" submodule update --init --recursive; then + echo "WARNING: could not initialize the drivers-evergreen-tools submodule;" \ + "set DRIVERS_TOOLS to a drivers-evergreen-tools checkout instead." +fi + +# Mirror configure-env.sh's uv config boundary; see it for the full explanation. +if [ -d "$ROOT/drivers-evergreen-tools" ] && [ ! -f "$ROOT/drivers-evergreen-tools/uv.toml" ]; then + cat < "$ROOT/drivers-evergreen-tools/uv.toml" +# Written by mongo-python-driver to stop uv's config discovery here; see +# .evergreen/scripts/configure-env.sh. +EOT +fi + +# Keep the boundary out of git status via the submodule's local exclude; +# no-op without git. +if _git_dir=$(git -C "$ROOT/drivers-evergreen-tools" rev-parse --absolute-git-dir 2>/dev/null); then + mkdir -p "${_git_dir}/info" + grep -qxF "uv.toml" "${_git_dir}/info/exclude" 2>/dev/null || + printf "uv.toml\n" >> "${_git_dir}/info/exclude" +fi + # Ensure dependencies are installed. bash $HERE/install-dependencies.sh diff --git a/.evergreen/scripts/setup-uv-python.sh b/.evergreen/scripts/setup-uv-python.sh index e83abce4b9..09397803f5 100755 --- a/.evergreen/scripts/setup-uv-python.sh +++ b/.evergreen/scripts/setup-uv-python.sh @@ -1,13 +1,22 @@ #!/bin/bash -# Set up the UV_PYTHON variable. -set -eu +# Choose the Python that uv uses, and make sure uv can provide it. +# +# Input: +# UV_PYTHON - a version ("3.10", "3.14t"), an implementation ("pypy3.11"), +# or an interpreter path ("/usr/bin/python3.11"). +# +# Exports: +# UV_PYTHON_SEARCH_PATH - the Python toolchain bin dir for the request, so uv +# uses the toolchain Python instead of downloading one. +# UV_PYTHON_PREFERENCE - "system" so the toolchain wins over managed installs. +# A preference set by the task is left alone. +# UV_PYTHON - the Python interpreter uv uses; defaults to CPython +# 3.10 when the task does not set one. +set -euo pipefail HERE=$(dirname ${BASH_SOURCE:-$0}) HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" -# Use min supported version by default. -_python="3.10" - # Source the env files to pick up common variables. if [ -f $HERE/env.sh ]; then . $HERE/env.sh @@ -18,43 +27,74 @@ if [ -f $HERE/test-env.sh ]; then . $HERE/test-env.sh fi -if [ -z "${UV_PYTHON:-}" ]; then - set -x - # Translate a TOOLCHAIN_VERSION to UV_PYTHON. - if [ -n "${TOOLCHAIN_VERSION:-}" ]; then - _python=$TOOLCHAIN_VERSION - if [ "$(uname -s)" = "Darwin" ]; then - if [[ "$_python" == *"t"* ]]; then - binary_name="python3t" - framework_dir="PythonT" - else - binary_name="python3" - framework_dir="Python" - fi - _python=$(echo "$_python" | sed 's/t//g') - _python="/Library/Frameworks/$framework_dir.Framework/Versions/$_python/bin/$binary_name" - elif [ "Windows_NT" = "${OS:-}" ]; then - _python=$(echo $_python | cut -d. -f1,2 | sed 's/\.//g; s/t//g') - if [[ "$TOOLCHAIN_VERSION" == *"t"* ]]; then - _exe="python${TOOLCHAIN_VERSION}.exe" - else - _exe="python.exe" - fi - if [ -n "${IS_WIN32:-}" ]; then - _win_python="C:/python/32/Python${_python}/${_exe}" - else - _win_python="C:/python/Python${_python}/${_exe}" - fi - # Fall back to the version string if the toolchain path doesn't exist. - # uv will then download and manage Python itself. - if [ -f "$_win_python" ]; then - _python="$_win_python" - else - _python="$TOOLCHAIN_VERSION" - fi - elif [ -d "/opt/python/$_python/bin" ]; then - _python="/opt/python/$_python/bin/python3" +# Default to a known-good Python so behavior is deterministic when a task does +# not pin one. Selecting the uv binary is separate and driven by the required +# uv version in pyproject.toml. +export UV_PYTHON="${UV_PYTHON:-3.10}" + +# Print the Python toolchain bin dir for a version like "3.10" or "3.14t". +function _toolchain_dir() { + local version="$1" dir + # Only plain CPython versions live in the toolchain. + [[ "$version" =~ ^[0-9]+\.[0-9]+t?$ ]] || return 1 + if [ "Windows_NT" = "${OS:-}" ]; then + local winver="${version%t}" + winver="${winver//./}" + if [ -n "${IS_WIN32:-}" ]; then + dir="C:/python/32/Python${winver}" + else + dir="C:/python/Python${winver}" fi + elif [ "$(uname -s)" = "Darwin" ]; then + if [[ "$version" == *t ]]; then + dir="/Library/Frameworks/PythonT.Framework/Versions/${version%t}/bin" + else + dir="/Library/Frameworks/Python.Framework/Versions/${version}/bin" + fi + else + dir="/opt/python/${version}/bin" fi - export UV_PYTHON="$_python" + [ -d "$dir" ] || return 1 + echo "$dir" +} + +# Make sure uv can provide the requested Python, downloading it if needed. +function _ensure_python() { + local request="$1" + if uv python find "$request" > /dev/null 2>&1; then + return 0 + fi + echo "Python \"$request\" was not found on this host; asking uv to install it..." + if uv python install "$request"; then + return 0 + fi + echo "ERROR: uv could not find or install Python \"$request\"." >&2 + return 1 +} + +_search_path="" +_skip_ensure="" + +if [[ "$UV_PYTHON" == */* || "$UV_PYTHON" == *\\* ]]; then + # An explicit interpreter path; uv uses it as-is, so probe/install is skipped. + if [ ! -x "$UV_PYTHON" ]; then + echo "ERROR: UV_PYTHON=$UV_PYTHON does not exist or is not executable." >&2 + exit 1 + fi + _skip_ensure=1 +elif _dir=$(_toolchain_dir "$UV_PYTHON"); then + _search_path="$_dir" +fi + +# Point uv at the toolchain Python when there is one and the task did not set +# its own preference: a task-level UV_PYTHON_PREFERENCE wins (PYTHON-6135). +# On CI the toolchain dir is already first on PATH (configure-env.sh), so this +# mainly benefits local hosts and later steps. +if [ -n "$_search_path" ] && [ -z "${UV_PYTHON_PREFERENCE:-}" ]; then + export UV_PYTHON_SEARCH_PATH="$_search_path" + export UV_PYTHON_PREFERENCE="system" +fi + +if [ -z "$_skip_ensure" ]; then + _ensure_python "${UV_PYTHON:-}" fi diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 5a3f441f58..c8a15ad4a6 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -13,10 +13,12 @@ ENV_FILE, HERE, LOGGER, + PERF_PYTHON_VERSION, PLATFORM, ROOT, TEST_SUITE_MAP, Distro, + check_drivers_tools, get_test_options, read_env, run_command, @@ -30,6 +32,8 @@ "MONGODB_API_VERSION", "DEBUG_LOG", "UV_PYTHON", + "UV_PYTHON_SEARCH_PATH", + "UV_PYTHON_PREFERENCE", "REQUIRE_FIPS", "IS_WIN32", ] @@ -49,9 +53,6 @@ # Map the test name to test group. GROUP_MAP = dict(mockupdb="mockupdb", perf="perf") -# The python version used for perf tests. -PERF_PYTHON_VERSION = "3.10.11" - # The libmongocrypt release used when LIBMONGOCRYPT_URL is not set. Must be at # least 1.20.0 for the GA "substring" query type. LIBMONGOCRYPT_VERSION = "1.20.2" @@ -327,8 +328,7 @@ def handle_test_env() -> None: MULTI_MONGOS_LB_URI += "&tls=true" write_env("SINGLE_MONGOS_LB_URI", SINGLE_MONGOS_LB_URI) write_env("MULTI_MONGOS_LB_URI", MULTI_MONGOS_LB_URI) - if not DRIVERS_TOOLS: - raise RuntimeError("Missing DRIVERS_TOOLS") + check_drivers_tools() cmd = f'bash "{DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh" start' run_command(cmd) @@ -358,7 +358,9 @@ def handle_test_env() -> None: # The mock OCSP responder MUST BE started before the mongod as the mongod expects that # a responder will be available upon startup. - version = os.environ.get("VERSION", "latest") + # Default to the newest stable release; the "latest" nightly build is + # downloaded from a private S3 bucket and needs AWS credentials. + version = os.environ.get("VERSION", "latest-stable") cmd = [ "bash", f"{DRIVERS_TOOLS}/.evergreen/run-mongodb.sh", @@ -374,8 +376,7 @@ def handle_test_env() -> None: run_command(cmd, cwd=DRIVERS_TOOLS) if SSL != "nossl": - if not DRIVERS_TOOLS: - raise RuntimeError("Missing DRIVERS_TOOLS") + check_drivers_tools() write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem") write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem") @@ -427,8 +428,7 @@ def handle_test_env() -> None: # PATH is updated by configure-env.sh for access to mongocryptd. if test_name == "encryption": - if not DRIVERS_TOOLS: - raise RuntimeError("Missing DRIVERS_TOOLS") + check_drivers_tools() csfle_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/csfle") # Opt in to corporate Azure credentials (DRIVERS-3392) os.environ["FLE_AZURE_USE_CORPORATE"] = "YES" @@ -511,8 +511,12 @@ def handle_test_env() -> None: run_command("tar xf single_and_multi_document.tgz", cwd=data_dir) write_env("TEST_PATH", str(data_dir)) write_env("OUTPUT_FILE", str(ROOT / "results.json")) - # Overwrite the UV_PYTHON from the env.sh file. - write_env("UV_PYTHON", "") + # The toolchain build may be this same patch version but is not + # optimized, which regressed CPU-bound benchmarks (PYTHON-6135). + write_env("UV_PYTHON", PERF_PYTHON_VERSION) + write_env("UV_PYTHON_PREFERENCE", "only-managed") + with ENV_FILE.open("a", newline="\n") as fid: + fid.write("unset UV_PYTHON_SEARCH_PATH\n") UV_ARGS.append(f"--python={PERF_PYTHON_VERSION}") diff --git a/.evergreen/scripts/stop-server.sh b/.evergreen/scripts/stop-server.sh index 045a655cbd..a254a2715a 100755 --- a/.evergreen/scripts/stop-server.sh +++ b/.evergreen/scripts/stop-server.sh @@ -4,6 +4,7 @@ set -eu HERE=$(dirname ${BASH_SOURCE:-$0}) HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" +ROOT=$(dirname "$(dirname $HERE)") # Try to source the env file. if [ -f $HERE/env.sh ]; then @@ -11,4 +12,7 @@ if [ -f $HERE/env.sh ]; then source $HERE/env.sh fi +# Default to the drivers-evergreen-tools submodule when unset. +: "${DRIVERS_TOOLS:=$ROOT/drivers-evergreen-tools}" + bash ${DRIVERS_TOOLS}/.evergreen/run-mongodb.sh stop diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index a7decdfaa8..57c460617e 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -12,7 +12,11 @@ HERE = Path(__file__).absolute().parent ROOT = HERE.parent.parent -DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/") +# DRIVERS_TOOLS defaults to the drivers-evergreen-tools submodule; an env var +# override wins. +DRIVERS_TOOLS = (os.environ.get("DRIVERS_TOOLS") or str(ROOT / "drivers-evergreen-tools")).replace( + os.sep, "/" +) TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz" # noqa: S108 LOGGER = logging.getLogger("test") @@ -81,6 +85,9 @@ class Distro: EXTRA_TESTS = ["aws_lambda", "doctest"] +# The managed Python version used for perf tests (PYTHON-6135). +PERF_PYTHON_VERSION = "3.10.11" + # Tests that do not use run-mongodb directly. NO_RUN_ORCHESTRATION = [ "auth_oidc", @@ -297,8 +304,8 @@ def run_command(cmd: str | list[str], **kwargs: Any) -> None: kwargs.setdefault("check", True) # Prevent overriding the python used by other tools. env = kwargs.pop("env", os.environ).copy() - if "UV_PYTHON" in env: - del env["UV_PYTHON"] + for var in ["UV_PYTHON", "UV_PYTHON_SEARCH_PATH", "UV_PYTHON_PREFERENCE"]: + env.pop(var, None) kwargs["env"] = env try: subprocess.run(shlex.split(cmd), **kwargs) # noqa: PLW1510, S603 @@ -309,6 +316,18 @@ def run_command(cmd: str | list[str], **kwargs: Any) -> None: LOGGER.info("Running command '%s'... done.", cmd) +def check_drivers_tools() -> None: + """Raise a clear error when the drivers-evergreen-tools checkout is missing.""" + # An uninitialized submodule can exist as an empty directory, so a bare + # is_dir() check passes. Require a script every consumer needs instead. + if not (Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh").is_file(): + raise RuntimeError( + "The drivers-evergreen-tools checkout is missing or empty; run `just " + "install` to initialize the submodule, or set DRIVERS_TOOLS to a " + "drivers-evergreen-tools checkout." + ) + + def create_archive() -> str: run_command("git add .", cwd=ROOT) run_command('git commit --no-verify -m "add files"', check=False, cwd=ROOT) diff --git a/.evergreen/spec-patch/PYTHON-5993.patch b/.evergreen/spec-patch/PYTHON-5993.patch new file mode 100644 index 0000000000..3880924e83 --- /dev/null +++ b/.evergreen/spec-patch/PYTHON-5993.patch @@ -0,0 +1,94 @@ +diff --git a/test/unified-test-format/valid-pass/operator-gte.json b/test/unified-test-format/valid-pass/operator-gte.json +new file mode 100644 +index 000000000..ae69da5c9 +--- /dev/null ++++ b/test/unified-test-format/valid-pass/operator-gte.json +@@ -0,0 +1,88 @@ ++{ ++ "description": "operator-gte", ++ "schemaVersion": "1.29", ++ "createEntities": [ ++ { ++ "client": { ++ "id": "client0", ++ "observeEvents": [ ++ "commandStartedEvent" ++ ] ++ } ++ }, ++ { ++ "database": { ++ "id": "database0", ++ "client": "client0", ++ "databaseName": "database0Name" ++ } ++ }, ++ { ++ "collection": { ++ "id": "collection0", ++ "database": "database0", ++ "collectionName": "coll0" ++ } ++ } ++ ], ++ "initialData": [ ++ { ++ "collectionName": "coll0", ++ "databaseName": "database0Name", ++ "documents": [] ++ } ++ ], ++ "tests": [ ++ { ++ "description": "special gte matching operator", ++ "operations": [ ++ { ++ "name": "insertOne", ++ "object": "collection0", ++ "arguments": { ++ "document": { ++ "_id": 1, ++ "x": 2, ++ "y": 3, ++ "z": 4 ++ } ++ } ++ } ++ ], ++ "expectEvents": [ ++ { ++ "client": "client0", ++ "events": [ ++ { ++ "commandStartedEvent": { ++ "command": { ++ "insert": "coll0", ++ "documents": [ ++ { ++ "_id": { ++ "$$gte": 1 ++ }, ++ "x": { ++ "$$gte": 1.9 ++ }, ++ "y": { ++ "$$gte": { ++ "$numberLong": "3" ++ } ++ }, ++ "z": { ++ "$$gte": 4 ++ } ++ } ++ ] ++ }, ++ "commandName": "insert", ++ "databaseName": "database0Name" ++ } ++ } ++ ] ++ } ++ ] ++ } ++ ] ++} diff --git a/.github/actions/mod-wsgi-versions/action.yml b/.github/actions/mod-wsgi-versions/action.yml new file mode 100644 index 0000000000..63994a3c14 --- /dev/null +++ b/.github/actions/mod-wsgi-versions/action.yml @@ -0,0 +1,23 @@ +name: Compute Mod WSGI versions +description: Compute the CPython and MongoDB versions for a Mod WSGI job. +inputs: + index: + description: Row of the version matrix to use. + required: true +outputs: + python-version: + description: The CPython version to test with. + value: ${{ steps.versions.outputs.python-version }} + mongodb-version: + description: The MongoDB version to test against. + value: ${{ steps.versions.outputs.mongodb-version }} +runs: + using: composite + steps: + - id: versions + # The versions come from the Evergreen config's version lists. + shell: bash + env: + INDEX: ${{ inputs.index }} + run: | + uv run --no-project --with 'shrub.py>=3.10.0' python .evergreen/scripts/mod_wsgi_matrix.py --index "$INDEX" >> "$GITHUB_OUTPUT" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 72d1e1e084..603c47d7fe 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,3 +19,29 @@ updates: schedule: interval: "weekly" open-pull-requests-limit: 0 + # drivers-evergreen-tools submodule + - package-ecosystem: "gitsubmodule" + directory: "/" + schedule: + interval: "weekly" + cooldown: + default-days: 7 + allow: + - dependency-name: "drivers-evergreen-tools" + # Python (pip). Manages requirements/mod_wsgi.txt only; the other + # requirements files are resolved in uv.lock. + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + exclude-paths: + - "/pyproject.toml" + - "/requirements.txt" + - "/requirements/aws.txt" + - "/requirements/docs.txt" + - "/requirements/encryption.txt" + - "/requirements/gssapi.txt" + - "/requirements/ocsp.txt" + - "/requirements/snappy.txt" + - "/requirements/test.txt" + - "/requirements/zstd.txt" diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index 819261bd9d..c0856cf2d8 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -61,7 +61,7 @@ jobs: - name: Set up QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0 + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 with: # setup-qemu-action by default uses `tonistiigi/binfmt:latest` image, # which is out of date. This causes seg faults during build. diff --git a/.github/workflows/sbom.yml b/.github/workflows/sbom.yml index 91af8d840c..62b0ca3ceb 100644 --- a/.github/workflows/sbom.yml +++ b/.github/workflows/sbom.yml @@ -14,6 +14,7 @@ on: - 'requirements/**.txt' - '!requirements/docs.txt' - '!requirements/test.txt' + - '!requirements/mod_wsgi.txt' permissions: contents: write diff --git a/.github/workflows/test-mod-wsgi.yml b/.github/workflows/test-mod-wsgi.yml new file mode 100644 index 0000000000..2ae2310ab6 --- /dev/null +++ b/.github/workflows/test-mod-wsgi.yml @@ -0,0 +1,179 @@ +name: Mod WSGI Tests + +on: + push: + branches: ["main", "v**"] + paths: + - ".evergreen/scripts/mod_wsgi_*" + - ".evergreen/scripts/mod_wsgi.sh" + - ".evergreen/scripts/dispatch.sh" + - ".evergreen/scripts/generate_config_utils.py" + - ".github/workflows/test-mod-wsgi.yml" + - ".github/actions/mod-wsgi-*/**" + - "test/mod_wsgi_test/**" + - "pymongo/**" + - "bson/**" + - "justfile" + - "pyproject.toml" + - "uv.lock" + - "requirements/mod_wsgi.txt" + pull_request: + paths: + - ".evergreen/scripts/mod_wsgi_*" + - ".evergreen/scripts/mod_wsgi.sh" + - ".evergreen/scripts/dispatch.sh" + - ".evergreen/scripts/generate_config_utils.py" + - ".github/workflows/test-mod-wsgi.yml" + - ".github/actions/mod-wsgi-*/**" + - "test/mod_wsgi_test/**" + - "pymongo/**" + - "bson/**" + - "justfile" + - "pyproject.toml" + - "uv.lock" + - "requirements/mod_wsgi.txt" + workflow_dispatch: + +concurrency: + group: mod-wsgi-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash -eux {0} + +permissions: + contents: read + +jobs: + mod_wsgi_min_deps: + # Exercise the minimum dependency versions against Fedora's httpd, so the + # mod_wsgi tests cover httpd in addition to Ubuntu's apache2. + runs-on: ubuntu-latest + name: Mod WSGI Min Deps + container: + # fedora:latest, pinned by digest; bump periodically. + image: fedora@sha256:43b29f65a41eb9c35e1cd5323e3bdf3b655c2357a9f4f1ff2f9c2798e5045d80 + env: + # Install the interpreter where the Apache user can read it. + UV_PYTHON_INSTALL_DIR: /opt/uv-python + # The test rounds run against the lowest direct dependency versions + # installed below; a resync would reconcile them back to the lockfile. + JUST_NO_RESYNC: "1" + steps: + # actions/checkout needs git, which fedora:latest does not include. + - name: Install git + run: dnf install -y git + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - name: Install just + run: | + uv tool install rust-just + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Install Apache + # jq and curl are needed to pick the MongoDB download. + run: dnf install -y httpd httpd-devel jq curl + - id: versions + uses: $/.github/actions/mod-wsgi-versions + with: + index: 0 + - name: Install Python + env: + PYTHON_VERSION: ${{ steps.versions.outputs.python-version }} + run: | + uv python install "$PYTHON_VERSION" + echo "UV_PYTHON=$PYTHON_VERSION" >> "$GITHUB_ENV" + - name: Install dependencies + run: | + export PYMONGO_C_EXT_MUST_BUILD=1 + just install + # just install skips the sync when uv is already set up. + uv sync --refresh-package pymongo + uv run --no-sync python tools/fail_if_no_c.py + # Exercise the mod_wsgi>=4.9.4 lower bound. + uv pip install -e ".[test]" -r requirements/mod_wsgi.txt --resolution=lowest-direct --force-reinstall + - name: Start MongoDB + # Fedora is not a supported MongoDB platform, so install the RHEL build + # directly instead of using the drivers-evergreen-tools action, which + # relies on actions/setup-python and does not run on Fedora. + env: + MONGODB_VERSION: ${{ steps.versions.outputs.mongodb-version }} + run: | + ARCH=$(uname -m) + URL=$(curl -fsSL https://downloads.mongodb.org/current.json | jq -r --arg v "$MONGODB_VERSION." --arg a "$ARCH" ' + .versions[] | select(.current and .production_release) | select(.version | startswith($v)) | + .downloads[] | select(.target=="rhel90" and .arch==$a) | .archive.url' | grep -v enterprise | head -1) + curl -fsSL "$URL" -o /tmp/mongo.tgz + tar -xz -C /tmp --strip-components=1 -f /tmp/mongo.tgz + mkdir -p /tmp/db + /tmp/bin/mongod --replSet rs0 --bind_ip 127.0.0.1 --port 27017 --dbpath /tmp/db --fork --logpath /tmp/mongod.log + # --no-sync: a sync would reconcile the lowest-direct dependency + # versions installed above back to the lockfile. + uv run --no-sync python -c " + from pymongo import MongoClient + client = MongoClient('127.0.0.1:27017', directConnection=True) + client.admin.command('replSetInitiate', {'_id': 'rs0', 'members': [{'_id': 0, 'host': '127.0.0.1:27017'}]}) + MongoClient().admin.command('hello') + " + - name: Run standalone tests + run: | + just setup-tests mod_wsgi standalone + just run-tests + - name: Teardown standalone + if: always() + run: just teardown-tests + - name: Run embedded tests + run: | + just setup-tests mod_wsgi embedded + just run-tests + - name: Teardown embedded + if: always() + run: just teardown-tests + + mod_wsgi_latest: + runs-on: ubuntu-22.04 + name: Mod WSGI Latest + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - id: versions + uses: $/.github/actions/mod-wsgi-versions + with: + index: 1 + - name: Install Python tooling + uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 + with: + python-version: ${{ steps.versions.outputs.python-version }} + - name: Install Apache + run: sudo apt-get install -y apache2 apache2-dev + - name: Install dependencies + run: | + export PYMONGO_C_EXT_MUST_BUILD=1 + just install + # just install skips the sync when uv is already set up. + uv sync --refresh-package pymongo + uv pip install -r requirements/mod_wsgi.txt + uv run --no-sync python tools/fail_if_no_c.py + - id: setup-mongodb + uses: mongodb-labs/drivers-evergreen-tools@49da51cebc0060b4d699d57bdef30ee7e071de44 + with: + version: ${{ steps.versions.outputs.mongodb-version }} + topology: "replica_set" + - name: Run standalone tests + run: | + just setup-tests mod_wsgi standalone + just run-tests + - name: Teardown standalone + if: always() + run: just teardown-tests + - name: Run embedded tests + run: | + just setup-tests mod_wsgi embedded + just run-tests + - name: Teardown embedded + if: always() + run: just teardown-tests diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 5e0d89f7a4..f5b1069e30 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -85,7 +85,7 @@ jobs: # The beta Python here relies on the action's prerelease default. python-version: ${{ matrix.python-version }} - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "${{ matrix.mongodb-version }}" - name: Run tests @@ -101,12 +101,13 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false + submodules: true - name: Install Python tooling uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 with: python-version: "3.10" - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "8.0" - name: Setup tests @@ -116,7 +117,7 @@ jobs: - name: Generate xml report run: uv tool run --with "coverage[toml]" coverage xml - name: Upload test results to Codecov - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} doctest: @@ -126,6 +127,7 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false + submodules: true - name: Install Python tooling uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 with: @@ -133,7 +135,7 @@ jobs: - name: Install dependencies run: just install - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "8.0" - name: Run tests @@ -184,6 +186,7 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false + submodules: true - name: Install Python tooling uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 with: @@ -191,12 +194,12 @@ jobs: - name: Install dependencies run: just install - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 - name: Run tests run: | just integration-tests - id: setup-mongodb-ssl - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: ssl: true - name: Run tests @@ -251,7 +254,7 @@ jobs: # Test sdist on lowest supported Python python-version: "3.9" - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 - name: Run connect test from sdist shell: bash run: | @@ -275,7 +278,7 @@ jobs: with: python-version: "3.9" - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "8.0" - name: Run tests diff --git a/.github/zizmor.yml b/.github/zizmor.yml index af9f715dd4..2a69f0dd1a 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -4,4 +4,4 @@ rules: policies: actions/*: ref-pin mongodb-labs/drivers-github-tools/*: hash-pin - mongodb-labs/drivers-evergreen-tools: ref-pin + mongodb-labs/drivers-evergreen-tools: hash-pin diff --git a/.gitignore b/.gitignore index 8546aeabb7..63813daf02 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ build/ doc/_build/ dist/ tools/settings.py -drivers-evergreen-tools pymongo.egg-info/ *.so *.egg* @@ -47,3 +46,10 @@ xunit-results/ coverage.xml server.log .coverage +test-results.json +access_log +error_log + +# AI-assistant review workflow logs (kept untracked) +/REVIEW_STATE.md +/REVIEW.md diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..d89986aa20 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "drivers-evergreen-tools"] + path = drivers-evergreen-tools + url = https://github.com/mongodb-labs/drivers-evergreen-tools.git diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 20b7210736..7f24a3aca3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -203,10 +203,15 @@ the pages will re-render and the browser will automatically refresh. `just install` installs the pinned version of `uv` (from `[tool.uv] required-version`) into `$HOME/.local/bin`, and adds that directory to your shell rc file when missing, so the pinned `uv` takes effect in new shells. If a project `uv` command (e.g. `just test`) runs with a different `uv` version, `uv` fails fast and tells you how to update. + Scripts inside the vendored `drivers-evergreen-tools` submodule are exempt: the setup scripts write a `uv.toml` + boundary file into that checkout (see `.evergreen/scripts/configure-env.sh`), which stops uv's config discovery + there, since the tools' scripts run uv versions this project does not pin. - Ensure you have started the appropriate Mongo Server(s). You can run `just run-server` with optional args to set up the server. All given options will be passed to - [`run-mongodb.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-mongodb.sh). Run `$DRIVERS_TOOLS/.evergreen/run-mongodb.sh start -h` - for a full list of options. + [`run-mongodb.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-mongodb.sh). Run `${DRIVERS_TOOLS:-drivers-evergreen-tools}/.evergreen/run-mongodb.sh start -h` + for a full list of options. By default the newest stable server release is used. To test against the + nightly build instead, pass `--version latest`, which is downloaded from a private S3 bucket and requires + an `AWS_PROFILE` with [Drivers test secrets](https://github.com/mongodb-labs/drivers-evergreen-tools/tree/master/.evergreen/secrets_handling#secrets-handling) credentials. - Run `just test` or `pytest` to run all of the tests. - Append `test/.py::::` to run specific tests. You can omit the `` to test a full class @@ -220,10 +225,10 @@ the pages will re-render and the browser will automatically refresh. ### Prerequisites -- Clone `drivers-evergreen-tools`: - `git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`. -- Run `export DRIVERS_TOOLS=$PWD/drivers-evergreen-tools`. This can be put into a `.bashrc` file - for convenience. +- The `drivers-evergreen-tools` submodule (see + [The drivers-evergreen-tools submodule](#the-drivers-evergreen-tools-submodule)). + `just install` initializes it; no manual clone or `export DRIVERS_TOOLS` is needed. + `DRIVERS_TOOLS` remains an optional override for an existing local checkout. - Some tests require access to [Drivers test secrets](https://github.com/mongodb-labs/drivers-evergreen-tools/tree/master/.evergreen/secrets_handling#secrets-handling). ### Usage @@ -353,10 +358,28 @@ You will need to set up access to the `drivers-test-secrets-role`, see the [Wiki - Run `just setup-tests aws_lambda`. - Run `just run-tests`. +### mod_wsgi tests + +Continuous integration runs the tests on pull requests that change +mod_wsgi-relevant files, in the `test-mod-wsgi.yml` workflow. + +To run the tests by hand, install Apache and mod_wsgi (`sudo apt-get install -y apache2 +apache2-dev` and `uv pip install -r requirements/mod_wsgi.txt` on Ubuntu), then: + +- On Linux, run `TOPOLOGY=replica_set just run-server`. +- Run `just setup-tests mod_wsgi `. +- Run `just run-tests`. +- Run `just teardown-tests`. + +The `mode` can be `standalone` or `embedded`. On non-Linux hosts the same +commands run inside an ubuntu container, which bootstraps its own replica +set (so the `run-server` step does not apply), or use `just smoke-mod-wsgi` +to run both modes. + ### OCSP tests - Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`. -This corresponds to a config file in `$DRIVERS_TOOLS/.evergreen/orchestration/configs/servers`. +This corresponds to a config file in `${DRIVERS_TOOLS:-drivers-evergreen-tools}/.evergreen/orchestration/configs/servers`. MongoDB servers on MacOS and Windows do not staple OCSP responses and only support RSA. NOTE: because the mock ocsp responder MUST be started prior to the server starting, the ocsp tests start the server as part of `setup-tests`. @@ -372,6 +395,44 @@ If you are running one of the `no-responder` tests, omit the `run-server` step. - Set up the tests with `sync` or `async`: `just setup-tests perf sync`. - Run the tests: `just run-tests`. +## The drivers-evergreen-tools submodule + +The `drivers-evergreen-tools` repository is consumed as a git submodule at the repo root, +pinned to a specific commit. Dependabot bumps the pin weekly. + +### Daily flow + +Nothing to do: `just install` initializes the submodule, and Dependabot keeps it fresh. + +### Manually bumping the submodule + +```bash +git -C drivers-evergreen-tools fetch --tags +git -C drivers-evergreen-tools checkout vX.Y.Z +git add drivers-evergreen-tools +``` + +### Evergreen patches that need tools changes + +Point the submodule at the needed commit and commit the new gitlink in the patch branch; +`configure-env.sh` checks out the recorded SHA on Evergreen hosts (`setup-dev-env.sh` does +the same for local checkouts with `just install`). + +### Using a local checkout instead + +Set `DRIVERS_TOOLS` to the path of a local clone — the environment variable wins over the +submodule default: + +```bash +export DRIVERS_TOOLS=/path/to/drivers-evergreen-tools +``` + +Alternatively, keep a local pin from being reset by `git submodule update`: + +```bash +git config submodule.drivers-evergreen-tools.update none +``` + ## Enable Debug Logs - Use `-o log_cli_level="DEBUG" -o log_cli=1` with `just test` or `pytest` to output all debug logs to the terminal. **Warning**: This will output a huge amount of logs. @@ -403,19 +464,21 @@ To run any of the test suites with minimum supported dependencies, pass `--test- - If there are any services or atlas clusters to teardown, handle them in `.evergreen/scripts/teardown_tests.py`. - Add functions to generate the test variant(s) and task(s) to the `.evergreen/scripts/generate_config.py`. - There are some considerations about the Python version used in the test: - - If a specific version of Python is needed in a task that is running on variants with a toolchain, use -``TOOLCHAIN_VERSION`` (e.g. `TOOLCHAIN_VERSION=3.10`). The actual path lookup needs to be done on the host, since -tasks are host-agnostic. + - To request a specific Python, set `UV_PYTHON` (e.g. `UV_PYTHON=3.10`, `UV_PYTHON=3.14t`, or +`UV_PYTHON=pypy3.11`). Tasks are host-agnostic, so the interpreter lookup happens on the host: for a plain +CPython version whose toolchain dir exists, `UV_PYTHON_SEARCH_PATH` points uv at it, and for anything else +(including PyPy, or a version the toolchain lacks) uv downloads it. - If a specific Python binary is needed (for example on the FIPS host), set `UV_PYTHON=/path/to/python`. - - If a specific Python version is needed and the toolchain will not be available, use `UV_PYTHON` (e.g. `UV_PYTHON=3.11`). - - The default if neither ``TOOLCHAIN_VERSION`` or ``UV_PYTHON`` is set is to use UV to install the minimum - supported version of Python and use that. This ensures a consistent behavior across host types that do not - have the Python toolchain (e.g. Azure VMs), by having a known version of Python with the build headers (`Python.h`) - needed to build the C extensions. + - The default if `UV_PYTHON` is not set is CPython 3.10. This is deterministic across host types, so a task + that does not pin a version always gets the same Python (with the build headers (`Python.h`) needed to build + the C extensions). - The uv binary version is pinned once in `[tool.uv] required-version` in `pyproject.toml`. `.evergreen/scripts/install-dependencies.sh` installs it with `uv tool install`, uv enforces it locally, and `astral-sh/setup-uv` reads it on GitHub. Bump it manually when a newer uv is needed. If uv cannot find the - requested Python, it installs it; if that fails, the task fails. + requested Python, it installs it; if that fails, the task fails. The pin is not enforced for scripts inside + the vendored `drivers-evergreen-tools` submodule: those scripts run uv versions this project does not pin, so + the setup scripts write a `uv.toml` boundary into the submodule checkout that stops uv's config discovery + (and with it the required-version check) at the submodule boundary. - Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. - Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. diff --git a/doc/changelog.rst b/doc/changelog.rst index 08d3908e62..9ea2f05c2e 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -16,6 +16,27 @@ PyMongo 4.19 brings a number of changes including: interpreter remain daemon threads and shutdown behavior is unchanged. Note that because these threads are non-daemon, a subinterpreter may block on teardown until any in-flight monitor work completes. +- Added the ``kms_connect_callback`` option on + :class:`~pymongo.encryption_options.AutoEncryptionOpts`, + :class:`~pymongo.encryption.ClientEncryption`, and + :class:`~pymongo.asynchronous.encryption.AsyncClientEncryption`: a callable + that opens the connection to a Key Management Service (KMS) host for + Client-Side Field Level Encryption and Queryable Encryption, e.g. to route + KMS requests through a proxy. The driver performs the KMS TLS handshake over + the callback's connection, so verification still targets the KMS host. +- Added the ``srv_host_validator`` keyword argument to + :class:`~pymongo.synchronous.mongo_client.MongoClient` and + :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient`, an alternative to + ``srvAllowedHostsSuffix`` for deployments whose acceptable SRV hosts cannot be + expressed as a single suffix. The callback is invoked once per SRV-returned + host and returns ``True`` to accept it. It is mutually exclusive with + ``srvAllowedHostsSuffix`` and, because it takes a callable, cannot be set in a + connection string. See the + :class:`~pymongo.synchronous.mongo_client.MongoClient` and + :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` documentation for + security considerations. +- ``srvAllowedHostsSuffix`` may now be set to a single label reserved for + private or special use, such as ``localhost``, ``test``, or ``internal``. Bug fixes ......... @@ -23,8 +44,25 @@ Bug fixes - Fixed a bug where the synchronous client could permanently deadlock under gevent when a greenlet was killed while checking a connection back into the pool (`PYTHON-6074`_). +- ``MongoClient.append_metadata()`` and ``AsyncMongoClient.append_metadata()`` + now detect duplicates by comparing the whole + :class:`~pymongo.driver_info.DriverInfo` instead of only its name. The + comparison is exact, so drivers that differ in name case or platform are no + longer treated as duplicates (`PYTHON-6040`_). +- ``driver.name`` and ``driver.version`` in the handshake metadata are now + ``|``-delimited lists with 1:1 index correspondence, including empty version + entries for the built-in ``|c`` and ``|async`` name segments + (`PYTHON-6040`_). +- Fixed a bug where truncating the handshake metadata to 512 bytes could leave + ``driver.name`` and ``driver.version`` with different numbers of ``|`` + delimiters (`PYTHON-6040`_). +- :class:`~pymongo.driver_info.DriverInfo` now raises :class:`ValueError` when + any field contains the reserved ``|`` delimiter (`PYTHON-6040`_). +- Fixed a bug in SRV polling where invalid hosts where topology would not be + updated if one returned host was invalid. .. _PYTHON-6074: https://jira.mongodb.org/browse/PYTHON-6074 +.. _PYTHON-6040: https://jira.mongodb.org/browse/PYTHON-6040 Changes in Version 4.18.2 (2026/09/24) -------------------------------------- diff --git a/drivers-evergreen-tools b/drivers-evergreen-tools new file mode 160000 index 0000000000..ec0b1497b3 --- /dev/null +++ b/drivers-evergreen-tools @@ -0,0 +1 @@ +Subproject commit ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 diff --git a/justfile b/justfile index df678cdbac..9b37109c74 100644 --- a/justfile +++ b/justfile @@ -12,9 +12,12 @@ mypy_args := "--install-types --non-interactive" default: @just --list +# Restore the default development environment. Set JUST_NO_RESYNC to skip the +# sync when a caller resolves dependencies differently, such as the +# minimum-dependencies job in test-mod-wsgi.yml. [private] resync: - @uv sync --quiet + @if [ -z "${JUST_NO_RESYNC:-}" ]; then uv sync --quiet; fi # Set up the development environment install: @@ -85,17 +88,17 @@ test-numpy *args="": && resync # Run tests via the Evergreen test runner script [group('test')] run-tests *args: && resync - bash ./.evergreen/run-tests.sh {{args}} + bash .evergreen/scripts/dispatch.sh run-tests {{args}} # Set up the test environment (auth, TLS, etc.) [group('test')] setup-tests *args="": - bash .evergreen/scripts/setup-tests.sh {{args}} + bash .evergreen/scripts/dispatch.sh setup-tests {{args}} # Tear down resources created by setup-tests [group('test')] -teardown-tests: - bash .evergreen/scripts/teardown-tests.sh +teardown-tests *args="": + bash .evergreen/scripts/dispatch.sh teardown-tests {{args}} [group('test')] integration-tests: @@ -124,6 +127,29 @@ coverage-xml: uv tool run --with "coverage[toml]" coverage xml @echo "Coverage report generated in coverage.xml" +# Run the mod_wsgi tests (natively on Linux, else in an ubuntu container) +[group('test')] +smoke-mod-wsgi: + #!/usr/bin/env bash + set -euo pipefail + STATUS=0 + # Tear down on an early exit (a failed setup or teardown), so a partially + # started Apache or container and the dispatch marker never outlive the + # recipe; the per-mode teardown below clears the guard. + CLEANED=1 + cleanup() { + [ "$CLEANED" -eq 1 ] || just teardown-tests || true + } + trap cleanup EXIT + for mode in standalone embedded; do + CLEANED=0 + just setup-tests mod_wsgi $mode + just run-tests || STATUS=$? + just teardown-tests + CLEANED=1 + [ "$STATUS" -eq 0 ] || exit "$STATUS" + done + # Start a MongoDB server via drivers-evergreen-tools [group('server')] run-server *args="": diff --git a/pymongo/_client_bulk_shared.py b/pymongo/_client_bulk_shared.py index f4f7f8fe7f..0019d0c0bb 100644 --- a/pymongo/_client_bulk_shared.py +++ b/pymongo/_client_bulk_shared.py @@ -18,13 +18,22 @@ from __future__ import annotations from collections.abc import Mapping, MutableMapping -from typing import TYPE_CHECKING, Any, NoReturn +from typing import TYPE_CHECKING, Any, NoReturn, Optional, Union +from bson.objectid import ObjectId +from bson.raw_bson import RawBSONDocument +from pymongo.common import ( + validate_is_document_type, + validate_ok_for_replace, + validate_ok_for_update, +) from pymongo.errors import ClientBulkWriteException, OperationFailure from pymongo.helpers_shared import _get_wce_doc +from pymongo.message import _ClientBulkWriteContext +from pymongo.write_concern import WriteConcern if TYPE_CHECKING: - from pymongo.typings import _DocumentOut + from pymongo.typings import _AgnosticMongoClient, _DocumentOut, _Pipeline def _merge_command( @@ -79,3 +88,145 @@ def _throw_client_bulk_write_exception( if isinstance(full_result["error"], BaseException): raise ClientBulkWriteException(full_result, verbose_results) from full_result["error"] raise ClientBulkWriteException(full_result, verbose_results) + + +class _AgnosticClientBulkBase: + """The private guts of the client-level bulk write API.""" + + def __init__( + self, + client: _AgnosticMongoClient, + write_concern: WriteConcern, + ordered: bool = True, + bypass_document_validation: Optional[bool] = None, + comment: Optional[str] = None, + let: Optional[Any] = None, + verbose_results: bool = False, + ) -> None: + """Initialize a _AgnosticClientBulkBase instance.""" + self.client = client + self.write_concern = write_concern + self.let = let + if self.let is not None: + validate_is_document_type("let", self.let) + self.ordered = ordered + self.bypass_doc_val = bypass_document_validation + self.comment = comment + self.verbose_results = verbose_results + self.ops: list[tuple[str, Mapping[str, Any]]] = [] + self.namespaces: list[str] = [] + self.idx_offset: int = 0 + self.total_ops: int = 0 + self.executed = False + self.uses_collation = False + self.uses_array_filters = False + self.is_retryable = self.client.options.retry_writes + self.retrying = False + self.started_retryable_write = False + + @property + def bulk_ctx_class(self) -> type[_ClientBulkWriteContext]: + return _ClientBulkWriteContext + + def add_insert(self, namespace: str, document: _DocumentOut) -> None: + """Add an insert document to the list of ops.""" + validate_is_document_type("document", document) + # Generate ObjectId client side. + if not (isinstance(document, RawBSONDocument) or "_id" in document): + document["_id"] = ObjectId() + cmd = {"insert": -1, "document": document} + self.ops.append(("insert", cmd)) + self.namespaces.append(namespace) + self.total_ops += 1 + + def add_update( + self, + namespace: str, + selector: Mapping[str, Any], + update: Union[Mapping[str, Any], _Pipeline], + multi: bool, + upsert: Optional[bool] = None, + collation: Optional[Mapping[str, Any]] = None, + array_filters: Optional[list[Mapping[str, Any]]] = None, + hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, + ) -> None: + """Create an update document and add it to the list of ops.""" + validate_ok_for_update(update) + cmd = { + "update": -1, + "filter": selector, + "updateMods": update, + "multi": multi, + } + if upsert is not None: + cmd["upsert"] = upsert + if array_filters is not None: + self.uses_array_filters = True + cmd["arrayFilters"] = array_filters + if hint is not None: + cmd["hint"] = hint + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if sort is not None: + cmd["sort"] = sort + if multi: + # A bulk_write containing an update_many is not retryable. + self.is_retryable = False + self.ops.append(("update", cmd)) + self.namespaces.append(namespace) + self.total_ops += 1 + + def add_replace( + self, + namespace: str, + selector: Mapping[str, Any], + replacement: Mapping[str, Any], + upsert: Optional[bool] = None, + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, + ) -> None: + """Create a replace document and add it to the list of ops.""" + validate_ok_for_replace(replacement) + cmd = { + "update": -1, + "filter": selector, + "updateMods": replacement, + "multi": False, + } + if upsert is not None: + cmd["upsert"] = upsert + if hint is not None: + cmd["hint"] = hint + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if sort is not None: + cmd["sort"] = sort + self.ops.append(("replace", cmd)) + self.namespaces.append(namespace) + self.total_ops += 1 + + def add_delete( + self, + namespace: str, + selector: Mapping[str, Any], + multi: bool, + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + ) -> None: + """Create a delete document and add it to the list of ops.""" + cmd = {"delete": -1, "filter": selector, "multi": multi} + if hint is not None: + cmd["hint"] = hint + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if multi: + # A bulk_write containing a delete_many is not retryable. + self.is_retryable = False + self.ops.append(("delete", cmd)) + self.namespaces.append(namespace) + self.total_ops += 1 diff --git a/pymongo/_psl.py b/pymongo/_psl.py index b8f11f5bd7..e0130188ff 100644 --- a/pymongo/_psl.py +++ b/pymongo/_psl.py @@ -21,6 +21,26 @@ _PUBLIC_SUFFIXES: Optional[tuple[set[str], set[str], set[str]]] = None +# Single labels that srvAllowedHostsSuffix may be set to. See the +# srvAllowedHostsSuffix section of the Initial DNS Seedlist Discovery spec. +SPECIAL_USE_LABELS = frozenset( + [ + # RFC 6761 special use names. + "test", + "localhost", + "invalid", + "example", + # RFC 6762 multicast DNS. + "local", + # Reserved by ICANN for private use. + "internal", + # Not officially reserved by ICANN but commonly used privately. + "corp", + "home", + "mail", + ] +) + def _to_punycode(string: str) -> str: """Convert a string to Punycode.""" @@ -28,7 +48,7 @@ def _to_punycode(string: str) -> str: return string.encode("idna").decode("ascii") except UnicodeError: # Not every input is a valid IDNA label (e.g. empty or over-long - # labels). Leave it as-is and let the caller reject it. + # labels). Return it unchanged -- SRV host will fail verification. return string diff --git a/pymongo/_telemetry.py b/pymongo/_telemetry.py index 499309b336..b8c8fa00ff 100644 --- a/pymongo/_telemetry.py +++ b/pymongo/_telemetry.py @@ -746,6 +746,17 @@ def log_srv_monitor_failure(failure: Exception) -> None: _debug_log(_SDAM_LOGGER, message="SRV monitor check failed", failure=repr(failure)) +def log_srv_monitor_invalid_host(host: str, failure: Exception) -> None: + """Emit a log entry when the SRV monitor skips a host that failed verification.""" + if _is_debug_enabled(_SDAM_LOGGER): + _debug_log( + _SDAM_LOGGER, + message="SRV monitor skipped invalid host", + serverHost=host, + failure=repr(failure), + ) + + def log_command_retry( topology_id: Optional[ObjectId], command_name: str, diff --git a/pymongo/asynchronous/bulk.py b/pymongo/asynchronous/bulk.py index 86b9528b6e..81186499f6 100644 --- a/pymongo/asynchronous/bulk.py +++ b/pymongo/asynchronous/bulk.py @@ -29,9 +29,7 @@ Union, ) -from bson.objectid import ObjectId -from bson.raw_bson import RawBSONDocument -from pymongo import _csot, common +from pymongo import _csot from pymongo._telemetry import _generate_op_id_or_none from pymongo.asynchronous.client_session import AsyncClientSession from pymongo.asynchronous.command_runner import ( @@ -40,17 +38,11 @@ from pymongo.asynchronous.helpers import _handle_reauth from pymongo.bulk_shared import ( _COMMANDS, - _DELETE_ALL, + _AgnosticBulkBase, _merge_command, _raise_bulk_write_error, - _Run, ) from pymongo.client_session_shared import _validate_session_write_concern -from pymongo.common import ( - validate_is_document_type, - validate_ok_for_replace, - validate_ok_for_update, -) from pymongo.errors import ( ConfigurationError, InvalidOperation, @@ -59,7 +51,6 @@ from pymongo.helpers_shared import _RETRYABLE_ERROR_CODES from pymongo.message import ( _DELETE, - _INSERT, _UPDATE, _BulkWriteContext, _EncryptedBulkWriteContext, @@ -71,165 +62,14 @@ from pymongo.asynchronous.collection import AsyncCollection from pymongo.asynchronous.mongo_client import AsyncMongoClient from pymongo.asynchronous.pool import AsyncConnection - from pymongo.typings import _DocumentOut, _DocumentType, _Pipeline _IS_SYNC = False -class _AsyncBulk: +class _AsyncBulk(_AgnosticBulkBase): """The private guts of the bulk write API.""" - def __init__( - self, - collection: AsyncCollection[_DocumentType], - ordered: bool, - bypass_document_validation: Optional[bool], - comment: Optional[str] = None, - let: Optional[Any] = None, - ) -> None: - """Initialize a _AsyncBulk instance.""" - self.collection = collection.with_options( - codec_options=collection.codec_options._replace( - unicode_decode_error_handler="replace", document_class=dict - ) - ) - self.let = let - if self.let is not None: - common.validate_is_document_type("let", self.let) - self.comment: Optional[str] = comment - self.ordered = ordered - self.ops: list[tuple[int, Mapping[str, Any]]] = [] - self.executed = False - self.bypass_doc_val = bypass_document_validation - self.uses_collation = False - self.uses_array_filters = False - self.uses_sort = False - self.is_retryable = True - self.retrying = False - self.started_retryable_write = False - # Extra state so that we know where to pick up on a retry attempt. - self.current_run = None - self.next_run = None - self.is_encrypted = False - - @property - def bulk_ctx_class(self) -> type[_BulkWriteContext]: - encrypter = self.collection.database.client._encrypter - if encrypter and not encrypter._bypass_auto_encryption: - self.is_encrypted = True - return _EncryptedBulkWriteContext - else: - self.is_encrypted = False - return _BulkWriteContext - - def add_insert(self, document: _DocumentOut) -> None: - """Add an insert document to the list of ops.""" - validate_is_document_type("document", document) - # Generate ObjectId client side. - if not (isinstance(document, RawBSONDocument) or "_id" in document): - document["_id"] = ObjectId() - self.ops.append((_INSERT, document)) - - def add_update( - self, - selector: Mapping[str, Any], - update: Union[Mapping[str, Any], _Pipeline], - multi: bool, - upsert: Optional[bool], - collation: Optional[Mapping[str, Any]] = None, - array_filters: Optional[list[Mapping[str, Any]]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create an update document and add it to the list of ops.""" - validate_ok_for_update(update) - cmd: dict[str, Any] = {"q": selector, "u": update, "multi": multi} - if upsert is not None: - cmd["upsert"] = upsert - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if array_filters is not None: - self.uses_array_filters = True - cmd["arrayFilters"] = array_filters - if hint is not None: - cmd["hint"] = hint - if sort is not None: - self.uses_sort = True - cmd["sort"] = sort - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append((_UPDATE, cmd)) - - def add_replace( - self, - selector: Mapping[str, Any], - replacement: Mapping[str, Any], - upsert: Optional[bool], - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create a replace document and add it to the list of ops.""" - validate_ok_for_replace(replacement) - cmd: dict[str, Any] = {"q": selector, "u": replacement} - if upsert is not None: - cmd["upsert"] = upsert - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if hint is not None: - cmd["hint"] = hint - if sort is not None: - self.uses_sort = True - cmd["sort"] = sort - self.ops.append((_UPDATE, cmd)) - - def add_delete( - self, - selector: Mapping[str, Any], - limit: int, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - ) -> None: - """Create a delete document and add it to the list of ops.""" - cmd: dict[str, Any] = {"q": selector, "limit": limit} - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if hint is not None: - cmd["hint"] = hint - if limit == _DELETE_ALL: - # A bulk_write containing a delete_many is not retryable. - self.is_retryable = False - self.ops.append((_DELETE, cmd)) - - def gen_ordered(self) -> Iterator[Optional[_Run]]: - """Generate batches of operations, batched by type of - operation, in the order **provided**. - """ - run = None - for idx, (op_type, operation) in enumerate(self.ops): - if run is None: - run = _Run(op_type) - elif run.op_type != op_type: - yield run - run = _Run(op_type) - run.add(idx, operation) - yield run - - def gen_unordered(self) -> Iterator[_Run]: - """Generate batches of operations, batched by type of - operation, in arbitrary order. - """ - operations = [_Run(_INSERT), _Run(_UPDATE), _Run(_DELETE)] - for idx, (op_type, operation) in enumerate(self.ops): - operations[op_type].add(idx, operation) - - for run in operations: - if run.ops: - yield run + collection: AsyncCollection[Any] @_handle_reauth async def write_command( diff --git a/pymongo/asynchronous/client_bulk.py b/pymongo/asynchronous/client_bulk.py index 2847dcb2c0..bfbe9e2c6b 100644 --- a/pymongo/asynchronous/client_bulk.py +++ b/pymongo/asynchronous/client_bulk.py @@ -29,9 +29,7 @@ Union, ) -from bson.objectid import ObjectId -from bson.raw_bson import RawBSONDocument -from pymongo import _csot, common +from pymongo import _csot from pymongo._telemetry import _generate_op_id_or_none from pymongo.asynchronous.client_session import AsyncClientSession from pymongo.asynchronous.collection import AsyncCollection @@ -46,15 +44,11 @@ from pymongo.asynchronous.mongo_client import AsyncMongoClient from pymongo.asynchronous.pool import AsyncConnection from pymongo._client_bulk_shared import ( + _AgnosticClientBulkBase, _merge_command, _throw_client_bulk_write_exception, ) from pymongo.client_session_shared import _validate_session_write_concern -from pymongo.common import ( - validate_is_document_type, - validate_ok_for_replace, - validate_ok_for_update, -) from pymongo.errors import ( ConfigurationError, ConnectionFailure, @@ -76,152 +70,15 @@ InsertOneResult, UpdateResult, ) -from pymongo.typings import _DocumentOut, _Pipeline from pymongo.write_concern import WriteConcern _IS_SYNC = False -class _AsyncClientBulk: +class _AsyncClientBulk(_AgnosticClientBulkBase): """The private guts of the client-level bulk write API.""" - def __init__( - self, - client: AsyncMongoClient[Any], - write_concern: WriteConcern, - ordered: bool = True, - bypass_document_validation: Optional[bool] = None, - comment: Optional[str] = None, - let: Optional[Any] = None, - verbose_results: bool = False, - ) -> None: - """Initialize a _AsyncClientBulk instance.""" - self.client = client - self.write_concern = write_concern - self.let = let - if self.let is not None: - common.validate_is_document_type("let", self.let) - self.ordered = ordered - self.bypass_doc_val = bypass_document_validation - self.comment = comment - self.verbose_results = verbose_results - self.ops: list[tuple[str, Mapping[str, Any]]] = [] - self.namespaces: list[str] = [] - self.idx_offset: int = 0 - self.total_ops: int = 0 - self.executed = False - self.uses_collation = False - self.uses_array_filters = False - self.is_retryable = self.client.options.retry_writes - self.retrying = False - self.started_retryable_write = False - - @property - def bulk_ctx_class(self) -> type[_ClientBulkWriteContext]: - return _ClientBulkWriteContext - - def add_insert(self, namespace: str, document: _DocumentOut) -> None: - """Add an insert document to the list of ops.""" - validate_is_document_type("document", document) - # Generate ObjectId client side. - if not (isinstance(document, RawBSONDocument) or "_id" in document): - document["_id"] = ObjectId() - cmd = {"insert": -1, "document": document} - self.ops.append(("insert", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_update( - self, - namespace: str, - selector: Mapping[str, Any], - update: Union[Mapping[str, Any], _Pipeline], - multi: bool, - upsert: Optional[bool] = None, - collation: Optional[Mapping[str, Any]] = None, - array_filters: Optional[list[Mapping[str, Any]]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create an update document and add it to the list of ops.""" - validate_ok_for_update(update) - cmd = { - "update": -1, - "filter": selector, - "updateMods": update, - "multi": multi, - } - if upsert is not None: - cmd["upsert"] = upsert - if array_filters is not None: - self.uses_array_filters = True - cmd["arrayFilters"] = array_filters - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if sort is not None: - cmd["sort"] = sort - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append(("update", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_replace( - self, - namespace: str, - selector: Mapping[str, Any], - replacement: Mapping[str, Any], - upsert: Optional[bool] = None, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create a replace document and add it to the list of ops.""" - validate_ok_for_replace(replacement) - cmd = { - "update": -1, - "filter": selector, - "updateMods": replacement, - "multi": False, - } - if upsert is not None: - cmd["upsert"] = upsert - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if sort is not None: - cmd["sort"] = sort - self.ops.append(("replace", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_delete( - self, - namespace: str, - selector: Mapping[str, Any], - multi: bool, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - ) -> None: - """Create a delete document and add it to the list of ops.""" - cmd = {"delete": -1, "filter": selector, "multi": multi} - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append(("delete", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 + client: AsyncMongoClient[Any] @_handle_reauth async def write_command( diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index 9ba2758f78..4ae63716e3 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -17,8 +17,11 @@ from __future__ import annotations import asyncio +import contextlib import functools +import inspect import socket +import ssl import time as time # noqa: PLC0414 # needed in sync version import uuid import weakref @@ -60,7 +63,9 @@ from pymongo.common import CONNECT_TIMEOUT from pymongo.daemon import _spawn_daemon from pymongo.encryption_options import ( + AsyncKMSConnectCallback, AutoEncryptionOpts, + KMSConnectContext, RangeOpts, StringOpts, # Re-exported for backwards compatibility: TextOpts is deprecated but must @@ -90,6 +95,8 @@ from pymongo.pool_options import PoolOptions from pymongo.pool_shared import ( _async_configured_socket, + _async_wrap_socket_tls, + _close_late_socket, _raise_connection_failure, ) from pymongo.read_concern import ReadConcern @@ -120,11 +127,108 @@ _KEY_VAULT_OPTS = CodecOptions(document_class=RawBSONDocument) -async def _connect_kms(address: _Address, opts: PoolOptions) -> Union[socket.socket, _sslConn]: +def _close_rejected_kms_socket(obj: Any) -> None: + """Close a callback return value that failed validation, best effort. + + ``_connect_kms`` raises on a contract violation instead of returning the + value, so no caller ever takes ownership of it. Close it here, tolerating + non-socket values and ``close()`` failures. + """ + close = getattr(obj, "close", None) + if callable(close): + with contextlib.suppress(Exception): + close() + + +async def _connect_kms( + address: _Address, + opts: PoolOptions, + kms_connect_callback: Optional[AsyncKMSConnectCallback], + timeout: float, +) -> Union[socket.socket, _sslConn]: + """Connect to a KMS host and perform the TLS handshake over the socket. + + Uses ``kms_connect_callback`` when one is provided, otherwise connects + directly, and always verifies against ``address`` (the KMS host). + """ + if kms_connect_callback is None: + try: + return await _async_configured_socket(address, opts) + except Exception as exc: + _raise_connection_failure(address, exc, timeout_details=_get_timeout_details(opts)) + + # TLS targets address, not the peer, so verification follows the KMS host. + # A plain callable would block the event loop before we could reject it, + # so check the callback first. + if not _IS_SYNC: + callback_any: Any = kms_connect_callback + is_coro = inspect.iscoroutinefunction(callback_any) + if not is_coro and callable(callback_any): + is_coro = inspect.iscoroutinefunction(callback_any.__call__) + if not is_coro: + raise ConfigurationError( + "kms_connect_callback must be a coroutine function for the async API." + ) + # Any: the sync version's callback returns a plain socket. + result: Any = kms_connect_callback( + KMSConnectContext(host=address[0], port=cast(int, address[1]), timeout=timeout) + ) + remaining = _csot.remaining() + if remaining is None or _IS_SYNC: + # The synchronous API cannot interrupt a callback that has started + # running; honoring the deadline is the callback's contract there. + sock = await result + else: + # CSOT is cooperative: a callback that ignores the timeout could block + # past the deadline. Shield the task so stopping the wait does not + # cancel it mid-flight, and close any socket it yields later. + task = asyncio.ensure_future(result) + try: + sock = await asyncio.wait_for(asyncio.shield(task), remaining) + except asyncio.CancelledError: + task.add_done_callback(_close_late_socket) + raise + except asyncio.TimeoutError: + task.add_done_callback(_close_late_socket) + _raise_connection_failure( + address, + socket.timeout("timed out"), + timeout_details=_get_timeout_details(opts), + ) + if not isinstance(sock, socket.socket) or isinstance(sock, ssl.SSLSocket): + _close_rejected_kms_socket(sock) + raise ConfigurationError( + "kms_connect_callback must return a connected, unwrapped " + f"socket.socket, not {type(sock)}." + ) + # wrap_socket refuses a non-blocking socket, so normalize the mode here. + try: + sock.getpeername() + except OSError: + _close_rejected_kms_socket(sock) + raise ConfigurationError( + "kms_connect_callback must return an already connected socket." + ) from None + if sock.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE) != socket.SOCK_STREAM: + _close_rejected_kms_socket(sock) + raise ConfigurationError( + "kms_connect_callback must return a stream socket, not a datagram one." + ) + # The callback may have consumed much of the CSOT budget, and wrapping + # resets the socket timeout, so recompute the remaining time here and for + # the KMS request that follows. + sock.settimeout(max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001)) try: - return await _async_configured_socket(address, opts) + conn = await _async_wrap_socket_tls(sock, address, opts) + except asyncio.CancelledError: + # The executor may still be wrapping the socket; close it so a TLS + # proxy's relay threads wind down instead of leaking. + sock.close() + raise except Exception as exc: _raise_connection_failure(address, exc, timeout_details=_get_timeout_details(opts)) + conn.settimeout(max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001)) + return conn class _EncryptionIO(AsyncMongoCryptCallback): # type: ignore[misc] @@ -179,13 +283,6 @@ async def kms_request(self, kms_context: MongoCryptKmsContext) -> None: False, # disable_ocsp_endpoint_check _IS_SYNC, ) - # CSOT: set timeout for socket creation. - connect_timeout = max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001) - opts = PoolOptions( - connect_timeout=connect_timeout, - socket_timeout=connect_timeout, - ssl_context=ctx, - ) address = parse_host(endpoint, _HTTPS_PORT) if address[0].endswith(".sock"): raise ConfigurationError(f"Invalid KMS endpoint {endpoint!r}") @@ -193,8 +290,21 @@ async def kms_request(self, kms_context: MongoCryptKmsContext) -> None: if sleep_u: sleep_sec = float(sleep_u) / 1e6 await asyncio.sleep(sleep_sec) + # Set the connect timeout after the retry backoff so the budget + # reflects the sleep. + connect_timeout = max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001) + opts = PoolOptions( + connect_timeout=connect_timeout, + socket_timeout=connect_timeout, + ssl_context=ctx, + ) try: - conn = await _connect_kms(address, opts) + conn = await _connect_kms( + address, + opts, + self.opts._kms_connect_callback, + connect_timeout, + ) try: await async_socket_sendall(conn, message) while kms_context.bytes_needed > 0: @@ -230,6 +340,8 @@ async def kms_request(self, kms_context: MongoCryptKmsContext) -> None: conn.close() except MongoCryptError: raise # Propagate MongoCryptError errors directly. + except ConfigurationError: + raise # A callback contract violation is not transient. except Exception as exc: remaining = _csot.remaining() if isinstance(exc, NetworkTimeout) or (remaining is not None and remaining <= 0): @@ -526,6 +638,7 @@ def __init__( codec_options: CodecOptions[_DocumentTypeArg], kms_tls_options: Optional[Mapping[str, Any]] = None, key_expiration_ms: Optional[int] = None, + kms_connect_callback: Optional[AsyncKMSConnectCallback] = None, ) -> None: """Explicit client-side field level encryption. @@ -595,7 +708,18 @@ def __init__( :param key_expiration_ms: The cache expiration time for data encryption keys. Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000. Set to 0 to disable key expiration. - + :param kms_connect_callback: A callable that opens the connection to a + KMS host, e.g. to route KMS requests through a proxy. It receives + a :class:`~pymongo.encryption_options.KMSConnectContext` and + returns a connected, unwrapped :class:`socket.socket`, over which + the driver performs the KMS TLS handshake. + Must be a coroutine function for the asynchronous API. + The callback is responsible for honoring ``context.timeout``. + Defaults to ``None``, meaning the driver connects to KMS hosts + directly. + + .. versionchanged:: 4.19 + Added the `kms_connect_callback` parameter. .. versionchanged:: 4.12 Added the `key_expiration_ms` parameter. .. versionchanged:: 4.0 @@ -639,6 +763,7 @@ def __init__( key_vault_namespace, kms_tls_options=kms_tls_options, key_expiration_ms=key_expiration_ms, + kms_connect_callback=kms_connect_callback, ) self._kms_ssl_contexts = _parse_kms_tls_options(opts._kms_tls_options, _IS_SYNC) self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO( diff --git a/pymongo/asynchronous/mongo_client.py b/pymongo/asynchronous/mongo_client.py index a0f1397a4d..65aecce446 100644 --- a/pymongo/asynchronous/mongo_client.py +++ b/pymongo/asynchronous/mongo_client.py @@ -470,6 +470,31 @@ def __init__( srvAllowedHostsSuffix=".internal.example.com", ) + - `srv_host_validator`: (callable or None) A callback used in place of the + default parent-domain check for hosts returned by SRV DNS records. It is + called once per returned host with the lowercased hostname as its only + argument, and must return ``True`` to accept the host or ``False`` to + reject it. During initial DNS discovery, rejecting a host raises + :exc:`~pymongo.errors.ConfigurationError`, as does an exception raised by + the callback itself. Use this when the set of acceptable hosts cannot be + expressed as a single suffix:: + + def validator(host: str) -> bool: + return host.endswith((".a.example.com", ".b.example.com")) + + AsyncMongoClient( + "mongodb+srv://cluster.example.com/", + srv_host_validator=validator, + ) + + The callback must not block. It is mutually exclusive with + ``srvAllowedHostsSuffix``. Because this option is a callable, it can + only be passed in as a keyword argument, not through the connection string. + + .. warning:: + + This option replaces the built-in DNS spoofing safeguards. + Please use with caution. | **Write Concern options:** | (Only set if passed. No default values.) @@ -637,6 +662,12 @@ def __init__( .. seealso:: The MongoDB documentation on `connections `_. + .. versionchanged:: 4.19 + Added the ``srv_host_validator`` keyword argument. + + .. versionchanged:: 4.18 + Added the ``srvAllowedHostsSuffix`` URI and keyword argument. + .. versionchanged:: 4.17 Added the ``max_adaptive_retries`` and ``enable_overload_retargeting`` URI and keyword arguments. @@ -823,6 +854,7 @@ def __init__( srv_service_name = keyword_opts.get("srvservicename") srv_max_hosts = keyword_opts.get("srvmaxhosts") srv_allowed_hosts_suffix = keyword_opts.get("srvallowedhostssuffix") + srv_host_validator = keyword_opts.get("srv_host_validator") if len([h for h in self._host if "/" in h]) > 1: raise ConfigurationError("host must not contain multiple MongoDB URIs") for entity in self._host: @@ -915,7 +947,11 @@ def __init__( ) self._init_based_on_options( - self._seeds, srv_max_hosts, srv_service_name, srv_allowed_hosts_suffix + self._seeds, + srv_max_hosts, + srv_service_name, + srv_allowed_hosts_suffix, + srv_host_validator, ) self._opened = False @@ -935,6 +971,7 @@ async def _resolve_srv(self) -> None: srv_service_name = keyword_opts.get("srvservicename") srv_max_hosts = keyword_opts.get("srvmaxhosts") srv_allowed_hosts_suffix = keyword_opts.get("srvallowedhostssuffix") + srv_host_validator = keyword_opts.get("srv_host_validator") for entity in self._host: # A hostname can only include a-z, 0-9, '-' and '.'. If we find a '/' # it must be a URI, @@ -956,6 +993,7 @@ async def _resolve_srv(self) -> None: srv_service_name=srv_service_name, srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, + srv_host_validator=srv_host_validator, ) seeds.update(res["nodelist"]) opts = res["options"] @@ -1000,7 +1038,11 @@ async def _resolve_srv(self) -> None: ) self._init_based_on_options( - seeds, srv_max_hosts, srv_service_name, srv_allowed_hosts_suffix + seeds, + srv_max_hosts, + srv_service_name, + srv_allowed_hosts_suffix, + srv_host_validator, ) def _init_based_on_options( @@ -1009,7 +1051,17 @@ def _init_based_on_options( srv_max_hosts: Any, srv_service_name: Any, srv_allowed_hosts_suffix: Any, + srv_host_validator: Any = None, ) -> None: + if srv_host_validator is not None: + if srv_allowed_hosts_suffix is not None: + raise ConfigurationError( + "Cannot specify both srv_host_validator and srvAllowedHostsSuffix" + ) + if not self._resolve_srv_info["is_srv"]: + raise ConfigurationError( + "The srv_host_validator option is only allowed with 'mongodb+srv://' URIs" + ) self._event_listeners = self._options.pool_options._event_listeners self._topology_settings = TopologySettings( seeds=seeds, @@ -1028,6 +1080,7 @@ def _init_based_on_options( srv_service_name=srv_service_name, srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, + srv_host_validator=srv_host_validator, server_monitoring_mode=self._options.server_monitoring_mode, topology_id=self._topology_settings._topology_id if self._topology_settings else None, ) diff --git a/pymongo/asynchronous/monitor.py b/pymongo/asynchronous/monitor.py index da261e37ea..8ed4078fd7 100644 --- a/pymongo/asynchronous/monitor.py +++ b/pymongo/asynchronous/monitor.py @@ -368,6 +368,7 @@ async def _get_seedlist(self) -> Optional[list[tuple[str, Any]]]: self._settings.pool_options.connect_timeout, self._settings.srv_service_name, srv_allowed_hosts_suffix=self._settings.srv_allowed_hosts_suffix, + srv_host_validator=self._settings.srv_host_validator, ) seedlist, ttl = await resolver.get_hosts_and_min_ttl() if len(seedlist) == 0: diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 69ac14e7e4..b7f7a1f6a0 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -106,6 +106,13 @@ _IS_SYNC = False +# Flags recording the counters a checkout has incremented, so a failed +# checkout can restore them exactly once (PYTHON-6136). +_UNDO_OPERATION_COUNT = 1 +_UNDO_REQUESTS = 2 +_UNDO_SOCKETS = 4 +_UNDO_PENDING = 8 + class AsyncConnection(_ConnectionTelemetryInfo): """Store a connection with some metadata. @@ -969,8 +976,12 @@ async def _get_conn( "Attempted to check out a connection from closed connection pool" ) + # Every counter increment sets a flag in ``applied`` so a failed + # checkout can restore them exactly once (PYTHON-6136). + applied = 0 async with self.lock: self.operation_count += 1 + applied |= _UNDO_OPERATION_COUNT # Get a free socket or create one. if _csot.get_timeout(): @@ -980,28 +991,32 @@ async def _get_conn( else: deadline = None - async with self.size_cond: - self._raise_if_not_ready(checkout_started_time, emit_event=True) - while not (self.requests < self.max_pool_size): - timeout = deadline - time.monotonic() if deadline else None - if not await _async_cond_wait(self.size_cond, timeout): - # Timed out, notify the next thread to ensure a - # timeout doesn't consume the condition. - if self.requests < self.max_pool_size: - self.size_cond.notify() - self._raise_wait_queue_timeout(checkout_started_time) + try: + async with self.size_cond: self._raise_if_not_ready(checkout_started_time, emit_event=True) - self.requests += 1 + while not (self.requests < self.max_pool_size): + timeout = deadline - time.monotonic() if deadline else None + if not await _async_cond_wait(self.size_cond, timeout): + # Timed out, notify the next thread to ensure a + # timeout doesn't consume the condition. + if self.requests < self.max_pool_size: + self.size_cond.notify() + self._raise_wait_queue_timeout(checkout_started_time) + self._raise_if_not_ready(checkout_started_time, emit_event=True) + self.requests += 1 + applied |= _UNDO_REQUESTS + except BaseException: + await self._restore_counters(applied) + raise # We've now acquired the semaphore and must release it on error. conn = None - incremented = False emitted_event = False is_new_conn = False try: async with self.lock: self.active_sockets += 1 - incremented = True + applied |= _UNDO_SOCKETS while conn is None: # CMAP: we MUST wait for either maxConnecting OR for a socket # to be checked back into the pool. @@ -1022,6 +1037,7 @@ async def _get_conn( conn = self.conns.popleft() except IndexError: self._pending += 1 + applied |= _UNDO_PENDING if conn: # We got a socket from the pool if await self._perished(conn): conn = None @@ -1033,7 +1049,17 @@ async def _get_conn( finally: async with self._max_connecting_cond: self._pending -= 1 - self._max_connecting_cond.notify() + applied &= ~_UNDO_PENDING + notified = False + try: + self._max_connecting_cond.notify() + notified = True + finally: + if not notified: + # A kill landed inside notify() (a gevent + # yield point); retry so a waiting + # checkout is not stranded (PYTHON-6136). + self._max_connecting_cond.notify() conn.active = True # connect() already adds cancel_context for new connections; only add @@ -1043,27 +1069,14 @@ async def _get_conn( self.active_contexts.add(conn.cancel_context) # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. except BaseException: - if conn: - # We checked out a socket but authentication failed. - await conn.close_conn(ConnectionClosedReason.ERROR) - # Re-apply the accounting if a GreenletExit interrupts - # during the size_cond acquisition; during unwind gevent - # lets the re-acquire complete (PYTHON-6074). - accounted = False try: - async with self.size_cond: - self.requests -= 1 - if incremented: - self.active_sockets -= 1 - accounted = True - self.size_cond.notify() + if conn is not None: + # We checked out a socket but authentication failed. + await conn.close_conn(ConnectionClosedReason.ERROR) finally: - if not accounted: - async with self.size_cond: - self.requests -= 1 - if incremented: - self.active_sockets -= 1 - self.size_cond.notify() + # Restore the counters even if the cleanup above was + # interrupted (PYTHON-6136). + await self._restore_counters(applied) if not emitted_event: self._telemetry.checkout_failed( @@ -1075,6 +1088,51 @@ async def _get_conn( return conn + def _restore_applied(self, applied: int) -> None: + """Restore the counters flagged in ``applied``. Caller holds ``size_cond``.""" + if applied & _UNDO_OPERATION_COUNT: + self.operation_count -= 1 + if applied & _UNDO_REQUESTS: + self.requests -= 1 + if applied & _UNDO_SOCKETS: + self.active_sockets -= 1 + if applied & _UNDO_PENDING: + self._pending -= 1 + + async def _restore_counters(self, applied: int) -> None: + """Restore the counters a failed checkout incremented (PYTHON-6136). + + Gevent grants the re-acquire during unwind (PYTHON-6074). A kill can + also land inside notify() (a yield point), so notifications are + tracked and retried separately from the counter restore. + """ + accounted = False + notified = 0 + try: + async with self.size_cond: + self._restore_applied(applied) + accounted = True + if applied & _UNDO_REQUESTS: + # A pool slot was freed; wake the next waiting thread. + self.size_cond.notify() + notified |= _UNDO_REQUESTS + if applied & _UNDO_PENDING: + # A maxConnecting slot was freed; wake the next waiting thread. + self._max_connecting_cond.notify() + notified |= _UNDO_PENDING + finally: + # Always reacquired: `applied` keeps restore-only flags + # `notified` never tracks, so skipping needs a mask synced + # to the notify sites; uncontended acquires don't yield (PYTHON-6136). + async with self.size_cond: + if not accounted: + self._restore_applied(applied) + missing = applied & ~notified + if missing & _UNDO_REQUESTS: + self.size_cond.notify() + if missing & _UNDO_PENDING: + self._max_connecting_cond.notify() + def _checkin_apply( self, conn: AsyncConnection, txn: bool, cursor: bool, forked: bool ) -> tuple[Optional[str], bool, bool]: @@ -1121,9 +1179,7 @@ async def checkin(self, conn: AsyncConnection) -> None: conn.pinned_cursor = False self._pinned_sockets.discard(conn) forked = self.pid != os.getpid() - # Re-apply the accounting if a gevent GreenletExit interrupts during - # the size_cond acquisition; gevent lets the re-acquire complete while - # unwinding (PYTHON-6074). + # Re-apply the accounting if a BaseException interrupts here (PYTHON-6074). close_conn_reason: Optional[str] = None emit_closed = False accounted = False diff --git a/pymongo/asynchronous/settings.py b/pymongo/asynchronous/settings.py index bdfb334ab7..4bc5f05f28 100644 --- a/pymongo/asynchronous/settings.py +++ b/pymongo/asynchronous/settings.py @@ -18,7 +18,7 @@ import threading from collections.abc import Collection -from typing import Optional +from typing import Callable, Optional from bson.objectid import ObjectId from pymongo import common @@ -51,6 +51,7 @@ def __init__( srv_service_name: str = common.SRV_SERVICE_NAME, srv_max_hosts: int = 0, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, server_monitoring_mode: str = common.SERVER_MONITORING_MODE, topology_id: Optional[ObjectId] = None, ): @@ -77,6 +78,7 @@ def __init__( srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix, + srv_host_validator, server_monitoring_mode, topology_id, ) diff --git a/pymongo/asynchronous/srv_resolver.py b/pymongo/asynchronous/srv_resolver.py index 5a068dbe72..554d226acf 100644 --- a/pymongo/asynchronous/srv_resolver.py +++ b/pymongo/asynchronous/srv_resolver.py @@ -18,9 +18,10 @@ import ipaddress import random -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Union -from pymongo._psl import is_public_suffix +from pymongo._psl import SPECIAL_USE_LABELS, _to_punycode, is_public_suffix +from pymongo._telemetry import log_srv_monitor_invalid_host from pymongo.common import CONNECT_TIMEOUT from pymongo.errors import ConfigurationError @@ -64,21 +65,40 @@ def __init__( srv_service_name: str, srv_max_hosts: int = 0, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ): self.__fqdn = fqdn.lower() self.__srv = srv_service_name self.__connect_timeout = connect_timeout or CONNECT_TIMEOUT self.__srv_max_hosts = srv_max_hosts or 0 - self.__srv_allowed_hosts_suffix = ( - "." + srv_allowed_hosts_suffix.lower().strip(".") if srv_allowed_hosts_suffix else None - ) # ensure there's a . at the beginning of the domain - if self.__srv_allowed_hosts_suffix is not None and is_public_suffix( - self.__srv_allowed_hosts_suffix - ): + self.__srv_host_validator = srv_host_validator + # parse_uri() can call this constructor independently of AsyncMongoClient. + if srv_host_validator is not None and srv_allowed_hosts_suffix is not None: raise ConfigurationError( - f"srvAllowedHostsSuffix must not be a public suffix, got: {srv_allowed_hosts_suffix}" + "Cannot specify both srv_host_validator and srvAllowedHostsSuffix" ) - # Validate the fully qualified domain name. + self.__srv_allowed_hosts_suffix = None + if srv_allowed_hosts_suffix is not None: + suffix = srv_allowed_hosts_suffix.strip(".") + if not suffix: + raise ConfigurationError( + f"srvAllowedHostsSuffix must not be empty, got: {srv_allowed_hosts_suffix!r}" + ) + suffix = _to_punycode(suffix).lower() + + is_special_use = suffix in SPECIAL_USE_LABELS + if len(suffix.split(".")) < 2 and not is_special_use: + raise ConfigurationError( + "srvAllowedHostsSuffix must contain at least two '.' separated labels, " + f"got: {srv_allowed_hosts_suffix}" + ) + + if not is_special_use and is_public_suffix(suffix): + raise ConfigurationError( + f"srvAllowedHostsSuffix must not be a public suffix, got: {srv_allowed_hosts_suffix}" + ) + self.__srv_allowed_hosts_suffix = "." + suffix + try: ipaddress.ip_address(fqdn) raise ConfigurationError(_INVALID_HOST_MSG % ("an IP address",)) @@ -106,23 +126,51 @@ async def get_options(self) -> Optional[str]: raise ConfigurationError("Only one TXT record is supported") return (b"&".join([b"".join(res.strings) for res in results])).decode("utf-8") # type: ignore[attr-defined] - async def _resolve_uri(self, encapsulate_errors: bool) -> resolver.Answer: + async def _resolve_uri(self, is_polling: bool) -> resolver.Answer: try: results = await _resolve( "_" + self.__srv + "._tcp." + self.__fqdn, "SRV", lifetime=self.__connect_timeout ) except Exception as exc: - if not encapsulate_errors: + if is_polling: # Raise the original error. raise # Else, raise all errors as ConfigurationError. raise ConfigurationError(str(exc)) from exc return results + def _validate_host(self, srv_host: str) -> None: + """Raise ConfigurationError if an SRV-returned host fails verification.""" + if self.__srv_host_validator is not None: + try: + allowed = self.__srv_host_validator(srv_host) + except Exception as exc: + raise ConfigurationError( + f"srv_host_validator raised an exception for SRV host {srv_host}: {exc}" + ) from exc + if not allowed: + raise ConfigurationError( + f"Invalid SRV host: {srv_host} was rejected by srv_host_validator" + ) + elif self.__srv_allowed_hosts_suffix is not None: + if not srv_host.endswith(self.__srv_allowed_hosts_suffix): + raise ConfigurationError(f"Invalid SRV host: {srv_host}") + else: + if self.__fqdn == srv_host and self.nparts < 3: + raise ConfigurationError( + "Invalid SRV host: return address is identical to SRV hostname" + ) + try: + nlist = srv_host.split(".")[1:][-self.__slen :] + except Exception as exc: + raise ConfigurationError(f"Invalid SRV host: {srv_host}") from exc + if self.__plist != nlist: + raise ConfigurationError(f"Invalid SRV host: {srv_host}") + async def _get_srv_response_and_hosts( - self, encapsulate_errors: bool + self, is_polling: bool ) -> tuple[resolver.Answer, list[tuple[str, Any]]]: - results = await self._resolve_uri(encapsulate_errors) + results = await self._resolve_uri(is_polling) # Construct address tuples nodes = [ @@ -130,33 +178,28 @@ async def _get_srv_response_and_hosts( for res in results ] - # Validate hosts + # Validate hosts. + valid_nodes = [] for node in nodes: - srv_host = node[0].lower() - if self.__fqdn == srv_host and self.nparts < 3: - raise ConfigurationError( - "Invalid SRV host: return address is identical to SRV hostname" - ) - if self.__srv_allowed_hosts_suffix is not None: - if not srv_host.endswith(self.__srv_allowed_hosts_suffix): - raise ConfigurationError(f"Invalid SRV host: {node[0]}") + try: + self._validate_host(node[0]) + except ConfigurationError as exc: + if not is_polling: + raise + log_srv_monitor_invalid_host(node[0], exc) else: - try: - nlist = srv_host.split(".")[1:][-self.__slen :] - except Exception as exc: - raise ConfigurationError(f"Invalid SRV host: {node[0]}") from exc - if self.__plist != nlist: - raise ConfigurationError(f"Invalid SRV host: {node[0]}") + valid_nodes.append(node) + nodes = valid_nodes if self.__srv_max_hosts: nodes = random.sample(nodes, min(self.__srv_max_hosts, len(nodes))) return results, nodes async def get_hosts(self) -> list[tuple[str, Any]]: - _, nodes = await self._get_srv_response_and_hosts(True) + _, nodes = await self._get_srv_response_and_hosts(is_polling=False) return nodes async def get_hosts_and_min_ttl(self) -> tuple[list[tuple[str, Any]], int]: - results, nodes = await self._get_srv_response_and_hosts(False) + results, nodes = await self._get_srv_response_and_hosts(is_polling=True) rrset = results.rrset ttl = rrset.ttl if rrset else 0 return nodes, ttl diff --git a/pymongo/asynchronous/uri_parser.py b/pymongo/asynchronous/uri_parser.py index 24d3427843..c42add6ae8 100644 --- a/pymongo/asynchronous/uri_parser.py +++ b/pymongo/asynchronous/uri_parser.py @@ -17,7 +17,7 @@ from __future__ import annotations -from typing import Any, Optional +from typing import Any, Callable, Optional from pymongo.asynchronous.srv_resolver import _SrvResolver from pymongo.common import SRV_SERVICE_NAME, _CaseInsensitiveDictionary @@ -48,6 +48,7 @@ async def parse_uri( srv_service_name: Optional[str] = None, srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ) -> dict[str, Any]: """Parse and validate a MongoDB URI. @@ -117,6 +118,7 @@ async def parse_uri( srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix, + srv_host_validator, ) ) result["options"] = _make_options_case_sensitive(result["options"]) @@ -133,6 +135,7 @@ async def _parse_srv( srv_service_name: Optional[str] = None, srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ) -> dict[str, Any]: if uri.startswith(SCHEME): is_srv = False @@ -168,7 +171,12 @@ async def _parse_srv( # argument overrides the same option passed in the connection string. connect_timeout = connect_timeout or options.get("connectTimeoutMS") dns_resolver = _SrvResolver( - fqdn, connect_timeout, srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix + fqdn, + connect_timeout, + srv_service_name, + srv_max_hosts, + srv_allowed_hosts_suffix, + srv_host_validator, ) nodes = await dns_resolver.get_hosts() dns_options = await dns_resolver.get_options() diff --git a/pymongo/bulk_shared.py b/pymongo/bulk_shared.py index a086611be5..54f0d4cd22 100644 --- a/pymongo/bulk_shared.py +++ b/pymongo/bulk_shared.py @@ -17,19 +17,28 @@ from __future__ import annotations -from collections.abc import Mapping, MutableMapping -from typing import TYPE_CHECKING, Any, NoReturn +from collections.abc import Iterator, Mapping, MutableMapping +from typing import TYPE_CHECKING, Any, NoReturn, Optional, Union +from bson.objectid import ObjectId +from bson.raw_bson import RawBSONDocument +from pymongo.common import ( + validate_is_document_type, + validate_ok_for_replace, + validate_ok_for_update, +) from pymongo.errors import BulkWriteError, OperationFailure from pymongo.helpers_shared import _get_wce_doc from pymongo.message import ( _DELETE, _INSERT, _UPDATE, + _BulkWriteContext, + _EncryptedBulkWriteContext, ) if TYPE_CHECKING: - from pymongo.typings import _DocumentOut + from pymongo.typings import _AgnosticCollection, _DocumentOut, _DocumentType, _Pipeline _DELETE_ALL: int = 0 @@ -131,3 +140,159 @@ def _raise_bulk_write_error(full_result: _DocumentOut) -> NoReturn: ) raise OperationFailure(errmsg, code, full_result) raise BulkWriteError(full_result) + + +class _AgnosticBulkBase: + """The private guts of the bulk write API.""" + + def __init__( + self, + collection: _AgnosticCollection[_DocumentType], + ordered: bool, + bypass_document_validation: Optional[bool], + comment: Optional[str] = None, + let: Optional[Any] = None, + ) -> None: + """Initialize an _AgnosticBulkBase instance.""" + self.collection = collection.with_options( + codec_options=collection.codec_options._replace( + unicode_decode_error_handler="replace", document_class=dict + ) + ) + self.let = let + if self.let is not None: + validate_is_document_type("let", self.let) + self.comment: Optional[str] = comment + self.ordered = ordered + self.ops: list[tuple[int, Mapping[str, Any]]] = [] + self.executed = False + self.bypass_doc_val = bypass_document_validation + self.uses_collation = False + self.uses_array_filters = False + self.uses_sort = False + self.is_retryable = True + self.retrying = False + self.started_retryable_write = False + # Extra state so that we know where to pick up on a retry attempt. + self.current_run = None + self.next_run = None + self.is_encrypted = False + + @property + def bulk_ctx_class(self) -> type[_BulkWriteContext]: + encrypter = self.collection.database.client._encrypter + if encrypter and not encrypter._bypass_auto_encryption: + self.is_encrypted = True + return _EncryptedBulkWriteContext + else: + self.is_encrypted = False + return _BulkWriteContext + + def add_insert(self, document: _DocumentOut) -> None: + """Add an insert document to the list of ops.""" + validate_is_document_type("document", document) + # Generate ObjectId client side. + if not (isinstance(document, RawBSONDocument) or "_id" in document): + document["_id"] = ObjectId() + self.ops.append((_INSERT, document)) + + def add_update( + self, + selector: Mapping[str, Any], + update: Union[Mapping[str, Any], _Pipeline], + multi: bool, + upsert: Optional[bool], + collation: Optional[Mapping[str, Any]] = None, + array_filters: Optional[list[Mapping[str, Any]]] = None, + hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, + ) -> None: + """Create an update document and add it to the list of ops.""" + validate_ok_for_update(update) + cmd: dict[str, Any] = {"q": selector, "u": update, "multi": multi} + if upsert is not None: + cmd["upsert"] = upsert + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if array_filters is not None: + self.uses_array_filters = True + cmd["arrayFilters"] = array_filters + if hint is not None: + cmd["hint"] = hint + if sort is not None: + self.uses_sort = True + cmd["sort"] = sort + if multi: + # A bulk_write containing an update_many is not retryable. + self.is_retryable = False + self.ops.append((_UPDATE, cmd)) + + def add_replace( + self, + selector: Mapping[str, Any], + replacement: Mapping[str, Any], + upsert: Optional[bool], + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + sort: Optional[Mapping[str, Any]] = None, + ) -> None: + """Create a replace document and add it to the list of ops.""" + validate_ok_for_replace(replacement) + cmd: dict[str, Any] = {"q": selector, "u": replacement} + if upsert is not None: + cmd["upsert"] = upsert + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if hint is not None: + cmd["hint"] = hint + if sort is not None: + self.uses_sort = True + cmd["sort"] = sort + self.ops.append((_UPDATE, cmd)) + + def add_delete( + self, + selector: Mapping[str, Any], + limit: int, + collation: Optional[Mapping[str, Any]] = None, + hint: Union[str, dict[str, Any], None] = None, + ) -> None: + """Create a delete document and add it to the list of ops.""" + cmd: dict[str, Any] = {"q": selector, "limit": limit} + if collation is not None: + self.uses_collation = True + cmd["collation"] = collation + if hint is not None: + cmd["hint"] = hint + if limit == _DELETE_ALL: + # A bulk_write containing a delete_many is not retryable. + self.is_retryable = False + self.ops.append((_DELETE, cmd)) + + def gen_ordered(self) -> Iterator[Optional[_Run]]: + """Generate batches of operations, batched by type of + operation, in the order **provided**. + """ + run = None + for idx, (op_type, operation) in enumerate(self.ops): + if run is None: + run = _Run(op_type) + elif run.op_type != op_type: + yield run + run = _Run(op_type) + run.add(idx, operation) + yield run + + def gen_unordered(self) -> Iterator[_Run]: + """Generate batches of operations, batched by type of + operation, in arbitrary order. + """ + operations = [_Run(_INSERT), _Run(_UPDATE), _Run(_DELETE)] + for idx, (op_type, operation) in enumerate(self.ops): + operations[op_type].add(idx, operation) + + for run in operations: + if run.ops: + yield run diff --git a/pymongo/common.py b/pymongo/common.py index dc18b937e5..9fa3bb15db 100644 --- a/pymongo/common.py +++ b/pymongo/common.py @@ -784,6 +784,7 @@ def validate_server_monitoring_mode(option: str, value: str) -> str: "username": validate_string_or_none, "password": validate_string_or_none, "server_selector": validate_is_callable_or_none, + "srv_host_validator": validate_is_callable_or_none, "auto_encryption_opts": validate_auto_encryption_opts_or_none, "authoidcallowedhosts": validate_list, "max_adaptive_retries": validate_non_negative_integer, diff --git a/pymongo/driver_info.py b/pymongo/driver_info.py index 18a51ae638..a325ea1493 100644 --- a/pymongo/driver_info.py +++ b/pymongo/driver_info.py @@ -29,8 +29,14 @@ class DriverInfo(namedtuple("DriverInfo", ["name", "version", "platform"])): The MongoDB server logs PyMongo's name, version, and platform whenever PyMongo establishes a connection. A driver implemented on top of PyMongo can add its own info to this log message. Initialize with three strings - like 'MyDriver', '1.2.3', 'some platform info'. Any of these strings may be - None to accept PyMongo's default. + like 'MyDriver', '1.2.3', 'some platform info'. Any of these strings may + be None. A None or empty name or version appends an empty metadata + segment, keeping the ``driver.name`` and ``driver.version`` segment + counts aligned. A None or empty platform omits the platform. + + The ``|`` character is the reserved delimiter used to join appended + metadata, so it must not appear in any of the fields. A + :class:`ValueError` is raised if it does. """ def __new__( @@ -42,5 +48,7 @@ def __new__( raise TypeError( f"Wrong type for DriverInfo {key} option, value must be an instance of str, not {type(value)}" ) + if value and "|" in value: + raise ValueError(f"DriverInfo {key} must not contain the '|' delimiter") return self diff --git a/pymongo/encryption_options.py b/pymongo/encryption_options.py index 0d90fe3b19..b1d13700da 100644 --- a/pymongo/encryption_options.py +++ b/pymongo/encryption_options.py @@ -19,9 +19,11 @@ from __future__ import annotations +import socket import warnings -from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Optional, TypedDict +from collections.abc import Awaitable, Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable, Optional, TypedDict from pymongo.uri_parser_shared import _parse_kms_tls_options @@ -55,6 +57,38 @@ def check_min_pymongocrypt() -> None: ) +@dataclass(frozen=True) +class KMSConnectContext: + """Information about a pending KMS connection. + + Passed to ``kms_connect_callback``, which must return a plain, unwrapped + :class:`socket.socket`. The driver performs the KMS TLS handshake over it, + verifying against ``host`` rather than the peer actually reached. + + :param host: Hostname of the KMS server, and the TLS verification target. + :param port: Port of the KMS server. + :param timeout: Seconds allowed for the connection: the default KMS + connect timeout, capped by the remaining time of an active operation + timeout (``timeoutMS``). + + .. note:: ``timeoutMS`` does not constrain KMS requests for explicit + encryption, so ``timeout`` is always the default there. Automatic + encryption passes the remaining budget. This deviates from the Client + Side Operations Timeout specification; see PYTHON-6037. + + .. versionadded:: 4.19 + """ + + host: str + port: int + timeout: float + + +# A callback that opens a connection to a KMS host. +AsyncKMSConnectCallback = Callable[[KMSConnectContext], Awaitable[socket.socket]] +KMSConnectCallback = Callable[[KMSConnectContext], socket.socket] + + class AutoEncryptionOpts: """Options to configure automatic client-side field level encryption.""" @@ -75,6 +109,7 @@ def __init__( bypass_query_analysis: bool = False, encrypted_fields_map: Optional[Mapping[str, Any]] = None, key_expiration_ms: Optional[int] = None, + kms_connect_callback: Optional[Callable[[KMSConnectContext], Any]] = None, ) -> None: """Options to configure automatic client-side field level encryption. @@ -212,7 +247,18 @@ def __init__( :param key_expiration_ms: The cache expiration time for data encryption keys. Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000. Set to 0 to disable key expiration. - + :param kms_connect_callback: A callable that opens the connection to a + KMS host, used to route KMS requests through an HTTP proxy. It + receives a :class:`KMSConnectContext` and returns a connected, + unwrapped :class:`socket.socket`, over which the driver performs + the KMS TLS handshake. Must be a coroutine function for + :class:`~pymongo.asynchronous.mongo_client.AsyncMongoClient` and a + regular function for + :class:`~pymongo.synchronous.mongo_client.MongoClient`. Defaults + to ``None``, meaning the driver connects to KMS hosts directly. + + .. versionchanged:: 4.19 + Added the `kms_connect_callback` parameter. .. versionchanged:: 4.12 Added the `key_expiration_ms` parameter. .. versionchanged:: 4.2 @@ -259,6 +305,11 @@ def __init__( self._async_kms_ssl_contexts: Optional[dict[str, SSLContext]] = None self._bypass_query_analysis = bypass_query_analysis self._key_expiration_ms = key_expiration_ms + if kms_connect_callback is not None and not callable(kms_connect_callback): + raise TypeError( + f"kms_connect_callback must be callable, not {type(kms_connect_callback)}" + ) + self._kms_connect_callback = kms_connect_callback def _kms_ssl_contexts(self, is_sync: bool) -> dict[str, SSLContext]: if is_sync: diff --git a/pymongo/pool_options.py b/pymongo/pool_options.py index 8b26b4baf2..dd95db525e 100644 --- a/pymongo/pool_options.py +++ b/pymongo/pool_options.py @@ -24,6 +24,7 @@ import platform import sys from collections.abc import MutableMapping +from contextlib import AbstractContextManager, nullcontext from pathlib import Path from typing import TYPE_CHECKING, Any, Optional @@ -37,6 +38,7 @@ WAIT_QUEUE_TIMEOUT, has_c, ) +from pymongo.lock import _create_lock if TYPE_CHECKING: from pymongo.auth_shared import MongoCredential @@ -200,60 +202,117 @@ def _metadata_env() -> dict[str, Any]: _MAX_METADATA_SIZE = 512 +def _truncate_utf8(content: str, overflow: int) -> str: + """Trim `overflow` UTF-8 bytes from the end of content, keeping a valid prefix.""" + if overflow <= 0: + return content + data = content.encode("utf-8") + if len(data) <= overflow: + return "" + return data[: len(data) - overflow].decode("utf-8", errors="ignore") + + +def _normalize_driver(driver: DriverInfo) -> DriverInfo: + """Treat None and "" as equivalent unset fields for deduplication.""" + return driver._replace( + name=driver.name or "", + version=driver.version or "", + platform=driver.platform or "", + ) + + +def _element_size(key: str, value: Any) -> int: + """Size in bytes of the BSON element for ``key``, excluding doc overhead.""" + # bson.encode({key: value}) is 4 bytes of document header plus 1 byte of + # document terminator on top of the element itself. + return len(bson.encode({key: value})) - 5 + + # See: https://github.com/mongodb/specifications/blob/master/source/mongodb-handshake/handshake.md#limitations def _truncate_metadata(metadata: MutableMapping[str, Any]) -> None: """Perform metadata truncation.""" - if len(bson.encode(metadata)) <= _MAX_METADATA_SIZE: + # The only full encode is the initial size check; each step then shrinks + # the tracked size by the exact bytes its change removes. + size = len(bson.encode(metadata)) + if size <= _MAX_METADATA_SIZE: return # 1. Omit fields from env except env.name. env_name = metadata.get("env", {}).get("name") if env_name: - metadata["env"] = {"name": env_name} - if len(bson.encode(metadata)) <= _MAX_METADATA_SIZE: + env = {"name": env_name} + size += _element_size("env", env) - _element_size("env", metadata["env"]) + metadata["env"] = env + if size <= _MAX_METADATA_SIZE: return # 2. Omit fields from os except os.type. os_type = metadata.get("os", {}).get("type") if os_type: - metadata["os"] = {"type": os_type} - if len(bson.encode(metadata)) <= _MAX_METADATA_SIZE: + old_os = metadata["os"] + new_os = {"type": os_type} + size += _element_size("os", new_os) - _element_size("os", old_os) + metadata["os"] = new_os + if size <= _MAX_METADATA_SIZE: return # 3. Omit the env document entirely. - metadata.pop("env", None) - encoded_size = len(bson.encode(metadata)) - if encoded_size <= _MAX_METADATA_SIZE: + env = metadata.pop("env", None) + if env is not None: + size -= _element_size("env", env) + if size <= _MAX_METADATA_SIZE: return # 4. Truncate platform. - overflow = encoded_size - _MAX_METADATA_SIZE + overflow = size - _MAX_METADATA_SIZE plat = metadata.get("platform", "") if plat: - plat = plat[:-overflow] - if plat: - metadata["platform"] = plat + truncated = _truncate_utf8(plat, overflow) + if truncated: + size += _element_size("platform", truncated) - _element_size("platform", plat) + else: + size -= _element_size("platform", plat) + if truncated: + metadata["platform"] = truncated + else: + del metadata["platform"] else: - metadata.pop("platform", None) - encoded_size = len(bson.encode(metadata)) - if encoded_size <= _MAX_METADATA_SIZE: - return - # 5. Truncate driver info. - overflow = encoded_size - _MAX_METADATA_SIZE + # The platform field may be present but empty. + plat = metadata.pop("platform", None) + if plat is not None: + size -= _element_size("platform", plat) + # 5. Truncate driver info, keeping name and version 1:1 index-aligned. driver = metadata.get("driver", {}) if driver: - # Truncate driver version. - driver_version = driver.get("version")[:-overflow] - if len(driver_version) >= len(_METADATA["driver"]["version"]): - metadata["driver"]["version"] = driver_version - else: - metadata["driver"]["version"] = _METADATA["driver"]["version"] - encoded_size = len(bson.encode(metadata)) - if encoded_size <= _MAX_METADATA_SIZE: - return - # Truncate driver name. - overflow = encoded_size - _MAX_METADATA_SIZE - driver_name = driver.get("name")[:-overflow] - if len(driver_name) >= len(_METADATA["driver"]["name"]): - metadata["driver"]["name"] = driver_name - else: - metadata["driver"]["name"] = _METADATA["driver"]["name"] + # Keep the name and version segments paired so they stay 1:1 aligned, + # trimming wrapper content and dropping paired segments only as a + # last resort. Blank pairs (for example from a platform-only appended + # driver) are dropped first since they cost nothing. + name_str = driver.get("name", "") + version_str = driver.get("version", "") + raw_pairs = list(zip(name_str.split("|"), version_str.split("|"))) + pairs = [(name, version) for name, version in raw_pairs if name or version] + new_name = "|".join(name for name, _ in pairs) + new_version = "|".join(version for _, version in pairs) + size += (len(new_name.encode("utf-8")) - len(name_str.encode("utf-8"))) + ( + len(new_version.encode("utf-8")) - len(version_str.encode("utf-8")) + ) + driver["name"] = new_name + driver["version"] = new_version + overflow = size - _MAX_METADATA_SIZE + while overflow > 0 and len(pairs) > 1: + # A single remaining pair never exceeds the limit: it is the base + # driver pair, which steps 1-4 left small enough to fit. + last_name, last_version = pairs[-1] + if last_version: + new_version = _truncate_utf8(last_version, overflow) + overflow -= len(last_version.encode("utf-8")) - len(new_version.encode("utf-8")) + pairs[-1] = (last_name, new_version) + elif last_name: + new_name = _truncate_utf8(last_name, overflow) + overflow -= len(last_name.encode("utf-8")) - len(new_name.encode("utf-8")) + pairs[-1] = (new_name, last_version) + else: + pairs.pop() + overflow -= len(f"|{last_name}|{last_version}".encode()) + driver["name"] = "|".join(name for name, _ in pairs) + driver["version"] = "|".join(version for _, version in pairs) # If the first getaddrinfo call of this interpreter's life is on a thread, @@ -277,6 +336,7 @@ class PoolOptions: """ __slots__ = ( + "__appended_drivers", "__appname", "__compression_settings", "__connect_timeout", @@ -288,6 +348,7 @@ class PoolOptions: "__max_idle_time_seconds", "__max_pool_size", "__metadata", + "__metadata_lock", "__min_pool_size", "__pause_enabled", "__server_api", @@ -336,6 +397,11 @@ def __init__( self.__load_balanced = load_balanced self.__credentials = credentials self.__metadata = copy.deepcopy(_METADATA) + self.__appended_drivers: set[DriverInfo] = set() + # Only the synchronous client can append metadata from multiple threads. + self.__metadata_lock: AbstractContextManager[bool | None] = ( + _create_lock() if is_sync else nullcontext() + ) if appname: self.__metadata["application"] = {"name": appname} @@ -353,11 +419,19 @@ def __init__( self.__metadata["driver"]["name"], "c", ) + self.__metadata["driver"]["version"] = "{}|{}".format( + self.__metadata["driver"]["version"], + "", + ) if not is_sync: self.__metadata["driver"]["name"] = "{}|{}".format( self.__metadata["driver"]["name"], "async", ) + self.__metadata["driver"]["version"] = "{}|{}".format( + self.__metadata["driver"]["version"], + "", + ) if driver: self._update_metadata(driver) @@ -368,28 +442,42 @@ def __init__( _truncate_metadata(self.__metadata) def _update_metadata(self, driver: DriverInfo) -> None: - """Updates the client's metadata""" - if driver.name and driver.name.lower() in self.__metadata["driver"]["name"].lower().split( - "|" - ): - return - - metadata = copy.deepcopy(self.__metadata) - - if driver.name: - metadata["driver"]["name"] = "{}|{}".format( - metadata["driver"]["name"], - driver.name, - ) - if driver.version: + """Updates the client's metadata.""" + with self.__metadata_lock: + driver = _normalize_driver(driver) + if driver in self.__appended_drivers: + return + + # Only the top-level keys and the "driver" document are mutated, + # so shallow copies of those two are enough. + metadata = {**self.__metadata, "driver": dict(self.__metadata["driver"])} + + metadata["driver"]["name"] = "{}|{}".format(metadata["driver"]["name"], driver.name) metadata["driver"]["version"] = "{}|{}".format( - metadata["driver"]["version"], - driver.version, + metadata["driver"]["version"], driver.version ) - if driver.platform: - metadata["platform"] = "{}|{}".format(metadata["platform"], driver.platform) - - self.__metadata = metadata + if driver.platform: + if "platform" in metadata: + metadata["platform"] = "{}|{}".format(metadata["platform"], driver.platform) + else: + metadata["platform"] = driver.platform + + _truncate_metadata(metadata) + + self.__metadata = metadata + + # Only track drivers whose appended name/version pair survived + # truncation, so __appended_drivers stays bounded. Truncation + # keeps name and version index-aligned, so strict=True never + # raises. + names = metadata["driver"]["name"].split("|") + versions = metadata["driver"]["version"].split("|") + if sys.version_info >= (3, 10): + pairs = zip(names, versions, strict=True) + else: + pairs = zip(names, versions) + if (driver.name, driver.version) in pairs: + self.__appended_drivers.add(driver) @property def _credentials(self) -> Optional[MongoCredential]: diff --git a/pymongo/pool_shared.py b/pymongo/pool_shared.py index 8cd546bda6..04663f6fc9 100644 --- a/pymongo/pool_shared.py +++ b/pymongo/pool_shared.py @@ -18,6 +18,7 @@ import asyncio import collections +import contextlib import functools import socket import ssl @@ -304,16 +305,29 @@ async def _async_create_connection(address: _Address, options: PoolOptions) -> s raise OSError("getaddrinfo failed") -async def _async_configured_socket( - address: _Address, options: PoolOptions +def _close_late_socket(future: asyncio.Future[Any]) -> None: + """Close a socket produced after its awaiting task was cancelled.""" + if not future.cancelled() and future.exception() is None: + # The callback may have returned a non-socket; close best effort. + close = getattr(future.result(), "close", None) + if callable(close): + with contextlib.suppress(Exception): + close() + + +async def _async_wrap_socket_tls( + sock: socket.socket, address: _Address, options: PoolOptions ) -> Union[socket.socket, _sslConn]: - """Given (host, port) and PoolOptions, return a raw configured socket. + """Given a connected socket, (host, port), and PoolOptions, apply TLS. + + The handshake, SNI, and certificate/hostname verification all target + ``address``, which may differ from the peer ``sock`` is connected to, e.g. + when ``sock`` tunnels through an HTTP proxy. Can raise socket.error, ConnectionFailure, or _CertificateError. - Sets socket's SSL and timeout options. + Sets the socket's SSL and timeout options. """ - sock = await _async_create_connection(address, options) ssl_context = options._ssl_context if ssl_context is None: @@ -326,13 +340,19 @@ async def _async_configured_socket( # to use SSLContext.check_hostname. if _has_sni(False): loop = asyncio.get_running_loop() - ssl_sock = await loop.run_in_executor( - None, - functools.partial(ssl_context.wrap_socket, sock, server_hostname=host), # type: ignore[assignment, misc, unused-ignore] - ) + wrap = functools.partial(ssl_context.wrap_socket, sock, server_hostname=host) # type: ignore[assignment, misc, unused-ignore] else: loop = asyncio.get_running_loop() - ssl_sock = await loop.run_in_executor(None, ssl_context.wrap_socket, sock) # type: ignore[assignment, misc, unused-ignore] + wrap = functools.partial(ssl_context.wrap_socket, sock) # type: ignore[assignment, misc, unused-ignore] + # Shield the executor future: wrap_socket hands the fd to a new + # SSLSocket, so cancellation must not orphan the result it produces. + future = loop.run_in_executor(None, wrap) + try: + ssl_sock = await asyncio.shield(future) + except asyncio.CancelledError: + future.add_done_callback(_close_late_socket) + sock.close() + raise except _CertificateError: sock.close() # Raise _CertificateError directly like we do after match_hostname @@ -360,6 +380,19 @@ async def _async_configured_socket( return ssl_sock +async def _async_configured_socket( + address: _Address, options: PoolOptions +) -> Union[socket.socket, _sslConn]: + """Given (host, port) and PoolOptions, return a raw configured socket. + + Can raise socket.error, ConnectionFailure, or _CertificateError. + + Sets socket's SSL and timeout options. + """ + sock = await _async_create_connection(address, options) + return await _async_wrap_socket_tls(sock, address, options) + + async def _configured_protocol_interface( address: _Address, options: PoolOptions, @@ -510,14 +543,19 @@ def _create_connection(address: _Address, options: PoolOptions) -> socket.socket raise OSError("getaddrinfo failed") -def _configured_socket(address: _Address, options: PoolOptions) -> Union[socket.socket, _sslConn]: - """Given (host, port) and PoolOptions, return a raw configured socket. +def _wrap_socket_tls( + sock: socket.socket, address: _Address, options: PoolOptions +) -> Union[socket.socket, _sslConn]: + """Given a connected socket, (host, port), and PoolOptions, apply TLS. + + The handshake, SNI, and certificate/hostname verification all target + ``address``, which may differ from the peer ``sock`` is connected to, e.g. + when ``sock`` tunnels through an HTTP proxy. Can raise socket.error, ConnectionFailure, or _CertificateError. - Sets socket's SSL and timeout options. + Sets the socket's SSL and timeout options. """ - sock = _create_connection(address, options) ssl_context = options._ssl_context if ssl_context is None: @@ -559,6 +597,17 @@ def _configured_socket(address: _Address, options: PoolOptions) -> Union[socket. return ssl_sock +def _configured_socket(address: _Address, options: PoolOptions) -> Union[socket.socket, _sslConn]: + """Given (host, port) and PoolOptions, return a raw configured socket. + + Can raise socket.error, ConnectionFailure, or _CertificateError. + + Sets socket's SSL and timeout options. + """ + sock = _create_connection(address, options) + return _wrap_socket_tls(sock, address, options) + + def _configured_socket_interface( address: _Address, options: PoolOptions, diff --git a/pymongo/public_suffix_list.dat b/pymongo/public_suffix_list.dat index 78da70c4d4..0f09a04291 100644 --- a/pymongo/public_suffix_list.dat +++ b/pymongo/public_suffix_list.dat @@ -6962,7 +6962,6 @@ ltd.ua a2hosted.com cpserver.com activetrail.biz -adaptable.app myaddr.dev myaddr.io dyn.addr.tools @@ -6982,7 +6981,7 @@ africa.com *.auiusercontent.com beep.pl aiven.app -aivencloud.com +*.aivencloud.com akadns.net akamai.net akamai-staging.net @@ -7857,6 +7856,9 @@ eero-stage.online opentunnel.xyz antagonist.cloud claude.app +claudeusercontent.com +frame.claudeusercontent.com +*.cursorusercontent.com apigee.io panel.dev siiites.com @@ -8069,6 +8071,8 @@ sch.ac dev.cv store.cv codeberg.page +codepen.app +codepen.dev csb.app preview.csb.app co.nl @@ -8948,6 +8952,7 @@ hepforge.org onhercules.app hercules-app.com hercules-dev.com +here.now herokuapp.com heyflow.page heyflow.site @@ -9043,6 +9048,7 @@ botdash.gg botdash.net botda.sh botdash.xyz +online-server.cloud apps-1and1.com live-website.com webspace-host.com @@ -9882,6 +9888,9 @@ scbl.pl-waw.scw.cloud scalebook.scw.cloud smartlabeling.scw.cloud dedibox.fr +scw.site +ams.scw.site +waw.scw.site schokokeks.net gov.scot service.gov.scot @@ -10175,9 +10184,9 @@ toolforge.org wmcloud.org beta.wmcloud.org wmflabs.org -vps.hrsn.au hrsn.dev is-a.dev +vps.hrsn.net localcert.net windsurf.app windsurf.build diff --git a/pymongo/settings_shared.py b/pymongo/settings_shared.py index fbe87697c1..bf45c40334 100644 --- a/pymongo/settings_shared.py +++ b/pymongo/settings_shared.py @@ -19,7 +19,7 @@ import threading import traceback from collections.abc import Collection -from typing import Any, Generic, Optional, TypeVar, Union +from typing import Any, Callable, Generic, Optional, TypeVar, Union from bson.objectid import ObjectId from pymongo import common @@ -52,6 +52,7 @@ def __init__( srv_service_name: str = common.SRV_SERVICE_NAME, srv_max_hosts: int = 0, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, server_monitoring_mode: str = common.SERVER_MONITORING_MODE, topology_id: Optional[ObjectId] = None, ): @@ -79,6 +80,7 @@ def __init__( self._srv_service_name = srv_service_name self._srv_max_hosts = srv_max_hosts or 0 self._srv_allowed_hosts_suffix = srv_allowed_hosts_suffix + self._srv_host_validator = srv_host_validator self._server_monitoring_mode = server_monitoring_mode if topology_id is not None: self._topology_id = topology_id @@ -161,6 +163,11 @@ def srv_allowed_hosts_suffix(self) -> Optional[str]: """The srvAllowedHostsSuffix.""" return self._srv_allowed_hosts_suffix + @property + def srv_host_validator(self) -> Optional[Callable[[str], bool]]: + """The srv_host_validator callback.""" + return self._srv_host_validator + @property def server_monitoring_mode(self) -> str: """The serverMonitoringMode.""" diff --git a/pymongo/synchronous/bulk.py b/pymongo/synchronous/bulk.py index be12a72127..291965fca1 100644 --- a/pymongo/synchronous/bulk.py +++ b/pymongo/synchronous/bulk.py @@ -29,23 +29,15 @@ Union, ) -from bson.objectid import ObjectId -from bson.raw_bson import RawBSONDocument -from pymongo import _csot, common +from pymongo import _csot from pymongo._telemetry import _generate_op_id_or_none from pymongo.bulk_shared import ( _COMMANDS, - _DELETE_ALL, + _AgnosticBulkBase, _merge_command, _raise_bulk_write_error, - _Run, ) from pymongo.client_session_shared import _validate_session_write_concern -from pymongo.common import ( - validate_is_document_type, - validate_ok_for_replace, - validate_ok_for_update, -) from pymongo.errors import ( ConfigurationError, InvalidOperation, @@ -54,7 +46,6 @@ from pymongo.helpers_shared import _RETRYABLE_ERROR_CODES from pymongo.message import ( _DELETE, - _INSERT, _UPDATE, _BulkWriteContext, _EncryptedBulkWriteContext, @@ -71,165 +62,14 @@ from pymongo.synchronous.collection import Collection from pymongo.synchronous.mongo_client import MongoClient from pymongo.synchronous.pool import Connection - from pymongo.typings import _DocumentOut, _DocumentType, _Pipeline _IS_SYNC = True -class _Bulk: +class _Bulk(_AgnosticBulkBase): """The private guts of the bulk write API.""" - def __init__( - self, - collection: Collection[_DocumentType], - ordered: bool, - bypass_document_validation: Optional[bool], - comment: Optional[str] = None, - let: Optional[Any] = None, - ) -> None: - """Initialize a _Bulk instance.""" - self.collection = collection.with_options( - codec_options=collection.codec_options._replace( - unicode_decode_error_handler="replace", document_class=dict - ) - ) - self.let = let - if self.let is not None: - common.validate_is_document_type("let", self.let) - self.comment: Optional[str] = comment - self.ordered = ordered - self.ops: list[tuple[int, Mapping[str, Any]]] = [] - self.executed = False - self.bypass_doc_val = bypass_document_validation - self.uses_collation = False - self.uses_array_filters = False - self.uses_sort = False - self.is_retryable = True - self.retrying = False - self.started_retryable_write = False - # Extra state so that we know where to pick up on a retry attempt. - self.current_run = None - self.next_run = None - self.is_encrypted = False - - @property - def bulk_ctx_class(self) -> type[_BulkWriteContext]: - encrypter = self.collection.database.client._encrypter - if encrypter and not encrypter._bypass_auto_encryption: - self.is_encrypted = True - return _EncryptedBulkWriteContext - else: - self.is_encrypted = False - return _BulkWriteContext - - def add_insert(self, document: _DocumentOut) -> None: - """Add an insert document to the list of ops.""" - validate_is_document_type("document", document) - # Generate ObjectId client side. - if not (isinstance(document, RawBSONDocument) or "_id" in document): - document["_id"] = ObjectId() - self.ops.append((_INSERT, document)) - - def add_update( - self, - selector: Mapping[str, Any], - update: Union[Mapping[str, Any], _Pipeline], - multi: bool, - upsert: Optional[bool], - collation: Optional[Mapping[str, Any]] = None, - array_filters: Optional[list[Mapping[str, Any]]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create an update document and add it to the list of ops.""" - validate_ok_for_update(update) - cmd: dict[str, Any] = {"q": selector, "u": update, "multi": multi} - if upsert is not None: - cmd["upsert"] = upsert - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if array_filters is not None: - self.uses_array_filters = True - cmd["arrayFilters"] = array_filters - if hint is not None: - cmd["hint"] = hint - if sort is not None: - self.uses_sort = True - cmd["sort"] = sort - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append((_UPDATE, cmd)) - - def add_replace( - self, - selector: Mapping[str, Any], - replacement: Mapping[str, Any], - upsert: Optional[bool], - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create a replace document and add it to the list of ops.""" - validate_ok_for_replace(replacement) - cmd: dict[str, Any] = {"q": selector, "u": replacement} - if upsert is not None: - cmd["upsert"] = upsert - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if hint is not None: - cmd["hint"] = hint - if sort is not None: - self.uses_sort = True - cmd["sort"] = sort - self.ops.append((_UPDATE, cmd)) - - def add_delete( - self, - selector: Mapping[str, Any], - limit: int, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - ) -> None: - """Create a delete document and add it to the list of ops.""" - cmd: dict[str, Any] = {"q": selector, "limit": limit} - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if hint is not None: - cmd["hint"] = hint - if limit == _DELETE_ALL: - # A bulk_write containing a delete_many is not retryable. - self.is_retryable = False - self.ops.append((_DELETE, cmd)) - - def gen_ordered(self) -> Iterator[Optional[_Run]]: - """Generate batches of operations, batched by type of - operation, in the order **provided**. - """ - run = None - for idx, (op_type, operation) in enumerate(self.ops): - if run is None: - run = _Run(op_type) - elif run.op_type != op_type: - yield run - run = _Run(op_type) - run.add(idx, operation) - yield run - - def gen_unordered(self) -> Iterator[_Run]: - """Generate batches of operations, batched by type of - operation, in arbitrary order. - """ - operations = [_Run(_INSERT), _Run(_UPDATE), _Run(_DELETE)] - for idx, (op_type, operation) in enumerate(self.ops): - operations[op_type].add(idx, operation) - - for run in operations: - if run.ops: - yield run + collection: Collection[Any] @_handle_reauth def write_command( diff --git a/pymongo/synchronous/client_bulk.py b/pymongo/synchronous/client_bulk.py index 5e5d1a76e8..549c9d59ca 100644 --- a/pymongo/synchronous/client_bulk.py +++ b/pymongo/synchronous/client_bulk.py @@ -29,9 +29,7 @@ Union, ) -from bson.objectid import ObjectId -from bson.raw_bson import RawBSONDocument -from pymongo import _csot, common +from pymongo import _csot from pymongo._telemetry import _generate_op_id_or_none from pymongo.synchronous.client_session import ClientSession from pymongo.synchronous.collection import Collection @@ -46,15 +44,11 @@ from pymongo.synchronous.mongo_client import MongoClient from pymongo.synchronous.pool import Connection from pymongo._client_bulk_shared import ( + _AgnosticClientBulkBase, _merge_command, _throw_client_bulk_write_exception, ) from pymongo.client_session_shared import _validate_session_write_concern -from pymongo.common import ( - validate_is_document_type, - validate_ok_for_replace, - validate_ok_for_update, -) from pymongo.errors import ( ConfigurationError, ConnectionFailure, @@ -76,152 +70,15 @@ InsertOneResult, UpdateResult, ) -from pymongo.typings import _DocumentOut, _Pipeline from pymongo.write_concern import WriteConcern _IS_SYNC = True -class _ClientBulk: +class _ClientBulk(_AgnosticClientBulkBase): """The private guts of the client-level bulk write API.""" - def __init__( - self, - client: MongoClient[Any], - write_concern: WriteConcern, - ordered: bool = True, - bypass_document_validation: Optional[bool] = None, - comment: Optional[str] = None, - let: Optional[Any] = None, - verbose_results: bool = False, - ) -> None: - """Initialize a _ClientBulk instance.""" - self.client = client - self.write_concern = write_concern - self.let = let - if self.let is not None: - common.validate_is_document_type("let", self.let) - self.ordered = ordered - self.bypass_doc_val = bypass_document_validation - self.comment = comment - self.verbose_results = verbose_results - self.ops: list[tuple[str, Mapping[str, Any]]] = [] - self.namespaces: list[str] = [] - self.idx_offset: int = 0 - self.total_ops: int = 0 - self.executed = False - self.uses_collation = False - self.uses_array_filters = False - self.is_retryable = self.client.options.retry_writes - self.retrying = False - self.started_retryable_write = False - - @property - def bulk_ctx_class(self) -> type[_ClientBulkWriteContext]: - return _ClientBulkWriteContext - - def add_insert(self, namespace: str, document: _DocumentOut) -> None: - """Add an insert document to the list of ops.""" - validate_is_document_type("document", document) - # Generate ObjectId client side. - if not (isinstance(document, RawBSONDocument) or "_id" in document): - document["_id"] = ObjectId() - cmd = {"insert": -1, "document": document} - self.ops.append(("insert", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_update( - self, - namespace: str, - selector: Mapping[str, Any], - update: Union[Mapping[str, Any], _Pipeline], - multi: bool, - upsert: Optional[bool] = None, - collation: Optional[Mapping[str, Any]] = None, - array_filters: Optional[list[Mapping[str, Any]]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create an update document and add it to the list of ops.""" - validate_ok_for_update(update) - cmd = { - "update": -1, - "filter": selector, - "updateMods": update, - "multi": multi, - } - if upsert is not None: - cmd["upsert"] = upsert - if array_filters is not None: - self.uses_array_filters = True - cmd["arrayFilters"] = array_filters - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if sort is not None: - cmd["sort"] = sort - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append(("update", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_replace( - self, - namespace: str, - selector: Mapping[str, Any], - replacement: Mapping[str, Any], - upsert: Optional[bool] = None, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - sort: Optional[Mapping[str, Any]] = None, - ) -> None: - """Create a replace document and add it to the list of ops.""" - validate_ok_for_replace(replacement) - cmd = { - "update": -1, - "filter": selector, - "updateMods": replacement, - "multi": False, - } - if upsert is not None: - cmd["upsert"] = upsert - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if sort is not None: - cmd["sort"] = sort - self.ops.append(("replace", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 - - def add_delete( - self, - namespace: str, - selector: Mapping[str, Any], - multi: bool, - collation: Optional[Mapping[str, Any]] = None, - hint: Union[str, dict[str, Any], None] = None, - ) -> None: - """Create a delete document and add it to the list of ops.""" - cmd = {"delete": -1, "filter": selector, "multi": multi} - if hint is not None: - cmd["hint"] = hint - if collation is not None: - self.uses_collation = True - cmd["collation"] = collation - if multi: - # A bulk_write containing an update_many is not retryable. - self.is_retryable = False - self.ops.append(("delete", cmd)) - self.namespaces.append(namespace) - self.total_ops += 1 + client: MongoClient[Any] @_handle_reauth def write_command( diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index e7d8a366ca..ac2f710382 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -16,8 +16,12 @@ from __future__ import annotations +import asyncio +import contextlib import functools +import inspect import socket +import ssl import time as time # noqa: PLC0414 # needed in sync version import uuid import weakref @@ -56,6 +60,8 @@ from pymongo.daemon import _spawn_daemon from pymongo.encryption_options import ( AutoEncryptionOpts, + KMSConnectCallback, + KMSConnectContext, RangeOpts, StringOpts, # Re-exported for backwards compatibility: TextOpts is deprecated but must @@ -84,8 +90,10 @@ from pymongo.operations import UpdateOne from pymongo.pool_options import PoolOptions from pymongo.pool_shared import ( + _close_late_socket, _configured_socket, _raise_connection_failure, + _wrap_socket_tls, ) from pymongo.read_concern import ReadConcern from pymongo.results import DeleteResult @@ -119,11 +127,108 @@ _KEY_VAULT_OPTS = CodecOptions(document_class=RawBSONDocument) -def _connect_kms(address: _Address, opts: PoolOptions) -> Union[socket.socket, _sslConn]: +def _close_rejected_kms_socket(obj: Any) -> None: + """Close a callback return value that failed validation, best effort. + + ``_connect_kms`` raises on a contract violation instead of returning the + value, so no caller ever takes ownership of it. Close it here, tolerating + non-socket values and ``close()`` failures. + """ + close = getattr(obj, "close", None) + if callable(close): + with contextlib.suppress(Exception): + close() + + +def _connect_kms( + address: _Address, + opts: PoolOptions, + kms_connect_callback: Optional[KMSConnectCallback], + timeout: float, +) -> Union[socket.socket, _sslConn]: + """Connect to a KMS host and perform the TLS handshake over the socket. + + Uses ``kms_connect_callback`` when one is provided, otherwise connects + directly, and always verifies against ``address`` (the KMS host). + """ + if kms_connect_callback is None: + try: + return _configured_socket(address, opts) + except Exception as exc: + _raise_connection_failure(address, exc, timeout_details=_get_timeout_details(opts)) + + # TLS targets address, not the peer, so verification follows the KMS host. + # A plain callable would block the event loop before we could reject it, + # so check the callback first. + if not _IS_SYNC: + callback_any: Any = kms_connect_callback + is_coro = inspect.iscoroutinefunction(callback_any) + if not is_coro and callable(callback_any): + is_coro = inspect.iscoroutinefunction(callback_any.__call__) + if not is_coro: + raise ConfigurationError( + "kms_connect_callback must be a coroutine function for the async API." + ) + # Any: the sync version's callback returns a plain socket. + result: Any = kms_connect_callback( + KMSConnectContext(host=address[0], port=cast(int, address[1]), timeout=timeout) + ) + remaining = _csot.remaining() + if remaining is None or _IS_SYNC: + # The synchronous API cannot interrupt a callback that has started + # running; honoring the deadline is the callback's contract there. + sock = result + else: + # CSOT is cooperative: a callback that ignores the timeout could block + # past the deadline. Shield the task so stopping the wait does not + # cancel it mid-flight, and close any socket it yields later. + task = asyncio.ensure_future(result) + try: + sock = asyncio.wait_for(asyncio.shield(task), remaining) + except asyncio.CancelledError: + task.add_done_callback(_close_late_socket) + raise + except asyncio.TimeoutError: + task.add_done_callback(_close_late_socket) + _raise_connection_failure( + address, + socket.timeout("timed out"), + timeout_details=_get_timeout_details(opts), + ) + if not isinstance(sock, socket.socket) or isinstance(sock, ssl.SSLSocket): + _close_rejected_kms_socket(sock) + raise ConfigurationError( + "kms_connect_callback must return a connected, unwrapped " + f"socket.socket, not {type(sock)}." + ) + # wrap_socket refuses a non-blocking socket, so normalize the mode here. + try: + sock.getpeername() + except OSError: + _close_rejected_kms_socket(sock) + raise ConfigurationError( + "kms_connect_callback must return an already connected socket." + ) from None + if sock.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE) != socket.SOCK_STREAM: + _close_rejected_kms_socket(sock) + raise ConfigurationError( + "kms_connect_callback must return a stream socket, not a datagram one." + ) + # The callback may have consumed much of the CSOT budget, and wrapping + # resets the socket timeout, so recompute the remaining time here and for + # the KMS request that follows. + sock.settimeout(max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001)) try: - return _configured_socket(address, opts) + conn = _wrap_socket_tls(sock, address, opts) + except asyncio.CancelledError: + # The executor may still be wrapping the socket; close it so a TLS + # proxy's relay threads wind down instead of leaking. + sock.close() + raise except Exception as exc: _raise_connection_failure(address, exc, timeout_details=_get_timeout_details(opts)) + conn.settimeout(max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001)) + return conn class _EncryptionIO(MongoCryptCallback): # type: ignore[misc] @@ -178,13 +283,6 @@ def kms_request(self, kms_context: MongoCryptKmsContext) -> None: False, # disable_ocsp_endpoint_check _IS_SYNC, ) - # CSOT: set timeout for socket creation. - connect_timeout = max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001) - opts = PoolOptions( - connect_timeout=connect_timeout, - socket_timeout=connect_timeout, - ssl_context=ctx, - ) address = parse_host(endpoint, _HTTPS_PORT) if address[0].endswith(".sock"): raise ConfigurationError(f"Invalid KMS endpoint {endpoint!r}") @@ -192,8 +290,21 @@ def kms_request(self, kms_context: MongoCryptKmsContext) -> None: if sleep_u: sleep_sec = float(sleep_u) / 1e6 time.sleep(sleep_sec) + # Set the connect timeout after the retry backoff so the budget + # reflects the sleep. + connect_timeout = max(_csot.clamp_remaining(_KMS_CONNECT_TIMEOUT), 0.001) + opts = PoolOptions( + connect_timeout=connect_timeout, + socket_timeout=connect_timeout, + ssl_context=ctx, + ) try: - conn = _connect_kms(address, opts) + conn = _connect_kms( + address, + opts, + self.opts._kms_connect_callback, + connect_timeout, + ) try: sendall(conn, message) while kms_context.bytes_needed > 0: @@ -229,6 +340,8 @@ def kms_request(self, kms_context: MongoCryptKmsContext) -> None: conn.close() except MongoCryptError: raise # Propagate MongoCryptError errors directly. + except ConfigurationError: + raise # A callback contract violation is not transient. except Exception as exc: remaining = _csot.remaining() if isinstance(exc, NetworkTimeout) or (remaining is not None and remaining <= 0): @@ -523,6 +636,7 @@ def __init__( codec_options: CodecOptions[_DocumentTypeArg], kms_tls_options: Optional[Mapping[str, Any]] = None, key_expiration_ms: Optional[int] = None, + kms_connect_callback: Optional[KMSConnectCallback] = None, ) -> None: """Explicit client-side field level encryption. @@ -592,7 +706,18 @@ def __init__( :param key_expiration_ms: The cache expiration time for data encryption keys. Defaults to ``None`` which defers to libmongocrypt's default which is currently 60000. Set to 0 to disable key expiration. - + :param kms_connect_callback: A callable that opens the connection to a + KMS host, e.g. to route KMS requests through a proxy. It receives + a :class:`~pymongo.encryption_options.KMSConnectContext` and + returns a connected, unwrapped :class:`socket.socket`, over which + the driver performs the KMS TLS handshake. + Must be a regular function. + The callback is responsible for honoring ``context.timeout``. + Defaults to ``None``, meaning the driver connects to KMS hosts + directly. + + .. versionchanged:: 4.19 + Added the `kms_connect_callback` parameter. .. versionchanged:: 4.12 Added the `key_expiration_ms` parameter. .. versionchanged:: 4.0 @@ -632,6 +757,7 @@ def __init__( key_vault_namespace, kms_tls_options=kms_tls_options, key_expiration_ms=key_expiration_ms, + kms_connect_callback=kms_connect_callback, ) self._kms_ssl_contexts = _parse_kms_tls_options(opts._kms_tls_options, _IS_SYNC) self._io_callbacks: Optional[_EncryptionIO] = _EncryptionIO( diff --git a/pymongo/synchronous/mongo_client.py b/pymongo/synchronous/mongo_client.py index 574fbc00de..df09b03b20 100644 --- a/pymongo/synchronous/mongo_client.py +++ b/pymongo/synchronous/mongo_client.py @@ -471,6 +471,31 @@ def __init__( srvAllowedHostsSuffix=".internal.example.com", ) + - `srv_host_validator`: (callable or None) A callback used in place of the + default parent-domain check for hosts returned by SRV DNS records. It is + called once per returned host with the lowercased hostname as its only + argument, and must return ``True`` to accept the host or ``False`` to + reject it. During initial DNS discovery, rejecting a host raises + :exc:`~pymongo.errors.ConfigurationError`, as does an exception raised by + the callback itself. Use this when the set of acceptable hosts cannot be + expressed as a single suffix:: + + def validator(host: str) -> bool: + return host.endswith((".a.example.com", ".b.example.com")) + + MongoClient( + "mongodb+srv://cluster.example.com/", + srv_host_validator=validator, + ) + + The callback must not block. It is mutually exclusive with + ``srvAllowedHostsSuffix``. Because this option is a callable, it can + only be passed in as a keyword argument, not through the connection string. + + .. warning:: + + This option replaces the built-in DNS spoofing safeguards. + Please use with caution. | **Write Concern options:** | (Only set if passed. No default values.) @@ -638,6 +663,12 @@ def __init__( .. seealso:: The MongoDB documentation on `connections `_. + .. versionchanged:: 4.19 + Added the ``srv_host_validator`` keyword argument. + + .. versionchanged:: 4.18 + Added the ``srvAllowedHostsSuffix`` URI and keyword argument. + .. versionchanged:: 4.17 Added the ``max_adaptive_retries`` and ``enable_overload_retargeting`` URI and keyword arguments. @@ -824,6 +855,7 @@ def __init__( srv_service_name = keyword_opts.get("srvservicename") srv_max_hosts = keyword_opts.get("srvmaxhosts") srv_allowed_hosts_suffix = keyword_opts.get("srvallowedhostssuffix") + srv_host_validator = keyword_opts.get("srv_host_validator") if len([h for h in self._host if "/" in h]) > 1: raise ConfigurationError("host must not contain multiple MongoDB URIs") for entity in self._host: @@ -916,7 +948,11 @@ def __init__( ) self._init_based_on_options( - self._seeds, srv_max_hosts, srv_service_name, srv_allowed_hosts_suffix + self._seeds, + srv_max_hosts, + srv_service_name, + srv_allowed_hosts_suffix, + srv_host_validator, ) self._opened = False @@ -936,6 +972,7 @@ def _resolve_srv(self) -> None: srv_service_name = keyword_opts.get("srvservicename") srv_max_hosts = keyword_opts.get("srvmaxhosts") srv_allowed_hosts_suffix = keyword_opts.get("srvallowedhostssuffix") + srv_host_validator = keyword_opts.get("srv_host_validator") for entity in self._host: # A hostname can only include a-z, 0-9, '-' and '.'. If we find a '/' # it must be a URI, @@ -957,6 +994,7 @@ def _resolve_srv(self) -> None: srv_service_name=srv_service_name, srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, + srv_host_validator=srv_host_validator, ) seeds.update(res["nodelist"]) opts = res["options"] @@ -1001,7 +1039,11 @@ def _resolve_srv(self) -> None: ) self._init_based_on_options( - seeds, srv_max_hosts, srv_service_name, srv_allowed_hosts_suffix + seeds, + srv_max_hosts, + srv_service_name, + srv_allowed_hosts_suffix, + srv_host_validator, ) def _init_based_on_options( @@ -1010,7 +1052,17 @@ def _init_based_on_options( srv_max_hosts: Any, srv_service_name: Any, srv_allowed_hosts_suffix: Any, + srv_host_validator: Any = None, ) -> None: + if srv_host_validator is not None: + if srv_allowed_hosts_suffix is not None: + raise ConfigurationError( + "Cannot specify both srv_host_validator and srvAllowedHostsSuffix" + ) + if not self._resolve_srv_info["is_srv"]: + raise ConfigurationError( + "The srv_host_validator option is only allowed with 'mongodb+srv://' URIs" + ) self._event_listeners = self._options.pool_options._event_listeners self._topology_settings = TopologySettings( seeds=seeds, @@ -1029,6 +1081,7 @@ def _init_based_on_options( srv_service_name=srv_service_name, srv_max_hosts=srv_max_hosts, srv_allowed_hosts_suffix=srv_allowed_hosts_suffix, + srv_host_validator=srv_host_validator, server_monitoring_mode=self._options.server_monitoring_mode, topology_id=self._topology_settings._topology_id if self._topology_settings else None, ) diff --git a/pymongo/synchronous/monitor.py b/pymongo/synchronous/monitor.py index 3a1e1a49d0..cbc5796a8a 100644 --- a/pymongo/synchronous/monitor.py +++ b/pymongo/synchronous/monitor.py @@ -366,6 +366,7 @@ def _get_seedlist(self) -> Optional[list[tuple[str, Any]]]: self._settings.pool_options.connect_timeout, self._settings.srv_service_name, srv_allowed_hosts_suffix=self._settings.srv_allowed_hosts_suffix, + srv_host_validator=self._settings.srv_host_validator, ) seedlist, ttl = resolver.get_hosts_and_min_ttl() if len(seedlist) == 0: diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index c977729ba8..dd562ac68e 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -106,6 +106,13 @@ _IS_SYNC = True +# Flags recording the counters a checkout has incremented, so a failed +# checkout can restore them exactly once (PYTHON-6136). +_UNDO_OPERATION_COUNT = 1 +_UNDO_REQUESTS = 2 +_UNDO_SOCKETS = 4 +_UNDO_PENDING = 8 + class Connection(_ConnectionTelemetryInfo): """Store a connection with some metadata. @@ -965,8 +972,12 @@ def _get_conn( "Attempted to check out a connection from closed connection pool" ) + # Every counter increment sets a flag in ``applied`` so a failed + # checkout can restore them exactly once (PYTHON-6136). + applied = 0 with self.lock: self.operation_count += 1 + applied |= _UNDO_OPERATION_COUNT # Get a free socket or create one. if _csot.get_timeout(): @@ -976,28 +987,32 @@ def _get_conn( else: deadline = None - with self.size_cond: - self._raise_if_not_ready(checkout_started_time, emit_event=True) - while not (self.requests < self.max_pool_size): - timeout = deadline - time.monotonic() if deadline else None - if not _cond_wait(self.size_cond, timeout): - # Timed out, notify the next thread to ensure a - # timeout doesn't consume the condition. - if self.requests < self.max_pool_size: - self.size_cond.notify() - self._raise_wait_queue_timeout(checkout_started_time) + try: + with self.size_cond: self._raise_if_not_ready(checkout_started_time, emit_event=True) - self.requests += 1 + while not (self.requests < self.max_pool_size): + timeout = deadline - time.monotonic() if deadline else None + if not _cond_wait(self.size_cond, timeout): + # Timed out, notify the next thread to ensure a + # timeout doesn't consume the condition. + if self.requests < self.max_pool_size: + self.size_cond.notify() + self._raise_wait_queue_timeout(checkout_started_time) + self._raise_if_not_ready(checkout_started_time, emit_event=True) + self.requests += 1 + applied |= _UNDO_REQUESTS + except BaseException: + self._restore_counters(applied) + raise # We've now acquired the semaphore and must release it on error. conn = None - incremented = False emitted_event = False is_new_conn = False try: with self.lock: self.active_sockets += 1 - incremented = True + applied |= _UNDO_SOCKETS while conn is None: # CMAP: we MUST wait for either maxConnecting OR for a socket # to be checked back into the pool. @@ -1018,6 +1033,7 @@ def _get_conn( conn = self.conns.popleft() except IndexError: self._pending += 1 + applied |= _UNDO_PENDING if conn: # We got a socket from the pool if self._perished(conn): conn = None @@ -1029,7 +1045,17 @@ def _get_conn( finally: with self._max_connecting_cond: self._pending -= 1 - self._max_connecting_cond.notify() + applied &= ~_UNDO_PENDING + notified = False + try: + self._max_connecting_cond.notify() + notified = True + finally: + if not notified: + # A kill landed inside notify() (a gevent + # yield point); retry so a waiting + # checkout is not stranded (PYTHON-6136). + self._max_connecting_cond.notify() conn.active = True # connect() already adds cancel_context for new connections; only add @@ -1039,27 +1065,14 @@ def _get_conn( self.active_contexts.add(conn.cancel_context) # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. except BaseException: - if conn: - # We checked out a socket but authentication failed. - conn.close_conn(ConnectionClosedReason.ERROR) - # Re-apply the accounting if a GreenletExit interrupts - # during the size_cond acquisition; during unwind gevent - # lets the re-acquire complete (PYTHON-6074). - accounted = False try: - with self.size_cond: - self.requests -= 1 - if incremented: - self.active_sockets -= 1 - accounted = True - self.size_cond.notify() + if conn is not None: + # We checked out a socket but authentication failed. + conn.close_conn(ConnectionClosedReason.ERROR) finally: - if not accounted: - with self.size_cond: - self.requests -= 1 - if incremented: - self.active_sockets -= 1 - self.size_cond.notify() + # Restore the counters even if the cleanup above was + # interrupted (PYTHON-6136). + self._restore_counters(applied) if not emitted_event: self._telemetry.checkout_failed( @@ -1071,6 +1084,51 @@ def _get_conn( return conn + def _restore_applied(self, applied: int) -> None: + """Restore the counters flagged in ``applied``. Caller holds ``size_cond``.""" + if applied & _UNDO_OPERATION_COUNT: + self.operation_count -= 1 + if applied & _UNDO_REQUESTS: + self.requests -= 1 + if applied & _UNDO_SOCKETS: + self.active_sockets -= 1 + if applied & _UNDO_PENDING: + self._pending -= 1 + + def _restore_counters(self, applied: int) -> None: + """Restore the counters a failed checkout incremented (PYTHON-6136). + + Gevent grants the re-acquire during unwind (PYTHON-6074). A kill can + also land inside notify() (a yield point), so notifications are + tracked and retried separately from the counter restore. + """ + accounted = False + notified = 0 + try: + with self.size_cond: + self._restore_applied(applied) + accounted = True + if applied & _UNDO_REQUESTS: + # A pool slot was freed; wake the next waiting thread. + self.size_cond.notify() + notified |= _UNDO_REQUESTS + if applied & _UNDO_PENDING: + # A maxConnecting slot was freed; wake the next waiting thread. + self._max_connecting_cond.notify() + notified |= _UNDO_PENDING + finally: + # Always reacquired: `applied` keeps restore-only flags + # `notified` never tracks, so skipping needs a mask synced + # to the notify sites; uncontended acquires don't yield (PYTHON-6136). + with self.size_cond: + if not accounted: + self._restore_applied(applied) + missing = applied & ~notified + if missing & _UNDO_REQUESTS: + self.size_cond.notify() + if missing & _UNDO_PENDING: + self._max_connecting_cond.notify() + def _checkin_apply( self, conn: Connection, txn: bool, cursor: bool, forked: bool ) -> tuple[Optional[str], bool, bool]: @@ -1117,9 +1175,7 @@ def checkin(self, conn: Connection) -> None: conn.pinned_cursor = False self._pinned_sockets.discard(conn) forked = self.pid != os.getpid() - # Re-apply the accounting if a gevent GreenletExit interrupts during - # the size_cond acquisition; gevent lets the re-acquire complete while - # unwinding (PYTHON-6074). + # Re-apply the accounting if a BaseException interrupts here (PYTHON-6074). close_conn_reason: Optional[str] = None emit_closed = False accounted = False diff --git a/pymongo/synchronous/settings.py b/pymongo/synchronous/settings.py index 2e8c6ddcd8..3416bbd130 100644 --- a/pymongo/synchronous/settings.py +++ b/pymongo/synchronous/settings.py @@ -18,7 +18,7 @@ import threading from collections.abc import Collection -from typing import Optional +from typing import Callable, Optional from bson.objectid import ObjectId from pymongo import common @@ -51,6 +51,7 @@ def __init__( srv_service_name: str = common.SRV_SERVICE_NAME, srv_max_hosts: int = 0, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, server_monitoring_mode: str = common.SERVER_MONITORING_MODE, topology_id: Optional[ObjectId] = None, ): @@ -77,6 +78,7 @@ def __init__( srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix, + srv_host_validator, server_monitoring_mode, topology_id, ) diff --git a/pymongo/synchronous/srv_resolver.py b/pymongo/synchronous/srv_resolver.py index 0e9745eb08..18083c8a0c 100644 --- a/pymongo/synchronous/srv_resolver.py +++ b/pymongo/synchronous/srv_resolver.py @@ -18,9 +18,10 @@ import ipaddress import random -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Union -from pymongo._psl import is_public_suffix +from pymongo._psl import SPECIAL_USE_LABELS, _to_punycode, is_public_suffix +from pymongo._telemetry import log_srv_monitor_invalid_host from pymongo.common import CONNECT_TIMEOUT from pymongo.errors import ConfigurationError @@ -64,21 +65,40 @@ def __init__( srv_service_name: str, srv_max_hosts: int = 0, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ): self.__fqdn = fqdn.lower() self.__srv = srv_service_name self.__connect_timeout = connect_timeout or CONNECT_TIMEOUT self.__srv_max_hosts = srv_max_hosts or 0 - self.__srv_allowed_hosts_suffix = ( - "." + srv_allowed_hosts_suffix.lower().strip(".") if srv_allowed_hosts_suffix else None - ) # ensure there's a . at the beginning of the domain - if self.__srv_allowed_hosts_suffix is not None and is_public_suffix( - self.__srv_allowed_hosts_suffix - ): + self.__srv_host_validator = srv_host_validator + # parse_uri() can call this constructor independently of MongoClient. + if srv_host_validator is not None and srv_allowed_hosts_suffix is not None: raise ConfigurationError( - f"srvAllowedHostsSuffix must not be a public suffix, got: {srv_allowed_hosts_suffix}" + "Cannot specify both srv_host_validator and srvAllowedHostsSuffix" ) - # Validate the fully qualified domain name. + self.__srv_allowed_hosts_suffix = None + if srv_allowed_hosts_suffix is not None: + suffix = srv_allowed_hosts_suffix.strip(".") + if not suffix: + raise ConfigurationError( + f"srvAllowedHostsSuffix must not be empty, got: {srv_allowed_hosts_suffix!r}" + ) + suffix = _to_punycode(suffix).lower() + + is_special_use = suffix in SPECIAL_USE_LABELS + if len(suffix.split(".")) < 2 and not is_special_use: + raise ConfigurationError( + "srvAllowedHostsSuffix must contain at least two '.' separated labels, " + f"got: {srv_allowed_hosts_suffix}" + ) + + if not is_special_use and is_public_suffix(suffix): + raise ConfigurationError( + f"srvAllowedHostsSuffix must not be a public suffix, got: {srv_allowed_hosts_suffix}" + ) + self.__srv_allowed_hosts_suffix = "." + suffix + try: ipaddress.ip_address(fqdn) raise ConfigurationError(_INVALID_HOST_MSG % ("an IP address",)) @@ -106,23 +126,51 @@ def get_options(self) -> Optional[str]: raise ConfigurationError("Only one TXT record is supported") return (b"&".join([b"".join(res.strings) for res in results])).decode("utf-8") # type: ignore[attr-defined] - def _resolve_uri(self, encapsulate_errors: bool) -> resolver.Answer: + def _resolve_uri(self, is_polling: bool) -> resolver.Answer: try: results = _resolve( "_" + self.__srv + "._tcp." + self.__fqdn, "SRV", lifetime=self.__connect_timeout ) except Exception as exc: - if not encapsulate_errors: + if is_polling: # Raise the original error. raise # Else, raise all errors as ConfigurationError. raise ConfigurationError(str(exc)) from exc return results + def _validate_host(self, srv_host: str) -> None: + """Raise ConfigurationError if an SRV-returned host fails verification.""" + if self.__srv_host_validator is not None: + try: + allowed = self.__srv_host_validator(srv_host) + except Exception as exc: + raise ConfigurationError( + f"srv_host_validator raised an exception for SRV host {srv_host}: {exc}" + ) from exc + if not allowed: + raise ConfigurationError( + f"Invalid SRV host: {srv_host} was rejected by srv_host_validator" + ) + elif self.__srv_allowed_hosts_suffix is not None: + if not srv_host.endswith(self.__srv_allowed_hosts_suffix): + raise ConfigurationError(f"Invalid SRV host: {srv_host}") + else: + if self.__fqdn == srv_host and self.nparts < 3: + raise ConfigurationError( + "Invalid SRV host: return address is identical to SRV hostname" + ) + try: + nlist = srv_host.split(".")[1:][-self.__slen :] + except Exception as exc: + raise ConfigurationError(f"Invalid SRV host: {srv_host}") from exc + if self.__plist != nlist: + raise ConfigurationError(f"Invalid SRV host: {srv_host}") + def _get_srv_response_and_hosts( - self, encapsulate_errors: bool + self, is_polling: bool ) -> tuple[resolver.Answer, list[tuple[str, Any]]]: - results = self._resolve_uri(encapsulate_errors) + results = self._resolve_uri(is_polling) # Construct address tuples nodes = [ @@ -130,33 +178,28 @@ def _get_srv_response_and_hosts( for res in results ] - # Validate hosts + # Validate hosts. + valid_nodes = [] for node in nodes: - srv_host = node[0].lower() - if self.__fqdn == srv_host and self.nparts < 3: - raise ConfigurationError( - "Invalid SRV host: return address is identical to SRV hostname" - ) - if self.__srv_allowed_hosts_suffix is not None: - if not srv_host.endswith(self.__srv_allowed_hosts_suffix): - raise ConfigurationError(f"Invalid SRV host: {node[0]}") + try: + self._validate_host(node[0]) + except ConfigurationError as exc: + if not is_polling: + raise + log_srv_monitor_invalid_host(node[0], exc) else: - try: - nlist = srv_host.split(".")[1:][-self.__slen :] - except Exception as exc: - raise ConfigurationError(f"Invalid SRV host: {node[0]}") from exc - if self.__plist != nlist: - raise ConfigurationError(f"Invalid SRV host: {node[0]}") + valid_nodes.append(node) + nodes = valid_nodes if self.__srv_max_hosts: nodes = random.sample(nodes, min(self.__srv_max_hosts, len(nodes))) return results, nodes def get_hosts(self) -> list[tuple[str, Any]]: - _, nodes = self._get_srv_response_and_hosts(True) + _, nodes = self._get_srv_response_and_hosts(is_polling=False) return nodes def get_hosts_and_min_ttl(self) -> tuple[list[tuple[str, Any]], int]: - results, nodes = self._get_srv_response_and_hosts(False) + results, nodes = self._get_srv_response_and_hosts(is_polling=True) rrset = results.rrset ttl = rrset.ttl if rrset else 0 return nodes, ttl diff --git a/pymongo/synchronous/uri_parser.py b/pymongo/synchronous/uri_parser.py index e4f521e3a9..92608b5efe 100644 --- a/pymongo/synchronous/uri_parser.py +++ b/pymongo/synchronous/uri_parser.py @@ -17,7 +17,7 @@ from __future__ import annotations -from typing import Any, Optional +from typing import Any, Callable, Optional from pymongo.common import SRV_SERVICE_NAME, _CaseInsensitiveDictionary from pymongo.errors import ConfigurationError, InvalidURI @@ -48,6 +48,7 @@ def parse_uri( srv_service_name: Optional[str] = None, srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ) -> dict[str, Any]: """Parse and validate a MongoDB URI. @@ -117,6 +118,7 @@ def parse_uri( srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix, + srv_host_validator, ) ) result["options"] = _make_options_case_sensitive(result["options"]) @@ -133,6 +135,7 @@ def _parse_srv( srv_service_name: Optional[str] = None, srv_max_hosts: Optional[int] = None, srv_allowed_hosts_suffix: Optional[str] = None, + srv_host_validator: Optional[Callable[[str], bool]] = None, ) -> dict[str, Any]: if uri.startswith(SCHEME): is_srv = False @@ -168,7 +171,12 @@ def _parse_srv( # argument overrides the same option passed in the connection string. connect_timeout = connect_timeout or options.get("connectTimeoutMS") dns_resolver = _SrvResolver( - fqdn, connect_timeout, srv_service_name, srv_max_hosts, srv_allowed_hosts_suffix + fqdn, + connect_timeout, + srv_service_name, + srv_max_hosts, + srv_allowed_hosts_suffix, + srv_host_validator, ) nodes = dns_resolver.get_hosts() dns_options = dns_resolver.get_options() diff --git a/pymongo/typings.py b/pymongo/typings.py index 0281c57c59..e0777e07e1 100644 --- a/pymongo/typings.py +++ b/pymongo/typings.py @@ -70,7 +70,6 @@ def strip_optional(elem: Optional[_T]) -> _T: __all__ = [ "_Address", - "_AgnosticMongoClient", "_CollationIn", "_DocumentOut", "_DocumentType", diff --git a/pyproject.toml b/pyproject.toml index 2f7a68c837..cf824a72f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,11 @@ validate-bump = false [tool.hatch.build.targets.wheel] packages = ["bson","gridfs", "pymongo"] +[tool.hatch.build.targets.sdist] +# The drivers-evergreen-tools submodule must not ship in the sdist; hatchling's +# walk would otherwise pick it up when the submodule is populated locally. +exclude = ["drivers-evergreen-tools"] + [tool.hatch.metadata.hooks.requirements_txt] files = ["requirements.txt"] diff --git a/requirements/mod_wsgi.txt b/requirements/mod_wsgi.txt new file mode 100644 index 0000000000..31bb5dfda8 --- /dev/null +++ b/requirements/mod_wsgi.txt @@ -0,0 +1,3 @@ +# Test-only; kept out of pyproject.toml so the uv lock never has to +# resolve it (building its sdist needs Apache). +mod_wsgi>=4.9.4 diff --git a/test/asynchronous/test_client.py b/test/asynchronous/test_client.py index 0c95ba978b..563d29cbad 100644 --- a/test/asynchronous/test_client.py +++ b/test/asynchronous/test_client.py @@ -90,7 +90,13 @@ WriteConcernError, ) from pymongo.monitoring import ServerHeartbeatListener, ServerHeartbeatStartedEvent -from pymongo.pool_options import _MAX_METADATA_SIZE, _METADATA, ENV_VAR_K8S, PoolOptions +from pymongo.pool_options import ( + _MAX_METADATA_SIZE, + _METADATA, + ENV_VAR_K8S, + PoolOptions, + _truncate_metadata, +) from pymongo.read_preferences import ReadPreference from pymongo.server_description import ServerDescription from pymongo.server_selectors import readable_server_selector, writable_server_selector @@ -124,6 +130,8 @@ NTHREADS, CMAPListener, FunctionCallRecorder, + _driver_version, + _metadata_with_appended_driver, delay, gevent_monkey_patched, is_greenthread_patched, @@ -387,6 +395,9 @@ async def test_metadata(self): metadata["driver"]["name"] = "PyMongo|c|async" else: metadata["driver"]["name"] = "PyMongo|async" + metadata["driver"]["version"] = _driver_version( + _METADATA["driver"]["version"], metadata["driver"]["name"] + ) metadata["application"] = {"name": "foobar"} client = self.simple_client("mongodb://foo:27017/?appname=foobar&connect=false") options = client.options @@ -398,6 +409,8 @@ async def test_metadata(self): self.simple_client(appname="x" * 128) with self.assertRaises(ValueError): self.simple_client(appname="x" * 129) + + async def test_metadata_bad_driver_options(self): # Bad "driver" options. self.assertRaises(TypeError, DriverInfo, "Foo", 1, "a") self.assertRaises(TypeError, DriverInfo, version="1", platform="a") @@ -408,12 +421,9 @@ async def test_metadata(self): self.simple_client(driver="abc") with self.assertRaises(TypeError): self.simple_client(driver=("Foo", "1", "a")) - # Test appending to driver info. - if has_c(): - metadata["driver"]["name"] = "PyMongo|c|async|FooDriver" - else: - metadata["driver"]["name"] = "PyMongo|async|FooDriver" - metadata["driver"]["version"] = "{}|1.2.3".format(_METADATA["driver"]["version"]) + + async def test_metadata_appends_driver_info(self): + metadata = _metadata_with_appended_driver(_IS_SYNC, "FooDriver", "1.2.3") client = self.simple_client( "foo", 27017, @@ -421,9 +431,9 @@ async def test_metadata(self): driver=DriverInfo("FooDriver", "1.2.3", None), connect=False, ) - options = client.options - self.assertEqual(options.pool_options.metadata, metadata) - metadata["platform"] = "{}|FooPlatform".format(_METADATA["platform"]) + self.assertEqual(client.options.pool_options.metadata, metadata) + + metadata = _metadata_with_appended_driver(_IS_SYNC, "FooDriver", "1.2.3", "FooPlatform") client = self.simple_client( "foo", 27017, @@ -431,27 +441,131 @@ async def test_metadata(self): driver=DriverInfo("FooDriver", "1.2.3", "FooPlatform"), connect=False, ) - options = client.options - self.assertEqual(options.pool_options.metadata, metadata) - # Test truncating driver info metadata. + self.assertEqual(client.options.pool_options.metadata, metadata) + + async def test_metadata_truncates_driver_info(self): + # Truncated driver info must stay within the limit and keep name and + # version index-aligned. client = self.simple_client( driver=DriverInfo(name="s" * _MAX_METADATA_SIZE), connect=False, ) options = client.options + truncated = options.pool_options.metadata["driver"] self.assertLessEqual( len(bson.encode(options.pool_options.metadata)), _MAX_METADATA_SIZE, ) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) client = self.simple_client( driver=DriverInfo(name="s" * _MAX_METADATA_SIZE, version="s" * _MAX_METADATA_SIZE), connect=False, ) options = client.options + truncated = options.pool_options.metadata["driver"] self.assertLessEqual( len(bson.encode(options.pool_options.metadata)), _MAX_METADATA_SIZE, ) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) + # An oversized wrapper name with no version must retain a truncated + # name rather than collapse to the base entry. + client = self.simple_client( + driver=DriverInfo(name="x" * (_MAX_METADATA_SIZE * 2), version=None), + connect=False, + ) + truncated = client.options.pool_options.metadata["driver"] + self.assertLessEqual( + len(bson.encode(client.options.pool_options.metadata)), + _MAX_METADATA_SIZE, + ) + self.assertIn("xxxx", truncated["name"]) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) + + async def test_metadata_truncation_drops_blank_pairs_first(self): + # A platform-only driver appends a blank name/version pair. Truncation + # must drop that pair before trimming a real driver's content. + def metadata_for(name_len: int) -> dict[str, Any]: + name = "PyMongo||" + "W" * name_len # blank pair in the middle + return {"driver": {"name": name, "version": "1.0||1.0"}} + + # Size the name so the document is exactly two bytes over the limit: + # dropping the blank pair alone must make it fit. + name_len = next( + n + for n in range(1, 2 * _MAX_METADATA_SIZE) + if len(bson.encode(metadata_for(n))) == _MAX_METADATA_SIZE + 2 + ) + metadata = metadata_for(name_len) + _truncate_metadata(metadata) + self.assertEqual(len(bson.encode(metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + metadata["driver"], + {"name": "PyMongo|" + "W" * name_len, "version": "1.0|1.0"}, + ) + + async def test_metadata_append_is_bounded(self): + # Successive appends must stay within the limit and keep name and + # version index-aligned after truncation. Once the metadata saturates, + # further appends must not grow the dedup tracking list. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name=f"D{i}", version=f"1.{i}")) + pool = client.options.pool_options + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + pool.metadata["driver"]["name"].count("|"), + pool.metadata["driver"]["version"].count("|"), + ) + count = len(pool._PoolOptions__appended_drivers) + for i in range(300, 600): + client.append_metadata(DriverInfo(name=f"D{i}", version=f"1.{i}")) + self.assertEqual(len(pool._PoolOptions__appended_drivers), count) + # Platform-only appends (empty name/version) stay bounded the same way. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name="", version="", platform=f"P{i}")) + pool = client.options.pool_options + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + count = len(pool._PoolOptions__appended_drivers) + for i in range(300, 600): + client.append_metadata(DriverInfo(name="", version="", platform=f"P{i}")) + self.assertEqual(len(pool._PoolOptions__appended_drivers), count) + + async def test_metadata_recreates_platform_after_truncation(self): + # Appending a platform after truncation has dropped it recreates the field. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name="", version="", platform=f"Q{i}")) + pool = client.options.pool_options + self.assertLess(len(pool._PoolOptions__appended_drivers), 300) + client.append_metadata(DriverInfo(name="Wrapper", version="1.0", platform="Recreated")) + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + pool.metadata["driver"]["name"].count("|"), + pool.metadata["driver"]["version"].count("|"), + ) + + async def test_metadata_deduplicates_none_and_empty(self): + # Empty strings are treated as unset, so a duplicate differing only in + # None vs "" is a no-op. + client = self.simple_client(connect=False) + client.append_metadata(DriverInfo("library", None, "Library Platform")) + names = client.options.pool_options.metadata["driver"]["name"] + vers = client.options.pool_options.metadata["driver"]["version"] + client.append_metadata(DriverInfo("library", "", "Library Platform")) + metadata = client.options.pool_options.metadata + self.assertEqual(metadata["driver"]["name"], names) + self.assertEqual(metadata["driver"]["version"], vers) @mock.patch.dict("os.environ", {ENV_VAR_K8S: "1"}) def test_container_metadata(self): @@ -2226,6 +2340,9 @@ async def _test_handshake(self, env_vars, expected_env): metadata["driver"]["name"] = "PyMongo|c|async" else: metadata["driver"]["name"] = "PyMongo|async" + metadata["driver"]["version"] = _driver_version( + _METADATA["driver"]["version"], metadata["driver"]["name"] + ) if expected_env is not None: metadata["env"] = expected_env @@ -2627,20 +2744,18 @@ def test_gevent_kill_churn_deadlock(self): import gevent.thread as _gthread from gevent import Timeout, spawn - # AMPLIFY_RACE=1 widens gevent's brief sleep on a contended lock - # so a kill lands there reliably. Only the bare sleep() is - # widened; timed sleeps pass through. The unfixed test then - # deadlocks within seconds. + # AMPLIFY_RACE=1 widens the kill windows below; keep AMPLIFY_SECONDS + # small or ops starve. + amplify_seconds = 0.0 if os.environ.get("AMPLIFY_RACE", "0") == "1": - _AMPLIFY_SECONDS = float(os.environ.get("AMPLIFY_SECONDS", "0.02")) + amplify_seconds = float(os.environ.get("AMPLIFY_SECONDS", "0.002")) _orig_thread_sleep = _gthread.sleep def _amplified_sleep(*args): - if not args: # bare sleep(): the courtesy yield on a failed - # non-blocking acquire (Condition.notify -> _is_owned -> acquire(False)) - _orig_thread_sleep(_AMPLIFY_SECONDS) - else: # sleep(0.001), sleep(2), etc.: passthrough + if not args: # bare sleep(): checkin courtesy yield + _orig_thread_sleep(amplify_seconds) + else: # timed sleeps pass through _orig_thread_sleep(*args) _gthread.sleep = _amplified_sleep @@ -2650,6 +2765,31 @@ def _amplified_sleep(*args): coll = client.pymongo_test.coll coll.insert_one({}) + pool = async_get_pool(client) # type:ignore + # Widen the post-gate checkout windows (PYTHON-6136). + if amplify_seconds: + + class _AmplifiedCondition: + def __init__(self, cond, seconds): + self._cond = cond + self._seconds = seconds + + def __enter__(self): + self._cond.__enter__() + return self + + def __exit__(self, *args): + self._cond.__exit__(*args) + time.sleep(self._seconds) + + def __getattr__(self, name): + return getattr(self._cond, name) + + pool.size_cond = _AmplifiedCondition(pool.size_cond, amplify_seconds) + pool._max_connecting_cond = _AmplifiedCondition( + pool._max_connecting_cond, amplify_seconds + ) + op_count = [0] running = [True] workers: list = [] @@ -2663,9 +2803,12 @@ def worker(): except Exception: return + # Scale the kill cadence or ops starve. + reaper_interval = max(0.003, amplify_seconds * 8) + def reaper(): while running[0]: - time.sleep(0.003) + time.sleep(reaper_interval) if not workers: continue idx = random.randrange(len(workers)) @@ -2706,11 +2849,6 @@ def reaper(): coll.find_one({}) except Timeout: self.fail("Pool gate saturated (PYTHON-6074)") - # Deterministic check: a saturated size gate pins the pool's - # checkout counters at maxPoolSize (PYTHON-6074). - pool = async_get_pool(client) # type:ignore - self.assertLess(pool.requests, pool.max_pool_size) - self.assertLess(pool.active_sockets, pool.max_pool_size) self.assertGreater(op_count[0], 0) finally: running[0] = False @@ -2728,6 +2866,14 @@ def reaper(): client.close() except Timeout: pass + # Post-settle: the counters must have fully drained. Note close() + # does not zero operation_count (only a fork does), so a leaked + # increment survives and is caught here. + time.sleep(1.0) + self.assertEqual(pool.requests, 0) + self.assertEqual(pool._pending, 0) + self.assertEqual(pool.active_sockets, 0) + self.assertEqual(pool.operation_count, 0) class TestClientLazyConnect(AsyncIntegrationTest): diff --git a/test/asynchronous/test_client_metadata.py b/test/asynchronous/test_client_metadata.py index 1a07e835a8..a0438fb72a 100644 --- a/test/asynchronous/test_client_metadata.py +++ b/test/asynchronous/test_client_metadata.py @@ -18,7 +18,7 @@ import pathlib import time import unittest -from typing import Any, Optional +from typing import Any, Optional, cast import pytest @@ -99,20 +99,14 @@ async def check_metadata_added( new_name, new_version, new_platform, new_metadata = await self.send_ping_and_get_metadata( client, True ) - if add_name is not None and add_name.lower() in name.lower().split("|"): - self.assertEqual(name, new_name) - self.assertEqual(version, new_version) - self.assertEqual(platform, new_platform) - else: - self.assertEqual(new_name, f"{name}|{add_name}" if add_name is not None else name) - self.assertEqual( - new_version, - f"{version}|{add_version}" if add_version is not None else version, - ) - self.assertEqual( - new_platform, - f"{platform}|{add_platform}" if add_platform is not None else platform, - ) + # Name and version always get a delimiter (empty string if None) to + # preserve 1:1 index correspondence. + self.assertEqual(new_name, f"{name}|{add_name or ''}") + self.assertEqual(new_version, f"{version}|{add_version or ''}") + self.assertEqual( + new_platform, + f"{platform}|{add_platform}" if add_platform is not None else platform, + ) metadata.pop("driver") metadata.pop("platform") @@ -120,7 +114,7 @@ async def check_metadata_added( new_metadata.pop("platform") self.assertEqual(metadata, new_metadata) - async def test_append_metadata(self): + async def test_1_test_that_the_driver_updates_metadata(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -128,7 +122,7 @@ async def test_append_metadata(self): ) await self.check_metadata_added(client, "framework", "2.0", "Framework Platform") - async def test_append_metadata_platform_none(self): + async def test_1_test_that_the_driver_updates_metadata_platform_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -136,7 +130,7 @@ async def test_append_metadata_platform_none(self): ) await self.check_metadata_added(client, "framework", "2.0", None) - async def test_append_metadata_version_none(self): + async def test_1_test_that_the_driver_updates_metadata_version_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -144,7 +138,7 @@ async def test_append_metadata_version_none(self): ) await self.check_metadata_added(client, "framework", None, "Framework Platform") - async def test_append_metadata_platform_version_none(self): + async def test_1_test_that_the_driver_updates_metadata_platform_version_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -152,14 +146,14 @@ async def test_append_metadata_platform_version_none(self): ) await self.check_metadata_added(client, "framework", None, None) - async def test_multiple_successive_metadata_updates(self): + async def test_2_multiple_successive_metadata_updates(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, connect=False ) client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) await self.check_metadata_added(client, "framework", "2.0", "Framework Platform") - async def test_multiple_successive_metadata_updates_platform_none(self): + async def test_2_multiple_successive_metadata_updates_platform_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -167,7 +161,7 @@ async def test_multiple_successive_metadata_updates_platform_none(self): client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) await self.check_metadata_added(client, "framework", "2.0", None) - async def test_multiple_successive_metadata_updates_version_none(self): + async def test_2_multiple_successive_metadata_updates_version_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -175,7 +169,7 @@ async def test_multiple_successive_metadata_updates_version_none(self): client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) await self.check_metadata_added(client, "framework", None, "Framework Platform") - async def test_multiple_successive_metadata_updates_platform_version_none(self): + async def test_2_multiple_successive_metadata_updates_platform_version_none(self): client = await self.async_rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -216,10 +210,16 @@ async def test_duplicate_driver_name_no_op(self): await self.check_metadata_added(client, "framework", None, None) # wait for connection to become idle await asyncio.sleep(0.005) - # add same metadata again - await self.check_metadata_added(client, "Framework", None, None) + # Append the exact same DriverInfo again: no-op. + name, version, platform, _ = await self.send_ping_and_get_metadata(client, True) + await asyncio.sleep(0.005) + client.append_metadata(DriverInfo("framework", None, None)) + new_name, new_version, new_platform, _ = await self.send_ping_and_get_metadata(client, True) + self.assertEqual(new_name, name) + self.assertEqual(new_version, version) + self.assertEqual(new_platform, platform) - async def test_handshake_documents_include_backpressure(self): + async def test_9_handshake_documents_include_backpressure(self): # Create a `MongoClient` that is configured to record all handshake documents sent to the server as a part of # connection establishment. client = await self.async_rs_or_single_client("mongodb://" + self.server.address_string) @@ -232,6 +232,131 @@ async def test_handshake_documents_include_backpressure(self): # the document has a field `backpressure` whose value is `"2"`. self.assertEqual(self.handshake_req["backpressure"], "2") + async def test_10_entries_in_driver_name_and_driver_version_correspond_by_index(self): + cases = [ + ("Gap in middle (name)", [(None, None), ("F2", None)], "||F2", "||"), + ("Gap in middle (version)", [("F1", None), ("F2", "2.0")], "|F1|F2", "||2.0"), + ("Trailing delimiter retained", [("F1", None)], "|F1", "|"), + ( + "Equal versions do not collapse", + [("F1", "{driver_version}")], + "|F1", + "|{driver_version}", + ), + ( + "Equal names do not collapse", + [("{driver_name}", "1.0")], + "|{driver_name}", + "|1.0", + ), + ("Duplicates deduplicate", [("F1", "1.0"), ("F1", "1.0")], "|F1", "|1.0"), + ("All versions absent", [("F1", None), ("F2", None)], "|F1|F2", "||"), + ("All names absent", [(None, "1.0"), (None, "2.0")], "||", "|1.0|2.0"), + ( + "Non-adjacent duplicate", + [("F1", "1.0"), ("F2", "2.0"), ("F1", "1.0")], + "|F1|F2", + "|1.0|2.0", + ), + ( + "Platform-only difference is not a duplicate", + [("F1", "1.0", "P1"), ("F1", "1.0", "P2")], + "|F1|F1", + "|1.0|1.0", + ), + ( + "Wrapper matching the driver's own identity", + [("{driver_name}", "{driver_version}")], + "|{driver_name}", + "|{driver_version}", + ), + ( + "Duplicates with an unset field deduplicate", + [("F1", None), ("F1", None)], + "|F1", + "|", + ), + ] + for ( + description, + appended, + expected_name_suffix, + expected_version_suffix, + ) in cases: + with self.subTest(description=description): + client = await self.async_rs_or_single_client( + "mongodb://" + self.server.address_string, + maxIdleTimeMS=1, + ) + self.addAsyncCleanup(client.close) + # Capture the driver's own name and version from the first handshake. + name0, version0, _, _ = await self.send_ping_and_get_metadata(client, True) + await asyncio.sleep(0.005) + + self.assertIsNotNone(name0) + self.assertIsNotNone(version0) + version0 = cast(str, version0) + driver_name = name0.split("|")[0] + driver_version = version0.split("|")[0] + + def resolve(value: Optional[str]) -> Optional[str]: + if value is None: + return None + return value.format(driver_name=driver_name, driver_version=driver_version) + + # Append each DriverInfo in order. + for opts in appended: + d_name = resolve(opts[0]) if len(opts) > 0 else None + d_version = resolve(opts[1]) if len(opts) > 1 else None + d_platform = resolve(opts[2]) if len(opts) > 2 else None + client.append_metadata(DriverInfo(d_name or "", d_version, d_platform)) + + # New handshake with the appended metadata. + name1, version1, _, _ = await self.send_ping_and_get_metadata(client, True) + + self.assertEqual( + name1, + name0 + + expected_name_suffix.format( + driver_name=driver_name, driver_version=driver_version + ), + ) + self.assertEqual( + version1, + version0 + + expected_version_suffix.format( + driver_name=driver_name, driver_version=driver_version + ), + ) + + async def test_11_appending_metadata_containing_the_delimiter_raises_an_error(self): + cases = [ + ("frame|work", "2.0", "Framework Platform"), + ("framework", "2|0", "Framework Platform"), + ("framework", "2.0", "Framework|Platform"), + ] + for name, version, platform in cases: + with self.subTest(name=name, version=version, platform=platform): + client = await self.async_rs_or_single_client( + "mongodb://" + self.server.address_string, + maxIdleTimeMS=1, + driver=DriverInfo("library", "1.2", "Library Platform"), + ) + self.addAsyncCleanup(client.close) + # Send initial handshake. + name0, version0, platform0, _metadata = await self.send_ping_and_get_metadata( + client, True + ) + await asyncio.sleep(0.005) + # Constructing metadata containing the delimiter raises. + with self.assertRaises(ValueError): + DriverInfo(name, version, platform) + # Metadata is unchanged on the next handshake. + name1, version1, platform1, _ = await self.send_ping_and_get_metadata(client, True) + self.assertEqual(name1, name0) + self.assertEqual(version1, version0) + self.assertEqual(platform1, platform0) + if __name__ == "__main__": unittest.main() diff --git a/test/asynchronous/test_dns.py b/test/asynchronous/test_dns.py index 6796ecff55..befa119b46 100644 --- a/test/asynchronous/test_dns.py +++ b/test/asynchronous/test_dns.py @@ -209,30 +209,40 @@ async def test_connect_case_insensitive(self): class TestInitialDnsSeedlistDiscovery(AsyncPyMongoTestCase): """ Initial DNS Seedlist Discovery prose tests - https://github.com/mongodb/specifications/blob/0a7a8b5/source/initial-dns-seedlist-discovery/tests/README.md#prose-tests + https://github.com/mongodb/specifications/blob/5036f26/source/initial-dns-seedlist-discovery/tests/README.md#prose-tests + + Numbered tests correspond to the numbered prose tests in the spec. The + unnumbered tests are PyMongo-specific additions with no spec counterpart. """ + async def _parse(self, srv_hostname, mock_target, **kwargs): + """Resolve mongodb+srv:// with SRV records naming mock_target.""" + with patch("dns.asyncresolver.resolve") as mock_resolver: + + async def mock_resolve(query, record_type, *args, **kwargs): + mock_srv = MagicMock() + # Mirror dnspython: the wire form keeps the root label, and the + # caller strips it via omit_final_dot. + mock_srv.target.to_text.side_effect = lambda omit_final_dot=False: ( + mock_target.rstrip(".") if omit_final_dot else mock_target + ) + return [mock_srv] + + mock_resolver.side_effect = mock_resolve + return await parse_uri(f"mongodb+srv://{srv_hostname}", **kwargs) + async def run_initial_dns_seedlist_discovery_prose_tests(self, test_cases): for case in test_cases: - with patch("dns.asyncresolver.resolve") as mock_resolver: - - async def mock_resolve(query, record_type, *args, **kwargs): - mock_srv = MagicMock() - mock_srv.target.to_text.return_value = case["mock_target"] - return [mock_srv] - - mock_resolver.side_effect = mock_resolve - domain = case["query"].split("._tcp.")[1] - connection_string = f"mongodb+srv://{domain}" - if "expected_error" not in case: - await parse_uri(connection_string) + domain = case["query"].split("._tcp.")[1] + if "expected_error" not in case: + await self._parse(domain, case["mock_target"]) + else: + try: + await self._parse(domain, case["mock_target"]) + except ConfigurationError as e: + self.assertIn(case["expected_error"], str(e)) else: - try: - await parse_uri(connection_string) - except ConfigurationError as e: - self.assertIn(case["expected_error"], str(e)) - else: - self.fail(f"ConfigurationError was not raised for query: {case['query']}") + self.fail(f"ConfigurationError was not raised for query: {case['query']}") async def test_1_allow_srv_hosts_with_fewer_than_three_dot_separated_parts(self): with patch("dns.asyncresolver.resolve"): @@ -252,8 +262,8 @@ async def test_2_throw_when_return_address_does_not_end_with_srv_domain(self): "expected_error": "Invalid SRV host", }, { - "query": "_mongodb._tcp.blogs.mongo.local", - "mock_target": "test_1.evil.com", + "query": "_mongodb._tcp.mongo.local", + "mock_target": "test_1.evil.local", "expected_error": "Invalid SRV host", }, ] @@ -296,7 +306,103 @@ async def test_4_throw_when_return_address_does_not_contain_dot_separating_share ] await self.run_initial_dns_seedlist_discovery_prose_tests(test_cases) - async def test_5_when_srv_hostname_has_three_or_more_dot_separated_parts_it_is_valid_for_the_returned_hostname_to_be_identical( + async def test_5_srv_host_validator_accepts_a_host_the_default_verification_would_reject(self): + # "blogs.evil.com" does not share a parent domain with the seed, so the + # default check rejects it; the callback overrides that decision. + res = await self._parse( + "blogs.mongodb.com", "blogs.evil.com", srv_host_validator=lambda host: True + ) + self.assertEqual(["blogs.evil.com"], [node[0] for node in res["nodelist"]]) + + # "mongo.local" does not add a domain level to an SRV hostname with fewer + # than three "." separated parts, which the default check also rejects. + res = await self._parse("mongo.local", "mongo.local", srv_host_validator=lambda host: True) + self.assertEqual(["mongo.local"], [node[0] for node in res["nodelist"]]) + + async def test_6_reject_a_host_the_default_verification_would_accept(self): + with self.assertRaisesRegex(ConfigurationError, "rejected by srv_host_validator"): + await self._parse( + "blogs.mongodb.com", "cluster.mongodb.com", srv_host_validator=lambda host: False + ) + + async def test_7_the_validator_receives_the_normalized_host_name(self): + seen = [] + + def validator(host): + seen.append(host) + return True + + await self._parse("blogs.mongodb.com", "CLUSTER.MONGODB.COM.", srv_host_validator=validator) + self.assertEqual(["cluster.mongodb.com"], seen) + + async def test_8_wrap_an_error_raised_by_the_validator(self): + original_exc = Exception("validator_error") + + def validator(host): + raise original_exc + + with self.assertRaisesRegex( + ConfigurationError, "srv_host_validator raised an exception" + ) as ctx: + await self._parse( + "blogs.mongodb.com", "cluster.mongodb.com", srv_host_validator=validator + ) + # The wrapping error must retain the error raised by the validator. + self.assertIs(original_exc, ctx.exception.__cause__) + self.assertIn("validator_error", str(ctx.exception)) + + async def test_9_throw_when_both_srv_allowed_hosts_suffix_and_srv_host_validator_are_configured( + self, + ): + # Rejected by the client + with self.assertRaisesRegex(ConfigurationError, "Cannot specify both"): + self.simple_client( + "mongodb+srv://blogs.mongodb.com", + srv_host_validator=lambda host: True, + srvAllowedHostsSuffix=".mongodb.com", + connect=False, + ) + + # Rejected by the resolver + with self.assertRaisesRegex(ConfigurationError, "Cannot specify both"): + await self._parse( + "blogs.mongodb.com", + "cluster.mongodb.com", + srv_host_validator=lambda host: True, + srv_allowed_hosts_suffix=".mongodb.com", + ) + + async def test_10_accept_a_mixed_case_returned_address_with_srv_allowed_hosts_suffix(self): + # Returned addresses are normalized before the suffix comparison, so + # the case DNS happens to use must not affect the result. + res = await self._parse( + "blogs.mongodb.com", "CLUSTER.MONGODB.COM.", srv_allowed_hosts_suffix=".mongodb.com" + ) + self.assertEqual(["cluster.mongodb.com"], [node[0] for node in res["nodelist"]]) + + async def test_11_throw_when_srv_host_validator_is_not_callable(self): + with self.assertRaisesRegex(ValueError, "must be a callable"): + self.simple_client("mongodb+srv://blogs.mongodb.com", srv_host_validator="notacallable") + + async def test_12_accept_a_reserved_single_label_as_srv_allowed_hosts_suffix(self): + # A single label is a public suffix under the Public Suffix List's "*" + # rule, but the reserved names are accepted despite that. + res = await self._parse( + "cluster.localhost", "db.cluster.localhost", srv_allowed_hosts_suffix="localhost" + ) + self.assertEqual(["db.cluster.localhost"], [node[0] for node in res["nodelist"]]) + + async def test_13_throw_when_srv_host_validator_is_used_with_a_non_srv_uri(self): + with self.assertRaisesRegex( + ConfigurationError, "only allowed with 'mongodb\\+srv://' URIs" + ): + self.simple_client( + "mongodb://localhost:27017", + srv_host_validator=lambda host: True, + connect=False, + ) + + async def test_srv_hostname_with_three_or_more_parts_may_equal_the_returned_hostname( self, ): test_cases = [ diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 128f26feb4..e22d185f77 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -60,7 +60,11 @@ from bson.son import SON from pymongo import ReadPreference from pymongo.asynchronous import encryption -from pymongo.asynchronous.encryption import Algorithm, AsyncClientEncryption, QueryType +from pymongo.asynchronous.encryption import ( + Algorithm, + AsyncClientEncryption, + QueryType, +) from pymongo.asynchronous.helpers import anext from pymongo.asynchronous.mongo_client import AsyncMongoClient from pymongo.cursor_shared import CursorType @@ -222,6 +226,9 @@ async def test_init_kms_tls_options(self): self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) +# KMS connect callback tests live in test_kms_connect.py. + + class TestClientOptions(AsyncPyMongoTestCase): async def test_default(self): client = self.simple_client(connect=False) @@ -316,9 +323,15 @@ def create_client_encryption( key_vault_client: AsyncMongoClient, codec_options: CodecOptions, kms_tls_options: Optional[Mapping[str, Any]] = None, + kms_connect_callback: Optional[Any] = None, ): client_encryption = AsyncClientEncryption( - kms_providers, key_vault_namespace, key_vault_client, codec_options, kms_tls_options + kms_providers, + key_vault_namespace, + key_vault_client, + codec_options, + kms_tls_options, + kms_connect_callback=kms_connect_callback, ) self.addAsyncCleanup(client_encryption.close) return client_encryption @@ -331,9 +344,15 @@ def unmanaged_create_client_encryption( key_vault_client: AsyncMongoClient, codec_options: CodecOptions, kms_tls_options: Optional[Mapping[str, Any]] = None, + kms_connect_callback: Optional[Any] = None, ): client_encryption = AsyncClientEncryption( - kms_providers, key_vault_namespace, key_vault_client, codec_options, kms_tls_options + kms_providers, + key_vault_namespace, + key_vault_client, + codec_options, + kms_tls_options, + kms_connect_callback=kms_connect_callback, ) return client_encryption @@ -1988,6 +2007,9 @@ async def test_invalid_hostname_in_kms_certificate(self): await self.client_encrypted.create_data_key("aws", master_key=key) +# KMS connect callback tests live in test_kms_connect.py. + + # https://github.com/mongodb/specifications/blob/master/source/client-side-encryption/tests/README.md#kms-tls-options-tests class TestKmsTLSOptions(AsyncEncryptionIntegrationTest): @unittest.skipUnless(any(AWS_CREDS.values()), "AWS environment credentials are not set") diff --git a/test/asynchronous/test_kms_connect.py b/test/asynchronous/test_kms_connect.py new file mode 100644 index 0000000000..6a577b90ee --- /dev/null +++ b/test/asynchronous/test_kms_connect.py @@ -0,0 +1,449 @@ +"""Tests for the KMS connect callback.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import os +import socket +import ssl +import threading +import time +import unittest +from asyncio.trsock import TransportSocket +from unittest import mock + +import pytest + +import pymongo +from pymongo.asynchronous.encryption import ( + AsyncClientEncryption, + _connect_kms, + _EncryptionIO, + _wrap_encryption_errors, +) +from pymongo.encryption_options import ( + AutoEncryptionOpts, + KMSConnectContext, +) +from pymongo.errors import ConfigurationError, ConnectionFailure, EncryptionError, NetworkTimeout +from pymongo.pool_options import PoolOptions +from pymongo.ssl_support import get_ssl_context +from test.asynchronous import AsyncPyMongoTestCase +from test.asynchronous.test_encryption import OPTS +from test.helpers_shared import CA_PEM, CERT_PATH, CLIENT_PEM + +_IS_SYNC = False + +pytestmark = pytest.mark.encryption + +_KMS_ADDRESS = ("kms.example.com", 443) + + +def _tls_server_context(cert=CLIENT_PEM): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(cert) + return ctx + + +def _client_tls_context(verify=False): + # verify=False matches the driver's test mode: the local certs don't verify. + if verify: + return get_ssl_context(None, None, CA_PEM, None, False, False, False, _IS_SYNC) + return get_ssl_context(None, None, None, None, True, True, False, _IS_SYNC) + + +async def _run_blocking(func, *args): + """Run a blocking callable off the event loop (inline in the sync version).""" + if _IS_SYNC: + return func(*args) + return await asyncio.get_running_loop().run_in_executor(None, func, *args) + + +def _callback_returning(value): + """A kms_connect_callback that always produces ``value``.""" + + async def callback(context): + return value + + return callback + + +class TestKmsConnectCallbackUnit(AsyncPyMongoTestCase): + """Contract checks for kms_connect_callback that need no KMS server.""" + + @staticmethod + def _pool_options(ssl_context=None): + return PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=ssl_context) + + def _listen(self, backlog=1): + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(backlog) + self.addCleanup(listener.close) + return listener + + def _socketpair(self): + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + return left, right + + async def test_init_kms_connect_callback(self): + opts = AutoEncryptionOpts({}, "k.d") + self.assertIsNone(opts._kms_connect_callback) + + async def callback(context): + raise AssertionError("not called") + + opts = AutoEncryptionOpts({}, "k.d", kms_connect_callback=callback) + self.assertIs(opts._kms_connect_callback, callback) + + for bad in [1, "not-callable", object()]: + with self.assertRaisesRegex(TypeError, "kms_connect_callback must be callable"): + AutoEncryptionOpts({}, "k.d", kms_connect_callback=bad) # type: ignore[arg-type] + + context = KMSConnectContext(host="kms.example.com", port=443, timeout=9.5) + self.assertEqual(context.host, "kms.example.com") + self.assertEqual(context.port, 443) + self.assertEqual(context.timeout, 9.5) + with self.assertRaises(dataclasses.FrozenInstanceError): + context.host = "evil.example.com" # type: ignore[misc] + + async def test_non_socket_return_raises_configuration_error(self): + with self.assertRaisesRegex(ConfigurationError, "must return a connected"): + await _connect_kms( + _KMS_ADDRESS, self._pool_options(), _callback_returning("not-a-socket"), 10.0 + ) + + async def test_already_wrapped_socket_is_rejected(self): + # ssl.SSLSocket passes isinstance but cannot be TLS-wrapped again. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + left, _right = self._socketpair() + # No peer needed to produce a genuine ssl.SSLSocket. + wrapped = ctx.wrap_socket(left, do_handshake_on_connect=False, server_hostname="x") + self.addCleanup(wrapped.close) + + with self.assertRaisesRegex(ConfigurationError, "unwrapped"): + await _connect_kms( + _KMS_ADDRESS, self._pool_options(), _callback_returning(wrapped), 10.0 + ) + + async def test_context_receives_host_port_and_timeout(self): + received = [] + left, _right = self._socketpair() + + async def callback(context): + received.append(context) + return left + + # ssl_context=None returns the socket unchanged, so a plain socket is accepted. + conn = await _connect_kms(_KMS_ADDRESS, self._pool_options(), callback, 12.5) + self.assertIs(conn, left) + + self.assertEqual(len(received), 1) + self.assertEqual(received[0].host, "kms.example.com") + self.assertEqual(received[0].port, 443) + self.assertEqual(received[0].timeout, 12.5) + + async def test_non_blocking_socket_from_callback_is_accepted(self): + # Without the driver normalizing the mode, this raises ValueError. + server_ctx = _tls_server_context() + listener = self._listen() + + def serve(): + try: + conn, _ = listener.accept() + server_ctx.wrap_socket(conn, server_side=True).close() + except OSError: + pass + + threading.Thread(target=serve, daemon=True).start() + + options = self._pool_options(_client_tls_context()) + + def connect(): + sock = socket.create_connection(listener.getsockname(), timeout=10) + sock.setblocking(False) + return sock + + async def callback(context): + return await _run_blocking(connect) + + conn = await _connect_kms(listener.getsockname(), options, callback, 10.0) + self.addCleanup(conn.close) + self.assertIsNotNone(conn.gettimeout()) + + async def test_tls_verification_targets_the_kms_host(self): + # The handshake must verify against the KMS address, not the peer the + # callback connected to. The server cert covers 127.0.0.1 (the peer) + # and localhost, but not the KMS hostname used below, so only + # address-based verification produces this outcome. + server_ctx = _tls_server_context(os.path.join(CERT_PATH, "server.pem")) + listener = self._listen(2) + + def serve(): + for _ in range(2): + try: + conn, _ = listener.accept() + server_ctx.wrap_socket(conn, server_side=True).close() + except (OSError, ssl.SSLError): + # The mismatched-name attempt fails mid-handshake. + pass + + threading.Thread(target=serve, daemon=True).start() + + # Full verification: trusted CA, invalid certs and hostnames rejected. + options = self._pool_options(_client_tls_context(verify=True)) + + created = [] + + def connect(): + sock = socket.create_connection(listener.getsockname(), timeout=10) + created.append(sock) + return sock + + async def callback(context): + return await _run_blocking(connect) + + port = listener.getsockname()[1] + # The cert covers localhost: verifying against the KMS address succeeds. + conn = await _connect_kms(("localhost", port), options, callback, 10.0) + self.addCleanup(conn.close) + # TLS-wrapped in either SSL flavor: a new object, not the plain socket. + self.assertIsNot(conn, created[0]) + # The cert does not cover this name: verification must fail even + # though the peer (127.0.0.1) presents a cert valid for itself. + with self.assertRaises(ConnectionFailure): + await _connect_kms(("kms.example.com", port), options, callback, 10.0) + + async def test_asyncio_transport_socket_is_rejected(self): + # get_extra_info("socket") is a TransportSocket, not a socket.socket. + left, _right = self._socketpair() + + with self.assertRaisesRegex(ConfigurationError, "TransportSocket"): + await _connect_kms( + _KMS_ADDRESS, self._pool_options(), _callback_returning(TransportSocket(left)), 10.0 + ) + + async def test_cancelled_tls_wrap_closes_late_socket(self): + # A cancelled wrap can leave the executor producing an SSLSocket; the + # done callback must close it. + if _IS_SYNC: + raise unittest.SkipTest("the cancel-safe wrap is an async path") + from pymongo.pool_shared import _close_late_socket + + left, _right = self._socketpair() + future = asyncio.get_running_loop().create_future() + future.set_result(left) + self.assertNotEqual(left.fileno(), -1) + _close_late_socket(future) + self.assertEqual(left.fileno(), -1) + + async def test_non_coroutine_callback_is_rejected(self): + # A plain def must be rejected before it blocks the event loop. + if _IS_SYNC: + raise unittest.SkipTest("a regular function is correct for the sync API") + + entered = [] + + def callback(context): + entered.append(context) + return None + + with self.assertRaisesRegex(ConfigurationError, "coroutine function"): + await _connect_kms(_KMS_ADDRESS, self._pool_options(), callback, 10.0) + self.assertEqual(entered, [], "invalid callback must not be entered") + + async def test_unconnected_socket_from_callback_is_rejected(self): + # An unconnected socket would fail later as a transient error and be retried. + bare = socket.socket() + self.addCleanup(bare.close) + + with self.assertRaisesRegex(ConfigurationError, "already connected"): + await _connect_kms(_KMS_ADDRESS, self._pool_options(), _callback_returning(bare), 10.0) + + async def test_datagram_socket_from_callback_is_rejected(self): + # TLS on a connected UDP socket raises NotImplementedError, which would be retried. + left = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + right = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(left.close) + self.addCleanup(right.close) + right.bind(("127.0.0.1", 0)) + left.connect(right.getsockname()) + + with self.assertRaisesRegex(ConfigurationError, "stream socket"): + await _connect_kms(_KMS_ADDRESS, self._pool_options(), _callback_returning(left), 10.0) + + async def test_kms_request_does_not_retry_a_contract_violation(self): + # _connect_kms has no retry loop; the no-retry guarantee is in + # kms_request, so exercise that instead. + calls = [] + + async def callback(context): + calls.append(context) + return "not-a-socket" + + opts = AutoEncryptionOpts({}, "k.d", kms_connect_callback=callback) + io = _EncryptionIO(None, mock.MagicMock(), None, opts) + + class StubKmsContext: + endpoint = "kms.example.com:443" + message = b"request" + kms_provider = "aws" + usleep = 0 + bytes_needed = 1 + + def feed(self, data): + raise AssertionError("should not reach the socket") + + def fail(self): + raise AssertionError("a contract violation must not be retried") + + with self.assertRaises(ConfigurationError): + await io.kms_request(StubKmsContext()) + self.assertEqual(len(calls), 1) + + async def test_contract_violation_surfaces_as_encryption_error(self): + # Callers see EncryptionError with ConfigurationError as its cause. + with self.assertRaises(EncryptionError) as caught: + with _wrap_encryption_errors(): + raise ConfigurationError("kms_connect_callback must return ...") + self.assertIsInstance(caught.exception.__cause__, ConfigurationError) + + async def test_network_error_from_callback_propagates(self): + async def callback(context): + raise OSError("proxy unreachable") + + # Not a ConfigurationError, so kms_request retries it. + with self.assertRaises(OSError): + await _connect_kms(_KMS_ADDRESS, self._pool_options(), callback, 10.0) + + async def test_csot_deadline_stops_a_hung_callback(self): + # A callback that ignores the timeout cannot block past the CSOT + # deadline, and a socket it yields later must be closed. + if _IS_SYNC: + raise unittest.SkipTest("the sync API cannot interrupt a callback") + + left, _right = self._socketpair() + + async def hung_callback(context): + await asyncio.sleep(0.5) + return left + + with self.assertRaises(NetworkTimeout): + with pymongo.timeout(0.1): + await _connect_kms(_KMS_ADDRESS, self._pool_options(), hung_callback, 10.0) + self.assertNotEqual(left.fileno(), -1) + # Let the shielded callback finish; the driver closes the late result. + await asyncio.sleep(0.75) + self.assertEqual(left.fileno(), -1) + + async def test_cancelling_kms_connect_closes_the_callback_socket(self): + # Cancelling during the TLS handshake must close the callback's socket, + # so a TLS proxy's relay threads wind down. + if _IS_SYNC: + raise unittest.SkipTest("cancellation is an async-only behavior") + + server_ctx = _tls_server_context() + listener = self._listen() + gate = threading.Event() + eof = threading.Event() + + def stub_server(): + conn = None + try: + conn, _ = listener.accept() + # The cancel may land before or after the executor starts the + # handshake. Peek for the ClientHello without consuming it, or + # for EOF if the driver closed it, before wrap_socket detaches conn. + while True: + data = conn.recv(4096, socket.MSG_PEEK) + if not data: + eof.set() + return + if data[:1] == b"\x16": # TLS handshake record + break + # Hold the handshake open until the test has cancelled. + if not gate.wait(5): + return + tls = server_ctx.wrap_socket(conn, server_side=True, do_handshake_on_connect=False) + try: + tls.do_handshake() + # A discarded connection may end in a reset rather than a + # clean EOF; either proves the driver closed it. + while tls.recv(4096): + pass + except OSError: + pass + eof.set() + tls.close() + except OSError: + pass + finally: + if conn is not None: + conn.close() + + threading.Thread(target=stub_server, daemon=True).start() + + options = self._pool_options(_client_tls_context()) + socks = [] + + def connect(): + sock = socket.create_connection(listener.getsockname(), timeout=10) + socks.append(sock) + return sock + + async def callback(context): + return await _run_blocking(connect) + + # The sync version returns a socket instead of a coroutine, so both + # error codes are needed depending on the version being checked. + pending = _connect_kms(listener.getsockname(), options, callback, 10.0) + task = asyncio.ensure_future(pending) # type: ignore[type-var,arg-type] + for _ in range(100): + if socks: + break + await asyncio.sleep(0.01) + self.assertTrue(socks, "callback was never invoked") + # Bias the cancel to land mid-handshake; the stub handles the earlier + # window too. + await asyncio.sleep(0.1) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + # The late SSLSocket (or raw socket) must be closed; the stub sees EOF. + gate.set() + for _ in range(50): + if eof.is_set(): + break + await asyncio.sleep(0.1) + self.assertTrue(eof.is_set(), "driver never closed the callback socket") + + async def test_client_encryption_accepts_callback(self): + async def callback(context): + raise AssertionError("not called") + + client = self.simple_client() + encryption = AsyncClientEncryption( + {"local": {"key": b"\x00" * 96}}, + "keyvault.datakeys", + client, + OPTS, + kms_connect_callback=callback, + ) + self.addAsyncCleanup(encryption.close) + self.assertIs(encryption._io_callbacks.opts._kms_connect_callback, callback) + + async def test_client_encryption_rejects_non_callable(self): + client = self.simple_client() + with self.assertRaisesRegex(TypeError, "kms_connect_callback must be callable"): + AsyncClientEncryption( + {"local": {"key": b"\x00" * 96}}, + "keyvault.datakeys", + client, + OPTS, + kms_connect_callback="not-callable", # type: ignore[arg-type] + ) diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py index 661bd4e3d2..9bd221f6e7 100644 --- a/test/asynchronous/test_pooling.py +++ b/test/asynchronous/test_pooling.py @@ -34,13 +34,19 @@ from pymongo.hello import HelloCompat from pymongo.lock import _async_create_lock from pymongo.monitoring import _EventListeners +from pymongo.pool_shared import _async_wrap_socket_tls from test.asynchronous.utils import async_get_pool, async_joinall, flaky sys.path[0:0] = [""] from pymongo.asynchronous.pool import Pool, PoolOptions from pymongo.socket_checker import SocketChecker -from test.asynchronous import AsyncIntegrationTest, async_client_context, unittest +from test.asynchronous import ( + AsyncIntegrationTest, + AsyncPyMongoTestCase, + async_client_context, + unittest, +) from test.asynchronous.helpers import ConcurrentRunner from test.utils_shared import CMAPListener, delay @@ -306,6 +312,7 @@ def notify(): # Accounting was applied exactly once. self.assertEqual(0, cx_pool.requests) self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool.operation_count) async def test_checkout_error_accounting_on_kill_during_acquire(self): # PYTHON-6074: an exception delivered while the checkout error @@ -341,6 +348,98 @@ async def __aexit__(self, *args): # The fallback applied the accounting exactly once. self.assertEqual(0, cx_pool.requests) self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool.operation_count) + + async def test_checkout_error_accounting_on_connect_keyboard_interrupt(self): + # PYTHON-6136: a KeyboardInterrupt from connect() must roll back the + # size gate, the maxConnecting gate, and active_sockets. + cx_pool = await self.create_pool(max_pool_size=1) + + with patch.object(cx_pool, "connect", side_effect=KeyboardInterrupt()): + with self.assertRaises(KeyboardInterrupt): + async with cx_pool.checkout(): + pass + + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) + + async def test_checkout_error_accounting_on_kill_during_pending_cleanup(self): + # PYTHON-6136: an interruption during the pending-gate cleanup must + # not skip the counter restore, which must wake threads waiting at + # the maxConnecting gate. + cx_pool = await self.create_pool(max_pool_size=1) + + class _InterruptOnSecondEnter(type(cx_pool._max_connecting_cond)): + def __init__(self, lock): + super().__init__(lock) + self.enters = 0 + self.notifies = 0 + + async def __aenter__(self): + self.enters += 1 + if self.enters == 2: + # First enter is the checkout wait, second is connect()'s + # cleanup. Simulate a kill delivered while waiting there. + raise KeyboardInterrupt() + return await super().__aenter__() + + async def __aexit__(self, *args): + return await super().__aexit__(*args) + + def notify(self, n=1): + # The counter restore wakes the maxConnecting gate, then is + # itself killed. + self.notifies += 1 + raise KeyboardInterrupt() + + cond = _InterruptOnSecondEnter(cx_pool._max_connecting_cond._lock) + cx_pool._max_connecting_cond = cond + + with patch.object(cx_pool, "connect", side_effect=asyncio.CancelledError()): + with self.assertRaises(KeyboardInterrupt): + async with cx_pool.checkout(): + pass + + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) + # The restore notified the maxConnecting gate; the notify interrupted + # by the kill was retried. + self.assertEqual(2, cond.notifies) + + async def test_checkout_error_accounting_on_kill_during_pending_notify(self): + # PYTHON-6136: a kill landing inside the cleanup notify must not + # strand a checkout waiting at the maxConnecting gate. + cx_pool = await self.create_pool(max_pool_size=1) + + class _InterruptOnFirstNotify(type(cx_pool._max_connecting_cond)): + def __init__(self, lock): + super().__init__(lock) + self.notifies = 0 + + def notify(self, n=1): + self.notifies += 1 + if self.notifies == 1: + # Simulate a kill delivered inside notify(). + raise KeyboardInterrupt() + + cond = _InterruptOnFirstNotify(cx_pool._max_connecting_cond._lock) + cx_pool._max_connecting_cond = cond + + with patch.object(cx_pool, "connect", side_effect=asyncio.CancelledError()): + with self.assertRaises(KeyboardInterrupt): + async with cx_pool.checkout(): + pass + + # The cleanup notify was interrupted, then retried. + self.assertEqual(2, cond.notifies) + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) async def test_pool_removes_closed_socket(self): # Test that Pool removes explicitly closed socket. @@ -463,6 +562,8 @@ async def test_wait_queue_timeout(self): 1, f"Waited {duration:.2f} seconds for a socket, expected {wait_queue_timeout:f}", ) + # The load metric must not be inflated by the failed checkout. + self.assertEqual(0, pool.operation_count) async def test_no_wait_queue_timeout(self): # Verify get_socket() with no wait_queue_timeout blocks forever. @@ -824,5 +925,18 @@ def test_certificate_error_is_not_labeled_overloaded(self): self.assertFalse(err.has_error_label("SystemOverloadedError")) +class TestWrapSocketTLS(AsyncPyMongoTestCase): + async def test_wrap_socket_tls_without_ssl_context_returns_same_socket(self): + options = PoolOptions(socket_timeout=7.5) + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + + result = await _async_wrap_socket_tls(left, ("kms.example.com", 443), options) + + self.assertIs(result, left) + self.assertEqual(result.gettimeout(), 7.5) + + if __name__ == "__main__": unittest.main() diff --git a/test/asynchronous/test_srv_polling.py b/test/asynchronous/test_srv_polling.py index b8b9129925..3d98a6fdc8 100644 --- a/test/asynchronous/test_srv_polling.py +++ b/test/asynchronous/test_srv_polling.py @@ -19,7 +19,9 @@ import asyncio import sys import time +from types import SimpleNamespace from typing import Any +from unittest.mock import patch from test.utils_shared import FunctionCallRecorder @@ -37,6 +39,19 @@ WAIT_TIME = 0.1 +class _FakeSrvAnswer(list): + """A minimal stand-in for the dnspython answer to an SRV query.""" + + def __init__(self, hosts, ttl=60): + import dns.name + + hosts = [(h, 27017) if isinstance(h, str) else h for h in hosts] + super().__init__( + SimpleNamespace(target=dns.name.from_text(h), port=port) for h, port in hosts + ) + self.rrset = SimpleNamespace(ttl=ttl) + + class SrvPollingKnobs: def __init__( self, @@ -64,9 +79,15 @@ def enable(self): async def mock_get_hosts_and_min_ttl(resolver, *args): assert self.old_dns_resolver_response is not None - nodes, ttl = await self.old_dns_resolver_response(resolver) - if self.nodelist_callback is not None: - nodes = self.nodelist_callback() + if self.nodelist_callback is None: + nodes, ttl = await self.old_dns_resolver_response(resolver) + else: + + async def mock_resolve_uri(is_polling): + return _FakeSrvAnswer(self.nodelist_callback()) + + with patch.object(resolver, "_resolve_uri", mock_resolve_uri): + nodes, ttl = await self.old_dns_resolver_response(resolver) if self.ttl_time is not None: ttl = self.ttl_time return nodes, ttl @@ -357,8 +378,92 @@ def nodelist_callback(): ) await client.aconnect() with SrvPollingKnobs(nodelist_callback=nodelist_callback): + expected = [(host.rstrip("."), port) for host, port in response] + await self.assert_nodelist_change(expected, client) + + async def test_14_the_validator_is_consulted_when_srv_records_are_rescanned(self): + seen = [] + + def validator(host): + seen.append(host) + return True + + response = self.BASE_SRV_RESPONSE[:] + response.append(("localhost.test.build.10gen.cc", 27019)) + + with SrvPollingKnobs(ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME): + client = self.simple_client(self.CONNECTION_STRING, srv_host_validator=validator) + await client.aconnect() + await self.assert_nodelist_change(self.BASE_SRV_RESPONSE, client) + seen.clear() + with SrvPollingKnobs(nodelist_callback=lambda: response): await self.assert_nodelist_change(response, client) + self.assertIn("localhost.test.build.10gen.cc", seen) + + async def test_15_a_rejecting_or_raising_validator_does_not_raise_or_stop_polling(self): + def rejecting(host): + return False + + def raising(host): + raise RuntimeError("boom") + + response = self.BASE_SRV_RESPONSE[:] + response.append(("localhost.test.build.10gen.cc", 27019)) + + for failing in (rejecting, raising): + with self.subTest(validator=failing.__name__): + # Accept everything until the client is connected, then start failing. + state = {"validator": lambda host: True} + + with SrvPollingKnobs(ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME): + client = self.simple_client( + self.CONNECTION_STRING, + srv_host_validator=lambda host: state["validator"](host), + ) + await client.aconnect() + await self.assert_nodelist_change(self.BASE_SRV_RESPONSE, client) + + # The rescan sees the new record but the validator fails: no + # error reaches the application and the topology is unchanged. + state["validator"] = failing + with SrvPollingKnobs( + nodelist_callback=lambda: response, count_resolver_calls=True + ): + await self.assert_nodelist_nochange(self.BASE_SRV_RESPONSE, client) + + # Polling was not stopped by the failures: once the validator + # accepts again, the new host is picked up. + state["validator"] = lambda host: True + with SrvPollingKnobs(nodelist_callback=lambda: response): + await self.assert_nodelist_change(response, client) + + await client.close() + + async def test_rescan_skips_only_the_hosts_that_fail_verification(self): + # Per the polling spec, a host that fails verification is left out of + # the topology without discarding the other hosts from the same rescan. + good_host = "localhost.test.build.10gen.cc" + bad_host = "bad.evil.com" + + def raising(host): + if host == bad_host: + raise RuntimeError("some error") + return True + + def rejecting(host): + return host != bad_host + + for validator in (rejecting, raising): + with self.subTest(validator=validator.__name__): + resolver = pymongo.asynchronous.srv_resolver._SrvResolver( + "test1.test.build.10gen.cc", None, "mongodb", srv_host_validator=validator + ) + answer = _FakeSrvAnswer([good_host, bad_host]) + with patch("dns.asyncresolver.resolve", return_value=answer): + nodes, _ = await resolver.get_hosts_and_min_ttl() + self.assertEqual([(good_host, 27017)], nodes) + async def test_srv_waits_to_poll(self): modified = [("localhost.test.build.10gen.cc", 27019)] diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 5d16bc5918..2ee4e68657 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -23,6 +23,7 @@ import binascii import copy import functools +import gc import os import platform import re @@ -539,6 +540,11 @@ def setUpClass(cls) -> None: @classmethod def tearDownClass(cls) -> None: cls.knobs.disable() + # Workaround for sockets leaked by tests (PYTHON-3923): force garbage + # collection so they are closed here, rather than during a later test + # where their ResourceWarnings could cause failures. Remove once the + # leaks are fixed. + gc.collect() async def asyncSetUp(self): # super call creates internal client cls.client diff --git a/test/bson_corpus/dbref.json b/test/bson_corpus/dbref.json index 41c0b09d0e..cf54ccf538 100644 --- a/test/bson_corpus/dbref.json +++ b/test/bson_corpus/dbref.json @@ -22,6 +22,21 @@ "canonical_bson": "4400000003646272656600380000000224726566000b000000636f6c6c656374696f6e00072469640058921b3e6e32ab156a22b59e02666f6f0004000000626172000000", "canonical_extjson": "{\"dbref\": {\"$ref\": \"collection\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}, \"foo\": \"bar\"}}" }, + { + "description": "DBRef with dotted collection name", + "canonical_bson": "3a000000036462726566002e0000000224726566000e00000064622e636f6c6c656374696f6e00072469640058921b3e6e32ab156a22b59e0000", + "canonical_extjson": "{\"dbref\": {\"$ref\": \"db.collection\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}}}" + }, + { + "description": "DBRef with dotted collection name and database", + "canonical_bson": "4b000000036462726566003f0000000224726566000e00000064622e636f6c6c656374696f6e00072469640058921b3e6e32ab156a22b59e0224646200080000006f746865726462000000", + "canonical_extjson": "{\"dbref\": {\"$ref\": \"db.collection\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}, \"$db\": \"otherdb\"}}" + }, + { + "description": "DBRef with multiple dots in collection name", + "canonical_bson": "32000000036462726566002600000002247265660006000000612e622e6300072469640058921b3e6e32ab156a22b59e0000", + "canonical_extjson": "{\"dbref\": {\"$ref\": \"a.b.c\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}}}" + }, { "description": "Document with key names similar to those of a DBRef", "canonical_bson": "3e0000000224726566000c0000006e6f742d612d646272656600072469640058921b3e6e32ab156a22b59e022462616e616e6100050000007065656c0000", diff --git a/test/mockupdb/test_handshake.py b/test/mockupdb/test_handshake.py index 2772e6f77a..e3a3fc0562 100644 --- a/test/mockupdb/test_handshake.py +++ b/test/mockupdb/test_handshake.py @@ -49,9 +49,11 @@ def _check_handshake_data(request): assert data["application"] == {"name": "my app"} if has_c(): name = "PyMongo|c" + version = pymongo_version + "|" else: name = "PyMongo" - assert data["driver"] == {"name": name, "version": pymongo_version} + version = pymongo_version + assert data["driver"] == {"name": name, "version": version} # Keep it simple, just check these fields exist. assert "os" in data diff --git a/test/mod_wsgi_test/README.rst b/test/mod_wsgi_test/README.rst index e96db9406c..5a9348feb2 100644 --- a/test/mod_wsgi_test/README.rst +++ b/test/mod_wsgi_test/README.rst @@ -13,44 +13,39 @@ mod_wsgi creates. Test Matrix ----------- -PyMongo should be tested with several versions of mod_wsgi and a selection -of Python versions. Each combination of mod_wsgi and Python version should -be tested with a standalone and a replica set. ``mod_wsgi_test.py`` -detects if the deployment is a replica set and connects to the whole set. +Continuous integration tests the oldest supported CPython against MongoDB +6.0, the oldest version that installs on the GitHub runners, with minimum +dependencies, and the newest supported CPython against the latest MongoDB, in +both daemon and embedded mode against a replica set. The Python and MongoDB +versions come from ``generate_config_utils.py`` in ``.evergreen/scripts``. +Other combinations can be tested manually. Setup ----- -Compile Python +Install Apache .............. -We need a Python interpreter built as a shared library. Download the -source tarball for each Python version tested, untar it, and run:: +On Ubuntu, install Apache and the headers used to build mod_wsgi:: - ./configure --prefix=/some/directory --enable-shared LDFLAGS="-Wl,--rpath=/some/directory/lib" - make - make install + sudo apt-get install -y apache2 apache2-dev -This results in an executable named "python" or "python3" and a shared -library named something like "libpython2.7.so.1.0" or "libpython3.3m.so.1.0". +On Fedora/RHEL, install httpd and its headers instead:: -Compile mod_wsgi -................ + sudo dnf install -y httpd httpd-devel + +The test harness detects ``apache2`` or ``httpd`` automatically and starts it +with the matching config file (``apache24ubuntu.conf`` or +``httpd24fedora.conf``). -Compile mod_wsgi for each combination for Python and mod_wsgi version in the -test matrix. For example, to compile mod_wsgi 3.4 for Python 2.7 on a -RedHat-like Linux:: +Install mod_wsgi +................ - sudo yum install -y httpd httpd-devel - wget https://modwsgi.googlecode.com/files/mod_wsgi-3.4.tar.gz - tar xzf mod_wsgi-3.4.tar.gz - cd mod_wsgi-3.4 - ./configure --with-python=/some/directory/bin/python LDFLAGS="-Wl,--rpath=/some/directory/lib" - make - make install +mod_wsgi is only needed by these tests, so it is not a project dependency. +Its constraint lives in ``requirements/mod_wsgi.txt``. pip builds mod_wsgi +against the interpreter it is installed with:: -To ease testing of several matrix combinations, copy the resulting -``mod_wsgi.so`` to a safe place. + uv pip install -r requirements/mod_wsgi.txt Start mongod ............ @@ -61,12 +56,21 @@ listening on port 27017. Configure Apache ................ -Set a MOD_WSGI_SO environment variable so our ``mod_wsgi_test.conf`` -can find and load mod_wsgi.so:: +The Apache configs expand environment variables, so export them before +starting Apache directly. All configs need ``MOD_WSGI_SO``, the path to +mod_wsgi.so, and include the mode config named by ``MOD_WSGI_CONF`` +(``mod_wsgi_test.conf`` for standalone or ``mod_wsgi_test_embedded.conf`` +for embedded mode) from the checkout at ``PROJECT_DIRECTORY``. The Fedora +config also needs ``MOD_WSGI_PYTHON_HOME``, the prefix of the interpreter +mod_wsgi was built with:: - export MOD_WSGI_SO=/path/to/mod_wsgi.so + export MOD_WSGI_SO=$(find .venv -name "mod_wsgi*.so") + export MOD_WSGI_CONF=mod_wsgi_test.conf + export PROJECT_DIRECTORY=$PWD + export MOD_WSGI_PYTHON_HOME=$(.venv/bin/python -c "import sys; print(sys.base_prefix)") -Start Apache with one of the config files in this directory. +``mod_wsgi_tester.py`` exports these variables itself, so ``just setup-tests +mod_wsgi `` (see `Automation`_) is the easier path. Run the test ------------ @@ -104,7 +108,13 @@ the workaround added in `PYTHON-569 Automation ---------- -At MongoDB, Inc. we use a continuous integration job that tests each -combination in the matrix. The job starts up Apache, starts a single server -or replica set, and runs ``test_client.py`` with the proper arguments. -See `run-mod-wsgi-tests.sh `_ +Continuous integration runs the test on pull requests that change +mod_wsgi-relevant files. The minimum dependency job runs in a Fedora +container against ``httpd`` and the latest dependency job runs on Ubuntu +against ``apache2`` (see the `test-mod-wsgi workflow +`_). +To run the same steps locally, use ``just setup-tests mod_wsgi ``, +``just run-tests``, and ``just teardown-tests``. On Linux they run against the +host's Apache, either ``apache2`` or ``httpd``; on other hosts they run in an +ubuntu container with Apache, a single-node replica set, and both test modes. +``just smoke-mod-wsgi`` runs both modes with one command. diff --git a/test/mod_wsgi_test/apache22amazon.conf b/test/mod_wsgi_test/apache22amazon.conf deleted file mode 100644 index 7755336b07..0000000000 --- a/test/mod_wsgi_test/apache22amazon.conf +++ /dev/null @@ -1,34 +0,0 @@ -# This is a minimal httpd.conf file written for Apache 2.2 on Amazon Linux - -# The modules directory is here by default. -# ServerRoot "/etc/httpd" -DocumentRoot ${PWD} -PidFile ${PWD}/apache2.pid - -User nobody -Group nobody - -# Bind to localhost only, don't require sudo. -Listen 127.0.0.1:8080 - -# Required modules. -LoadModule authz_host_module modules/mod_authz_host.so -# Needed so we can set a custom log location. -LoadModule log_config_module modules/mod_log_config.so - -ErrorLog ${PWD}/error_log -CustomLog ${PWD}/access_log combined - - - AllowOverride None - Order Deny,Allow - Deny from All - - - - AllowOverride None - Order Allow,Deny - Allow from All - - -Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/apache22ubuntu1204.conf b/test/mod_wsgi_test/apache22ubuntu1204.conf deleted file mode 100644 index 9fa4b2060b..0000000000 --- a/test/mod_wsgi_test/apache22ubuntu1204.conf +++ /dev/null @@ -1,29 +0,0 @@ -# This is a minimal httpd.conf file written for Apache 2.2 on Ubuntu 12.04 - -# The modules directory is here on Ubuntu. -ServerRoot "/usr/lib/apache2" -DocumentRoot ${PWD} -PidFile ${PWD}/apache2.pid - -# Bind to localhost only, don't require sudo. -Listen 127.0.0.1:8080 - -# Required modules. -LoadModule authz_host_module modules/mod_authz_host.so - -ErrorLog ${PWD}/error_log -CustomLog ${PWD}/access_log combined - - - AllowOverride None - Order Deny,Allow - Deny from All - - - - AllowOverride None - Order Allow,Deny - Allow from All - - -Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/apache24ubuntu.conf b/test/mod_wsgi_test/apache24ubuntu.conf new file mode 100644 index 0000000000..63161de86b --- /dev/null +++ b/test/mod_wsgi_test/apache24ubuntu.conf @@ -0,0 +1,47 @@ +# This is a minimal httpd.conf file written for Apache 2.4 on Ubuntu. + +# The modules directory is here on Ubuntu. +ServerRoot "/usr/lib/apache2" + +# mod_wsgi 6 destroys the interpreter as child processes exit. pymongo shuts +# its monitor threads down during interpreter finalization, so the default +# lifecycle (WSGIDestroyInterpreter On) is safe to use. +DocumentRoot ${PWD} +PidFile ${PWD}/apache2.pid + +# Bind to localhost only, don't require sudo. +Listen 127.0.0.1:8080 + +# Required modules. +LoadModule mpm_prefork_module modules/mod_mpm_prefork.so +LoadModule authz_core_module modules/mod_authz_core.so +# Loaded before the mod_wsgi directives in the included test config. +LoadModule wsgi_module ${MOD_WSGI_SO} + +# Bound the number of worker processes. mod_wsgi embedded mode loads a full +# Python interpreter and PyMongo in every worker, so the parallel test's +# concurrent requests would otherwise ramp prefork up to its default +# MaxRequestWorkers (256) and exhaust the container's memory. + + StartServers 2 + MinSpareServers 2 + MaxSpareServers 4 + ServerLimit 16 + MaxRequestWorkers 16 + MaxConnectionsPerChild 0 + + +ErrorLog ${PWD}/error_log +CustomLog ${PWD}/access_log combined + + + AllowOverride None + Require all denied + + + + AllowOverride None + Require all granted + + +Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/apache24ubuntu161404.conf b/test/mod_wsgi_test/apache24ubuntu161404.conf deleted file mode 100644 index eb5414f0f7..0000000000 --- a/test/mod_wsgi_test/apache24ubuntu161404.conf +++ /dev/null @@ -1,28 +0,0 @@ -# This is a minimal httpd.conf file written for Apache 2.4 on Ubuntu 14.04/16.04 - -# The modules directory is here on Ubuntu. -ServerRoot "/usr/lib/apache2" -DocumentRoot ${PWD} -PidFile ${PWD}/apache2.pid - -# Bind to localhost only, don't require sudo. -Listen 127.0.0.1:8080 - -# Required modules. -LoadModule mpm_prefork_module modules/mod_mpm_prefork.so -LoadModule authz_core_module modules/mod_authz_core.so - -ErrorLog ${PWD}/error_log -CustomLog ${PWD}/access_log combined - - - AllowOverride None - Require all denied - - - - AllowOverride None - Require all granted - - -Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/httpd24fedora.conf b/test/mod_wsgi_test/httpd24fedora.conf new file mode 100644 index 0000000000..6693b75447 --- /dev/null +++ b/test/mod_wsgi_test/httpd24fedora.conf @@ -0,0 +1,58 @@ +# This is a minimal httpd.conf file written for Apache 2.4 on Fedora/RHEL. + +# The modules directory is here on Fedora/RHEL. +ServerRoot "/etc/httpd" +DefaultRuntimeDir ${PWD} + +DocumentRoot ${PWD} +PidFile ${PWD}/httpd.pid + +# Bind to localhost only, don't require sudo. +Listen 127.0.0.1:8080 + +# Required modules. +LoadModule mpm_prefork_module modules/mod_mpm_prefork.so +LoadModule authz_core_module modules/mod_authz_core.so +LoadModule unixd_module modules/mod_unixd.so +LoadModule log_config_module modules/mod_log_config.so +# Loaded before any mod_wsgi directive below. +LoadModule wsgi_module ${MOD_WSGI_SO} + +# mod_wsgi is built against the interpreter used to install it. When Apache is +# started by a different executable than that interpreter, the embedded Python +# cannot locate its standard library, so point it at the installation. +WSGIPythonHome "${MOD_WSGI_PYTHON_HOME}" + +# Bound the number of worker processes. mod_wsgi embedded mode loads a full +# Python interpreter and PyMongo in every worker, so the parallel test's +# concurrent requests would otherwise ramp prefork up to its default +# MaxRequestWorkers (256) and exhaust the container's memory. + + StartServers 2 + MinSpareServers 2 + MaxSpareServers 4 + ServerLimit 16 + MaxRequestWorkers 16 + MaxConnectionsPerChild 0 + + +ServerName localhost + +ErrorLog ${PWD}/error_log +CustomLog ${PWD}/access_log combined + +# Apache must not run as root. The httpd package creates this user. +User apache +Group apache + + + AllowOverride None + Require all denied + + + + AllowOverride None + Require all granted + + +Include ${PROJECT_DIRECTORY}/test/mod_wsgi_test/${MOD_WSGI_CONF} diff --git a/test/mod_wsgi_test/mod_wsgi_test.conf b/test/mod_wsgi_test/mod_wsgi_test.conf index a5b09e437f..b4cbaa083e 100644 --- a/test/mod_wsgi_test/mod_wsgi_test.conf +++ b/test/mod_wsgi_test/mod_wsgi_test.conf @@ -14,8 +14,6 @@ # Minimal test of PyMongo in a WSGI application, see bug PYTHON-353 -LoadModule wsgi_module ${MOD_WSGI_SO} - # Avoid permissions issues WSGISocketPrefix /tmp/ diff --git a/test/mod_wsgi_test/mod_wsgi_test_embedded.conf b/test/mod_wsgi_test/mod_wsgi_test_embedded.conf index 306dab4ab6..970803e586 100644 --- a/test/mod_wsgi_test/mod_wsgi_test_embedded.conf +++ b/test/mod_wsgi_test/mod_wsgi_test_embedded.conf @@ -14,8 +14,6 @@ # Minimal test of PyMongo in an *Embedded mode* WSGI application. -LoadModule wsgi_module ${MOD_WSGI_SO} - # Avoid permissions issues WSGISocketPrefix /tmp/ diff --git a/test/test_bson.py b/test/test_bson.py index 191a706388..0800806b84 100644 --- a/test/test_bson.py +++ b/test/test_bson.py @@ -75,6 +75,25 @@ _NUMPY_AVAILABLE = importlib.util.find_spec("numpy") is not None +def _total_memory(): + """Return total physical memory in bytes, or None if it cannot be determined.""" + try: + return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") + except (ValueError, OSError, AttributeError): + return None + + +# PYTHON-6140: on Linux, skip if the host cannot allocate the ~4GiB this test +# needs. On other platforms, run the test and skip on MemoryError. +_TOTAL_MEMORY = _total_memory() +_ENCODE_SIZE_LIMIT_MIN_MEMORY = 6 << 30 +_SKIP_ENCODE_SIZE_LIMIT = ( + sys.platform == "linux" + and _TOTAL_MEMORY is not None + and _TOTAL_MEMORY < _ENCODE_SIZE_LIMIT_MIN_MEMORY +) + + class NotADict(abc.MutableMapping): """Non-dict type that implements the mapping protocol.""" @@ -692,12 +711,22 @@ def test_overflow(self): self.assertRaises(OverflowError, encode, {"x": -9223372036854775809}) @unittest.skipUnless(bson.has_c(), "This test requires the C extension") + @unittest.skipIf( + _SKIP_ENCODE_SIZE_LIMIT, + "Test requires at least 6GiB of memory (PYTHON-6140)", + ) def test_encode_size_limit(self): # PYTHON-5996: encoding must raise when a document's encoded size # exceeds the BSON size limit. - big_value = "a" * (1 << 30) - with self.assertRaises(ValueError): - encode({"a": big_value, "b": big_value, "c": big_value}) + try: + big_value = "a" * (1 << 30) + with self.assertRaises(ValueError): + encode({"a": big_value, "b": big_value, "c": big_value}) + except MemoryError: + # PYTHON-6140: some hosts report enough installed memory but + # cannot actually commit it (pagefile limits, mongod running on + # the same host). Skip rather than fail. + self.skipTest("Could not allocate enough memory (PYTHON-6140)") def test_small_long_encode_decode(self): encoded1 = encode({"x": 256}) diff --git a/test/test_client.py b/test/test_client.py index ee492e9b9b..0c8d0e7595 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -81,7 +81,13 @@ WriteConcernError, ) from pymongo.monitoring import ServerHeartbeatListener, ServerHeartbeatStartedEvent -from pymongo.pool_options import _MAX_METADATA_SIZE, _METADATA, ENV_VAR_K8S, PoolOptions +from pymongo.pool_options import ( + _MAX_METADATA_SIZE, + _METADATA, + ENV_VAR_K8S, + PoolOptions, + _truncate_metadata, +) from pymongo.read_preferences import ReadPreference from pymongo.server_description import ServerDescription from pymongo.server_selectors import readable_server_selector, writable_server_selector @@ -123,6 +129,8 @@ NTHREADS, CMAPListener, FunctionCallRecorder, + _driver_version, + _metadata_with_appended_driver, delay, gevent_monkey_patched, is_greenthread_patched, @@ -380,6 +388,9 @@ def test_metadata(self): metadata["driver"]["name"] = "PyMongo|c" else: metadata["driver"]["name"] = "PyMongo" + metadata["driver"]["version"] = _driver_version( + _METADATA["driver"]["version"], metadata["driver"]["name"] + ) metadata["application"] = {"name": "foobar"} client = self.simple_client("mongodb://foo:27017/?appname=foobar&connect=false") options = client.options @@ -391,6 +402,8 @@ def test_metadata(self): self.simple_client(appname="x" * 128) with self.assertRaises(ValueError): self.simple_client(appname="x" * 129) + + def test_metadata_bad_driver_options(self): # Bad "driver" options. self.assertRaises(TypeError, DriverInfo, "Foo", 1, "a") self.assertRaises(TypeError, DriverInfo, version="1", platform="a") @@ -401,12 +414,9 @@ def test_metadata(self): self.simple_client(driver="abc") with self.assertRaises(TypeError): self.simple_client(driver=("Foo", "1", "a")) - # Test appending to driver info. - if has_c(): - metadata["driver"]["name"] = "PyMongo|c|FooDriver" - else: - metadata["driver"]["name"] = "PyMongo|FooDriver" - metadata["driver"]["version"] = "{}|1.2.3".format(_METADATA["driver"]["version"]) + + def test_metadata_appends_driver_info(self): + metadata = _metadata_with_appended_driver(_IS_SYNC, "FooDriver", "1.2.3") client = self.simple_client( "foo", 27017, @@ -414,9 +424,9 @@ def test_metadata(self): driver=DriverInfo("FooDriver", "1.2.3", None), connect=False, ) - options = client.options - self.assertEqual(options.pool_options.metadata, metadata) - metadata["platform"] = "{}|FooPlatform".format(_METADATA["platform"]) + self.assertEqual(client.options.pool_options.metadata, metadata) + + metadata = _metadata_with_appended_driver(_IS_SYNC, "FooDriver", "1.2.3", "FooPlatform") client = self.simple_client( "foo", 27017, @@ -424,27 +434,131 @@ def test_metadata(self): driver=DriverInfo("FooDriver", "1.2.3", "FooPlatform"), connect=False, ) - options = client.options - self.assertEqual(options.pool_options.metadata, metadata) - # Test truncating driver info metadata. + self.assertEqual(client.options.pool_options.metadata, metadata) + + def test_metadata_truncates_driver_info(self): + # Truncated driver info must stay within the limit and keep name and + # version index-aligned. client = self.simple_client( driver=DriverInfo(name="s" * _MAX_METADATA_SIZE), connect=False, ) options = client.options + truncated = options.pool_options.metadata["driver"] self.assertLessEqual( len(bson.encode(options.pool_options.metadata)), _MAX_METADATA_SIZE, ) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) client = self.simple_client( driver=DriverInfo(name="s" * _MAX_METADATA_SIZE, version="s" * _MAX_METADATA_SIZE), connect=False, ) options = client.options + truncated = options.pool_options.metadata["driver"] self.assertLessEqual( len(bson.encode(options.pool_options.metadata)), _MAX_METADATA_SIZE, ) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) + # An oversized wrapper name with no version must retain a truncated + # name rather than collapse to the base entry. + client = self.simple_client( + driver=DriverInfo(name="x" * (_MAX_METADATA_SIZE * 2), version=None), + connect=False, + ) + truncated = client.options.pool_options.metadata["driver"] + self.assertLessEqual( + len(bson.encode(client.options.pool_options.metadata)), + _MAX_METADATA_SIZE, + ) + self.assertIn("xxxx", truncated["name"]) + self.assertEqual( + truncated["name"].count("|"), + truncated["version"].count("|"), + ) + + def test_metadata_truncation_drops_blank_pairs_first(self): + # A platform-only driver appends a blank name/version pair. Truncation + # must drop that pair before trimming a real driver's content. + def metadata_for(name_len: int) -> dict[str, Any]: + name = "PyMongo||" + "W" * name_len # blank pair in the middle + return {"driver": {"name": name, "version": "1.0||1.0"}} + + # Size the name so the document is exactly two bytes over the limit: + # dropping the blank pair alone must make it fit. + name_len = next( + n + for n in range(1, 2 * _MAX_METADATA_SIZE) + if len(bson.encode(metadata_for(n))) == _MAX_METADATA_SIZE + 2 + ) + metadata = metadata_for(name_len) + _truncate_metadata(metadata) + self.assertEqual(len(bson.encode(metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + metadata["driver"], + {"name": "PyMongo|" + "W" * name_len, "version": "1.0|1.0"}, + ) + + def test_metadata_append_is_bounded(self): + # Successive appends must stay within the limit and keep name and + # version index-aligned after truncation. Once the metadata saturates, + # further appends must not grow the dedup tracking list. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name=f"D{i}", version=f"1.{i}")) + pool = client.options.pool_options + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + pool.metadata["driver"]["name"].count("|"), + pool.metadata["driver"]["version"].count("|"), + ) + count = len(pool._PoolOptions__appended_drivers) + for i in range(300, 600): + client.append_metadata(DriverInfo(name=f"D{i}", version=f"1.{i}")) + self.assertEqual(len(pool._PoolOptions__appended_drivers), count) + # Platform-only appends (empty name/version) stay bounded the same way. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name="", version="", platform=f"P{i}")) + pool = client.options.pool_options + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + count = len(pool._PoolOptions__appended_drivers) + for i in range(300, 600): + client.append_metadata(DriverInfo(name="", version="", platform=f"P{i}")) + self.assertEqual(len(pool._PoolOptions__appended_drivers), count) + + def test_metadata_recreates_platform_after_truncation(self): + # Appending a platform after truncation has dropped it recreates the field. + client = self.simple_client(connect=False) + for i in range(300): + client.append_metadata(DriverInfo(name="", version="", platform=f"Q{i}")) + pool = client.options.pool_options + self.assertLess(len(pool._PoolOptions__appended_drivers), 300) + client.append_metadata(DriverInfo(name="Wrapper", version="1.0", platform="Recreated")) + self.assertLessEqual(len(bson.encode(pool.metadata)), _MAX_METADATA_SIZE) + self.assertEqual( + pool.metadata["driver"]["name"].count("|"), + pool.metadata["driver"]["version"].count("|"), + ) + + def test_metadata_deduplicates_none_and_empty(self): + # Empty strings are treated as unset, so a duplicate differing only in + # None vs "" is a no-op. + client = self.simple_client(connect=False) + client.append_metadata(DriverInfo("library", None, "Library Platform")) + names = client.options.pool_options.metadata["driver"]["name"] + vers = client.options.pool_options.metadata["driver"]["version"] + client.append_metadata(DriverInfo("library", "", "Library Platform")) + metadata = client.options.pool_options.metadata + self.assertEqual(metadata["driver"]["name"], names) + self.assertEqual(metadata["driver"]["version"], vers) @mock.patch.dict("os.environ", {ENV_VAR_K8S: "1"}) def test_container_metadata(self): @@ -2179,6 +2293,9 @@ def _test_handshake(self, env_vars, expected_env): metadata["driver"]["name"] = "PyMongo|c" else: metadata["driver"]["name"] = "PyMongo" + metadata["driver"]["version"] = _driver_version( + _METADATA["driver"]["version"], metadata["driver"]["name"] + ) if expected_env is not None: metadata["env"] = expected_env @@ -2578,20 +2695,18 @@ def test_gevent_kill_churn_deadlock(self): import gevent.thread as _gthread from gevent import Timeout, spawn - # AMPLIFY_RACE=1 widens gevent's brief sleep on a contended lock - # so a kill lands there reliably. Only the bare sleep() is - # widened; timed sleeps pass through. The unfixed test then - # deadlocks within seconds. + # AMPLIFY_RACE=1 widens the kill windows below; keep AMPLIFY_SECONDS + # small or ops starve. + amplify_seconds = 0.0 if os.environ.get("AMPLIFY_RACE", "0") == "1": - _AMPLIFY_SECONDS = float(os.environ.get("AMPLIFY_SECONDS", "0.02")) + amplify_seconds = float(os.environ.get("AMPLIFY_SECONDS", "0.002")) _orig_thread_sleep = _gthread.sleep def _amplified_sleep(*args): - if not args: # bare sleep(): the courtesy yield on a failed - # non-blocking acquire (Condition.notify -> _is_owned -> acquire(False)) - _orig_thread_sleep(_AMPLIFY_SECONDS) - else: # sleep(0.001), sleep(2), etc.: passthrough + if not args: # bare sleep(): checkin courtesy yield + _orig_thread_sleep(amplify_seconds) + else: # timed sleeps pass through _orig_thread_sleep(*args) _gthread.sleep = _amplified_sleep @@ -2601,6 +2716,31 @@ def _amplified_sleep(*args): coll = client.pymongo_test.coll coll.insert_one({}) + pool = get_pool(client) # type:ignore + # Widen the post-gate checkout windows (PYTHON-6136). + if amplify_seconds: + + class _AmplifiedCondition: + def __init__(self, cond, seconds): + self._cond = cond + self._seconds = seconds + + def __enter__(self): + self._cond.__enter__() + return self + + def __exit__(self, *args): + self._cond.__exit__(*args) + time.sleep(self._seconds) + + def __getattr__(self, name): + return getattr(self._cond, name) + + pool.size_cond = _AmplifiedCondition(pool.size_cond, amplify_seconds) + pool._max_connecting_cond = _AmplifiedCondition( + pool._max_connecting_cond, amplify_seconds + ) + op_count = [0] running = [True] workers: list = [] @@ -2614,9 +2754,12 @@ def worker(): except Exception: return + # Scale the kill cadence or ops starve. + reaper_interval = max(0.003, amplify_seconds * 8) + def reaper(): while running[0]: - time.sleep(0.003) + time.sleep(reaper_interval) if not workers: continue idx = random.randrange(len(workers)) @@ -2657,11 +2800,6 @@ def reaper(): coll.find_one({}) except Timeout: self.fail("Pool gate saturated (PYTHON-6074)") - # Deterministic check: a saturated size gate pins the pool's - # checkout counters at maxPoolSize (PYTHON-6074). - pool = get_pool(client) # type:ignore - self.assertLess(pool.requests, pool.max_pool_size) - self.assertLess(pool.active_sockets, pool.max_pool_size) self.assertGreater(op_count[0], 0) finally: running[0] = False @@ -2679,6 +2817,14 @@ def reaper(): client.close() except Timeout: pass + # Post-settle: the counters must have fully drained. Note close() + # does not zero operation_count (only a fork does), so a leaked + # increment survives and is caught here. + time.sleep(1.0) + self.assertEqual(pool.requests, 0) + self.assertEqual(pool._pending, 0) + self.assertEqual(pool.active_sockets, 0) + self.assertEqual(pool.operation_count, 0) class TestClientLazyConnect(IntegrationTest): diff --git a/test/test_client_metadata.py b/test/test_client_metadata.py index f5ec92f2f3..b35e3f0bc7 100644 --- a/test/test_client_metadata.py +++ b/test/test_client_metadata.py @@ -18,7 +18,7 @@ import pathlib import time import unittest -from typing import Any, Optional +from typing import Any, Optional, cast import pytest @@ -99,20 +99,14 @@ def check_metadata_added( new_name, new_version, new_platform, new_metadata = self.send_ping_and_get_metadata( client, True ) - if add_name is not None and add_name.lower() in name.lower().split("|"): - self.assertEqual(name, new_name) - self.assertEqual(version, new_version) - self.assertEqual(platform, new_platform) - else: - self.assertEqual(new_name, f"{name}|{add_name}" if add_name is not None else name) - self.assertEqual( - new_version, - f"{version}|{add_version}" if add_version is not None else version, - ) - self.assertEqual( - new_platform, - f"{platform}|{add_platform}" if add_platform is not None else platform, - ) + # Name and version always get a delimiter (empty string if None) to + # preserve 1:1 index correspondence. + self.assertEqual(new_name, f"{name}|{add_name or ''}") + self.assertEqual(new_version, f"{version}|{add_version or ''}") + self.assertEqual( + new_platform, + f"{platform}|{add_platform}" if add_platform is not None else platform, + ) metadata.pop("driver") metadata.pop("platform") @@ -120,7 +114,7 @@ def check_metadata_added( new_metadata.pop("platform") self.assertEqual(metadata, new_metadata) - def test_append_metadata(self): + def test_1_test_that_the_driver_updates_metadata(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -128,7 +122,7 @@ def test_append_metadata(self): ) self.check_metadata_added(client, "framework", "2.0", "Framework Platform") - def test_append_metadata_platform_none(self): + def test_1_test_that_the_driver_updates_metadata_platform_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -136,7 +130,7 @@ def test_append_metadata_platform_none(self): ) self.check_metadata_added(client, "framework", "2.0", None) - def test_append_metadata_version_none(self): + def test_1_test_that_the_driver_updates_metadata_version_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -144,7 +138,7 @@ def test_append_metadata_version_none(self): ) self.check_metadata_added(client, "framework", None, "Framework Platform") - def test_append_metadata_platform_version_none(self): + def test_1_test_that_the_driver_updates_metadata_platform_version_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -152,14 +146,14 @@ def test_append_metadata_platform_version_none(self): ) self.check_metadata_added(client, "framework", None, None) - def test_multiple_successive_metadata_updates(self): + def test_2_multiple_successive_metadata_updates(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, connect=False ) client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) self.check_metadata_added(client, "framework", "2.0", "Framework Platform") - def test_multiple_successive_metadata_updates_platform_none(self): + def test_2_multiple_successive_metadata_updates_platform_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -167,7 +161,7 @@ def test_multiple_successive_metadata_updates_platform_none(self): client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) self.check_metadata_added(client, "framework", "2.0", None) - def test_multiple_successive_metadata_updates_version_none(self): + def test_2_multiple_successive_metadata_updates_version_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -175,7 +169,7 @@ def test_multiple_successive_metadata_updates_version_none(self): client.append_metadata(DriverInfo("library", "1.2", "Library Platform")) self.check_metadata_added(client, "framework", None, "Framework Platform") - def test_multiple_successive_metadata_updates_platform_version_none(self): + def test_2_multiple_successive_metadata_updates_platform_version_none(self): client = self.rs_or_single_client( "mongodb://" + self.server.address_string, maxIdleTimeMS=1, @@ -216,10 +210,16 @@ def test_duplicate_driver_name_no_op(self): self.check_metadata_added(client, "framework", None, None) # wait for connection to become idle time.sleep(0.005) - # add same metadata again - self.check_metadata_added(client, "Framework", None, None) + # Append the exact same DriverInfo again: no-op. + name, version, platform, _ = self.send_ping_and_get_metadata(client, True) + time.sleep(0.005) + client.append_metadata(DriverInfo("framework", None, None)) + new_name, new_version, new_platform, _ = self.send_ping_and_get_metadata(client, True) + self.assertEqual(new_name, name) + self.assertEqual(new_version, version) + self.assertEqual(new_platform, platform) - def test_handshake_documents_include_backpressure(self): + def test_9_handshake_documents_include_backpressure(self): # Create a `MongoClient` that is configured to record all handshake documents sent to the server as a part of # connection establishment. client = self.rs_or_single_client("mongodb://" + self.server.address_string) @@ -232,6 +232,131 @@ def test_handshake_documents_include_backpressure(self): # the document has a field `backpressure` whose value is `"2"`. self.assertEqual(self.handshake_req["backpressure"], "2") + def test_10_entries_in_driver_name_and_driver_version_correspond_by_index(self): + cases = [ + ("Gap in middle (name)", [(None, None), ("F2", None)], "||F2", "||"), + ("Gap in middle (version)", [("F1", None), ("F2", "2.0")], "|F1|F2", "||2.0"), + ("Trailing delimiter retained", [("F1", None)], "|F1", "|"), + ( + "Equal versions do not collapse", + [("F1", "{driver_version}")], + "|F1", + "|{driver_version}", + ), + ( + "Equal names do not collapse", + [("{driver_name}", "1.0")], + "|{driver_name}", + "|1.0", + ), + ("Duplicates deduplicate", [("F1", "1.0"), ("F1", "1.0")], "|F1", "|1.0"), + ("All versions absent", [("F1", None), ("F2", None)], "|F1|F2", "||"), + ("All names absent", [(None, "1.0"), (None, "2.0")], "||", "|1.0|2.0"), + ( + "Non-adjacent duplicate", + [("F1", "1.0"), ("F2", "2.0"), ("F1", "1.0")], + "|F1|F2", + "|1.0|2.0", + ), + ( + "Platform-only difference is not a duplicate", + [("F1", "1.0", "P1"), ("F1", "1.0", "P2")], + "|F1|F1", + "|1.0|1.0", + ), + ( + "Wrapper matching the driver's own identity", + [("{driver_name}", "{driver_version}")], + "|{driver_name}", + "|{driver_version}", + ), + ( + "Duplicates with an unset field deduplicate", + [("F1", None), ("F1", None)], + "|F1", + "|", + ), + ] + for ( + description, + appended, + expected_name_suffix, + expected_version_suffix, + ) in cases: + with self.subTest(description=description): + client = self.rs_or_single_client( + "mongodb://" + self.server.address_string, + maxIdleTimeMS=1, + ) + self.addCleanup(client.close) + # Capture the driver's own name and version from the first handshake. + name0, version0, _, _ = self.send_ping_and_get_metadata(client, True) + time.sleep(0.005) + + self.assertIsNotNone(name0) + self.assertIsNotNone(version0) + version0 = cast(str, version0) + driver_name = name0.split("|")[0] + driver_version = version0.split("|")[0] + + def resolve(value: Optional[str]) -> Optional[str]: + if value is None: + return None + return value.format(driver_name=driver_name, driver_version=driver_version) + + # Append each DriverInfo in order. + for opts in appended: + d_name = resolve(opts[0]) if len(opts) > 0 else None + d_version = resolve(opts[1]) if len(opts) > 1 else None + d_platform = resolve(opts[2]) if len(opts) > 2 else None + client.append_metadata(DriverInfo(d_name or "", d_version, d_platform)) + + # New handshake with the appended metadata. + name1, version1, _, _ = self.send_ping_and_get_metadata(client, True) + + self.assertEqual( + name1, + name0 + + expected_name_suffix.format( + driver_name=driver_name, driver_version=driver_version + ), + ) + self.assertEqual( + version1, + version0 + + expected_version_suffix.format( + driver_name=driver_name, driver_version=driver_version + ), + ) + + def test_11_appending_metadata_containing_the_delimiter_raises_an_error(self): + cases = [ + ("frame|work", "2.0", "Framework Platform"), + ("framework", "2|0", "Framework Platform"), + ("framework", "2.0", "Framework|Platform"), + ] + for name, version, platform in cases: + with self.subTest(name=name, version=version, platform=platform): + client = self.rs_or_single_client( + "mongodb://" + self.server.address_string, + maxIdleTimeMS=1, + driver=DriverInfo("library", "1.2", "Library Platform"), + ) + self.addCleanup(client.close) + # Send initial handshake. + name0, version0, platform0, _metadata = self.send_ping_and_get_metadata( + client, True + ) + time.sleep(0.005) + # Constructing metadata containing the delimiter raises. + with self.assertRaises(ValueError): + DriverInfo(name, version, platform) + # Metadata is unchanged on the next handshake. + name1, version1, platform1, _ = self.send_ping_and_get_metadata(client, True) + self.assertEqual(name1, name0) + self.assertEqual(version1, version0) + self.assertEqual(platform1, platform0) + if __name__ == "__main__": unittest.main() diff --git a/test/test_dns.py b/test/test_dns.py index e631420643..89dee6f3a7 100644 --- a/test/test_dns.py +++ b/test/test_dns.py @@ -207,30 +207,40 @@ def test_connect_case_insensitive(self): class TestInitialDnsSeedlistDiscovery(PyMongoTestCase): """ Initial DNS Seedlist Discovery prose tests - https://github.com/mongodb/specifications/blob/0a7a8b5/source/initial-dns-seedlist-discovery/tests/README.md#prose-tests + https://github.com/mongodb/specifications/blob/5036f26/source/initial-dns-seedlist-discovery/tests/README.md#prose-tests + + Numbered tests correspond to the numbered prose tests in the spec. The + unnumbered tests are PyMongo-specific additions with no spec counterpart. """ + def _parse(self, srv_hostname, mock_target, **kwargs): + """Resolve mongodb+srv:// with SRV records naming mock_target.""" + with patch("dns.resolver.resolve") as mock_resolver: + + def mock_resolve(query, record_type, *args, **kwargs): + mock_srv = MagicMock() + # Mirror dnspython: the wire form keeps the root label, and the + # caller strips it via omit_final_dot. + mock_srv.target.to_text.side_effect = lambda omit_final_dot=False: ( + mock_target.rstrip(".") if omit_final_dot else mock_target + ) + return [mock_srv] + + mock_resolver.side_effect = mock_resolve + return parse_uri(f"mongodb+srv://{srv_hostname}", **kwargs) + def run_initial_dns_seedlist_discovery_prose_tests(self, test_cases): for case in test_cases: - with patch("dns.resolver.resolve") as mock_resolver: - - def mock_resolve(query, record_type, *args, **kwargs): - mock_srv = MagicMock() - mock_srv.target.to_text.return_value = case["mock_target"] - return [mock_srv] - - mock_resolver.side_effect = mock_resolve - domain = case["query"].split("._tcp.")[1] - connection_string = f"mongodb+srv://{domain}" - if "expected_error" not in case: - parse_uri(connection_string) + domain = case["query"].split("._tcp.")[1] + if "expected_error" not in case: + self._parse(domain, case["mock_target"]) + else: + try: + self._parse(domain, case["mock_target"]) + except ConfigurationError as e: + self.assertIn(case["expected_error"], str(e)) else: - try: - parse_uri(connection_string) - except ConfigurationError as e: - self.assertIn(case["expected_error"], str(e)) - else: - self.fail(f"ConfigurationError was not raised for query: {case['query']}") + self.fail(f"ConfigurationError was not raised for query: {case['query']}") def test_1_allow_srv_hosts_with_fewer_than_three_dot_separated_parts(self): with patch("dns.resolver.resolve"): @@ -250,8 +260,8 @@ def test_2_throw_when_return_address_does_not_end_with_srv_domain(self): "expected_error": "Invalid SRV host", }, { - "query": "_mongodb._tcp.blogs.mongo.local", - "mock_target": "test_1.evil.com", + "query": "_mongodb._tcp.mongo.local", + "mock_target": "test_1.evil.local", "expected_error": "Invalid SRV host", }, ] @@ -294,7 +304,101 @@ def test_4_throw_when_return_address_does_not_contain_dot_separating_shared_part ] self.run_initial_dns_seedlist_discovery_prose_tests(test_cases) - def test_5_when_srv_hostname_has_three_or_more_dot_separated_parts_it_is_valid_for_the_returned_hostname_to_be_identical( + def test_5_srv_host_validator_accepts_a_host_the_default_verification_would_reject(self): + # "blogs.evil.com" does not share a parent domain with the seed, so the + # default check rejects it; the callback overrides that decision. + res = self._parse( + "blogs.mongodb.com", "blogs.evil.com", srv_host_validator=lambda host: True + ) + self.assertEqual(["blogs.evil.com"], [node[0] for node in res["nodelist"]]) + + # "mongo.local" does not add a domain level to an SRV hostname with fewer + # than three "." separated parts, which the default check also rejects. + res = self._parse("mongo.local", "mongo.local", srv_host_validator=lambda host: True) + self.assertEqual(["mongo.local"], [node[0] for node in res["nodelist"]]) + + def test_6_reject_a_host_the_default_verification_would_accept(self): + with self.assertRaisesRegex(ConfigurationError, "rejected by srv_host_validator"): + self._parse( + "blogs.mongodb.com", "cluster.mongodb.com", srv_host_validator=lambda host: False + ) + + def test_7_the_validator_receives_the_normalized_host_name(self): + seen = [] + + def validator(host): + seen.append(host) + return True + + self._parse("blogs.mongodb.com", "CLUSTER.MONGODB.COM.", srv_host_validator=validator) + self.assertEqual(["cluster.mongodb.com"], seen) + + def test_8_wrap_an_error_raised_by_the_validator(self): + original_exc = Exception("validator_error") + + def validator(host): + raise original_exc + + with self.assertRaisesRegex( + ConfigurationError, "srv_host_validator raised an exception" + ) as ctx: + self._parse("blogs.mongodb.com", "cluster.mongodb.com", srv_host_validator=validator) + # The wrapping error must retain the error raised by the validator. + self.assertIs(original_exc, ctx.exception.__cause__) + self.assertIn("validator_error", str(ctx.exception)) + + def test_9_throw_when_both_srv_allowed_hosts_suffix_and_srv_host_validator_are_configured( + self, + ): + # Rejected by the client + with self.assertRaisesRegex(ConfigurationError, "Cannot specify both"): + self.simple_client( + "mongodb+srv://blogs.mongodb.com", + srv_host_validator=lambda host: True, + srvAllowedHostsSuffix=".mongodb.com", + connect=False, + ) + + # Rejected by the resolver + with self.assertRaisesRegex(ConfigurationError, "Cannot specify both"): + self._parse( + "blogs.mongodb.com", + "cluster.mongodb.com", + srv_host_validator=lambda host: True, + srv_allowed_hosts_suffix=".mongodb.com", + ) + + def test_10_accept_a_mixed_case_returned_address_with_srv_allowed_hosts_suffix(self): + # Returned addresses are normalized before the suffix comparison, so + # the case DNS happens to use must not affect the result. + res = self._parse( + "blogs.mongodb.com", "CLUSTER.MONGODB.COM.", srv_allowed_hosts_suffix=".mongodb.com" + ) + self.assertEqual(["cluster.mongodb.com"], [node[0] for node in res["nodelist"]]) + + def test_11_throw_when_srv_host_validator_is_not_callable(self): + with self.assertRaisesRegex(ValueError, "must be a callable"): + self.simple_client("mongodb+srv://blogs.mongodb.com", srv_host_validator="notacallable") + + def test_12_accept_a_reserved_single_label_as_srv_allowed_hosts_suffix(self): + # A single label is a public suffix under the Public Suffix List's "*" + # rule, but the reserved names are accepted despite that. + res = self._parse( + "cluster.localhost", "db.cluster.localhost", srv_allowed_hosts_suffix="localhost" + ) + self.assertEqual(["db.cluster.localhost"], [node[0] for node in res["nodelist"]]) + + def test_13_throw_when_srv_host_validator_is_used_with_a_non_srv_uri(self): + with self.assertRaisesRegex( + ConfigurationError, "only allowed with 'mongodb\\+srv://' URIs" + ): + self.simple_client( + "mongodb://localhost:27017", + srv_host_validator=lambda host: True, + connect=False, + ) + + def test_srv_hostname_with_three_or_more_parts_may_equal_the_returned_hostname( self, ): test_cases = [ diff --git a/test/test_encryption.py b/test/test_encryption.py index adb6005ea1..2f9ed382ca 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -82,7 +82,11 @@ ) from pymongo.operations import InsertOne, ReplaceOne, UpdateOne from pymongo.synchronous import encryption -from pymongo.synchronous.encryption import Algorithm, ClientEncryption, QueryType +from pymongo.synchronous.encryption import ( + Algorithm, + ClientEncryption, + QueryType, +) from pymongo.synchronous.helpers import next from pymongo.synchronous.mongo_client import MongoClient from pymongo.write_concern import WriteConcern @@ -222,6 +226,9 @@ def test_init_kms_tls_options(self): self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) +# KMS connect callback tests live in test_kms_connect.py. + + class TestClientOptions(PyMongoTestCase): def test_default(self): client = self.simple_client(connect=False) @@ -316,9 +323,15 @@ def create_client_encryption( key_vault_client: MongoClient, codec_options: CodecOptions, kms_tls_options: Optional[Mapping[str, Any]] = None, + kms_connect_callback: Optional[Any] = None, ): client_encryption = ClientEncryption( - kms_providers, key_vault_namespace, key_vault_client, codec_options, kms_tls_options + kms_providers, + key_vault_namespace, + key_vault_client, + codec_options, + kms_tls_options, + kms_connect_callback=kms_connect_callback, ) self.addCleanup(client_encryption.close) return client_encryption @@ -331,9 +344,15 @@ def unmanaged_create_client_encryption( key_vault_client: MongoClient, codec_options: CodecOptions, kms_tls_options: Optional[Mapping[str, Any]] = None, + kms_connect_callback: Optional[Any] = None, ): client_encryption = ClientEncryption( - kms_providers, key_vault_namespace, key_vault_client, codec_options, kms_tls_options + kms_providers, + key_vault_namespace, + key_vault_client, + codec_options, + kms_tls_options, + kms_connect_callback=kms_connect_callback, ) return client_encryption @@ -1980,6 +1999,9 @@ def test_invalid_hostname_in_kms_certificate(self): self.client_encrypted.create_data_key("aws", master_key=key) +# KMS connect callback tests live in test_kms_connect.py. + + # https://github.com/mongodb/specifications/blob/master/source/client-side-encryption/tests/README.md#kms-tls-options-tests class TestKmsTLSOptions(EncryptionIntegrationTest): @unittest.skipUnless(any(AWS_CREDS.values()), "AWS environment credentials are not set") diff --git a/test/test_kms_connect.py b/test/test_kms_connect.py new file mode 100644 index 0000000000..8bd59ca689 --- /dev/null +++ b/test/test_kms_connect.py @@ -0,0 +1,447 @@ +"""Tests for the KMS connect callback.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import os +import socket +import ssl +import threading +import time +import unittest +from asyncio.trsock import TransportSocket +from unittest import mock + +import pytest + +import pymongo +from pymongo.encryption_options import ( + AutoEncryptionOpts, + KMSConnectContext, +) +from pymongo.errors import ConfigurationError, ConnectionFailure, EncryptionError, NetworkTimeout +from pymongo.pool_options import PoolOptions +from pymongo.ssl_support import get_ssl_context +from pymongo.synchronous.encryption import ( + ClientEncryption, + _connect_kms, + _EncryptionIO, + _wrap_encryption_errors, +) +from test import PyMongoTestCase +from test.helpers_shared import CA_PEM, CERT_PATH, CLIENT_PEM +from test.test_encryption import OPTS + +_IS_SYNC = True + +pytestmark = pytest.mark.encryption + +_KMS_ADDRESS = ("kms.example.com", 443) + + +def _tls_server_context(cert=CLIENT_PEM): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(cert) + return ctx + + +def _client_tls_context(verify=False): + # verify=False matches the driver's test mode: the local certs don't verify. + if verify: + return get_ssl_context(None, None, CA_PEM, None, False, False, False, _IS_SYNC) + return get_ssl_context(None, None, None, None, True, True, False, _IS_SYNC) + + +def _run_blocking(func, *args): + """Run a blocking callable off the event loop (inline in the sync version).""" + if _IS_SYNC: + return func(*args) + return asyncio.get_running_loop().run_in_executor(None, func, *args) + + +def _callback_returning(value): + """A kms_connect_callback that always produces ``value``.""" + + def callback(context): + return value + + return callback + + +class TestKmsConnectCallbackUnit(PyMongoTestCase): + """Contract checks for kms_connect_callback that need no KMS server.""" + + @staticmethod + def _pool_options(ssl_context=None): + return PoolOptions(connect_timeout=10, socket_timeout=10, ssl_context=ssl_context) + + def _listen(self, backlog=1): + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(backlog) + self.addCleanup(listener.close) + return listener + + def _socketpair(self): + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + return left, right + + def test_init_kms_connect_callback(self): + opts = AutoEncryptionOpts({}, "k.d") + self.assertIsNone(opts._kms_connect_callback) + + def callback(context): + raise AssertionError("not called") + + opts = AutoEncryptionOpts({}, "k.d", kms_connect_callback=callback) + self.assertIs(opts._kms_connect_callback, callback) + + for bad in [1, "not-callable", object()]: + with self.assertRaisesRegex(TypeError, "kms_connect_callback must be callable"): + AutoEncryptionOpts({}, "k.d", kms_connect_callback=bad) # type: ignore[arg-type] + + context = KMSConnectContext(host="kms.example.com", port=443, timeout=9.5) + self.assertEqual(context.host, "kms.example.com") + self.assertEqual(context.port, 443) + self.assertEqual(context.timeout, 9.5) + with self.assertRaises(dataclasses.FrozenInstanceError): + context.host = "evil.example.com" # type: ignore[misc] + + def test_non_socket_return_raises_configuration_error(self): + with self.assertRaisesRegex(ConfigurationError, "must return a connected"): + _connect_kms( + _KMS_ADDRESS, self._pool_options(), _callback_returning("not-a-socket"), 10.0 + ) + + def test_already_wrapped_socket_is_rejected(self): + # ssl.SSLSocket passes isinstance but cannot be TLS-wrapped again. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + left, _right = self._socketpair() + # No peer needed to produce a genuine ssl.SSLSocket. + wrapped = ctx.wrap_socket(left, do_handshake_on_connect=False, server_hostname="x") + self.addCleanup(wrapped.close) + + with self.assertRaisesRegex(ConfigurationError, "unwrapped"): + _connect_kms(_KMS_ADDRESS, self._pool_options(), _callback_returning(wrapped), 10.0) + + def test_context_receives_host_port_and_timeout(self): + received = [] + left, _right = self._socketpair() + + def callback(context): + received.append(context) + return left + + # ssl_context=None returns the socket unchanged, so a plain socket is accepted. + conn = _connect_kms(_KMS_ADDRESS, self._pool_options(), callback, 12.5) + self.assertIs(conn, left) + + self.assertEqual(len(received), 1) + self.assertEqual(received[0].host, "kms.example.com") + self.assertEqual(received[0].port, 443) + self.assertEqual(received[0].timeout, 12.5) + + def test_non_blocking_socket_from_callback_is_accepted(self): + # Without the driver normalizing the mode, this raises ValueError. + server_ctx = _tls_server_context() + listener = self._listen() + + def serve(): + try: + conn, _ = listener.accept() + server_ctx.wrap_socket(conn, server_side=True).close() + except OSError: + pass + + threading.Thread(target=serve, daemon=True).start() + + options = self._pool_options(_client_tls_context()) + + def connect(): + sock = socket.create_connection(listener.getsockname(), timeout=10) + sock.setblocking(False) + return sock + + def callback(context): + return _run_blocking(connect) + + conn = _connect_kms(listener.getsockname(), options, callback, 10.0) + self.addCleanup(conn.close) + self.assertIsNotNone(conn.gettimeout()) + + def test_tls_verification_targets_the_kms_host(self): + # The handshake must verify against the KMS address, not the peer the + # callback connected to. The server cert covers 127.0.0.1 (the peer) + # and localhost, but not the KMS hostname used below, so only + # address-based verification produces this outcome. + server_ctx = _tls_server_context(os.path.join(CERT_PATH, "server.pem")) + listener = self._listen(2) + + def serve(): + for _ in range(2): + try: + conn, _ = listener.accept() + server_ctx.wrap_socket(conn, server_side=True).close() + except (OSError, ssl.SSLError): + # The mismatched-name attempt fails mid-handshake. + pass + + threading.Thread(target=serve, daemon=True).start() + + # Full verification: trusted CA, invalid certs and hostnames rejected. + options = self._pool_options(_client_tls_context(verify=True)) + + created = [] + + def connect(): + sock = socket.create_connection(listener.getsockname(), timeout=10) + created.append(sock) + return sock + + def callback(context): + return _run_blocking(connect) + + port = listener.getsockname()[1] + # The cert covers localhost: verifying against the KMS address succeeds. + conn = _connect_kms(("localhost", port), options, callback, 10.0) + self.addCleanup(conn.close) + # TLS-wrapped in either SSL flavor: a new object, not the plain socket. + self.assertIsNot(conn, created[0]) + # The cert does not cover this name: verification must fail even + # though the peer (127.0.0.1) presents a cert valid for itself. + with self.assertRaises(ConnectionFailure): + _connect_kms(("kms.example.com", port), options, callback, 10.0) + + def test_asyncio_transport_socket_is_rejected(self): + # get_extra_info("socket") is a TransportSocket, not a socket.socket. + left, _right = self._socketpair() + + with self.assertRaisesRegex(ConfigurationError, "TransportSocket"): + _connect_kms( + _KMS_ADDRESS, self._pool_options(), _callback_returning(TransportSocket(left)), 10.0 + ) + + def test_cancelled_tls_wrap_closes_late_socket(self): + # A cancelled wrap can leave the executor producing an SSLSocket; the + # done callback must close it. + if _IS_SYNC: + raise unittest.SkipTest("the cancel-safe wrap is an async path") + from pymongo.pool_shared import _close_late_socket + + left, _right = self._socketpair() + future = asyncio.get_running_loop().create_future() + future.set_result(left) + self.assertNotEqual(left.fileno(), -1) + _close_late_socket(future) + self.assertEqual(left.fileno(), -1) + + def test_non_coroutine_callback_is_rejected(self): + # A plain def must be rejected before it blocks the event loop. + if _IS_SYNC: + raise unittest.SkipTest("a regular function is correct for the sync API") + + entered = [] + + def callback(context): + entered.append(context) + return None + + with self.assertRaisesRegex(ConfigurationError, "coroutine function"): + _connect_kms(_KMS_ADDRESS, self._pool_options(), callback, 10.0) + self.assertEqual(entered, [], "invalid callback must not be entered") + + def test_unconnected_socket_from_callback_is_rejected(self): + # An unconnected socket would fail later as a transient error and be retried. + bare = socket.socket() + self.addCleanup(bare.close) + + with self.assertRaisesRegex(ConfigurationError, "already connected"): + _connect_kms(_KMS_ADDRESS, self._pool_options(), _callback_returning(bare), 10.0) + + def test_datagram_socket_from_callback_is_rejected(self): + # TLS on a connected UDP socket raises NotImplementedError, which would be retried. + left = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + right = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(left.close) + self.addCleanup(right.close) + right.bind(("127.0.0.1", 0)) + left.connect(right.getsockname()) + + with self.assertRaisesRegex(ConfigurationError, "stream socket"): + _connect_kms(_KMS_ADDRESS, self._pool_options(), _callback_returning(left), 10.0) + + def test_kms_request_does_not_retry_a_contract_violation(self): + # _connect_kms has no retry loop; the no-retry guarantee is in + # kms_request, so exercise that instead. + calls = [] + + def callback(context): + calls.append(context) + return "not-a-socket" + + opts = AutoEncryptionOpts({}, "k.d", kms_connect_callback=callback) + io = _EncryptionIO(None, mock.MagicMock(), None, opts) + + class StubKmsContext: + endpoint = "kms.example.com:443" + message = b"request" + kms_provider = "aws" + usleep = 0 + bytes_needed = 1 + + def feed(self, data): + raise AssertionError("should not reach the socket") + + def fail(self): + raise AssertionError("a contract violation must not be retried") + + with self.assertRaises(ConfigurationError): + io.kms_request(StubKmsContext()) + self.assertEqual(len(calls), 1) + + def test_contract_violation_surfaces_as_encryption_error(self): + # Callers see EncryptionError with ConfigurationError as its cause. + with self.assertRaises(EncryptionError) as caught: + with _wrap_encryption_errors(): + raise ConfigurationError("kms_connect_callback must return ...") + self.assertIsInstance(caught.exception.__cause__, ConfigurationError) + + def test_network_error_from_callback_propagates(self): + def callback(context): + raise OSError("proxy unreachable") + + # Not a ConfigurationError, so kms_request retries it. + with self.assertRaises(OSError): + _connect_kms(_KMS_ADDRESS, self._pool_options(), callback, 10.0) + + def test_csot_deadline_stops_a_hung_callback(self): + # A callback that ignores the timeout cannot block past the CSOT + # deadline, and a socket it yields later must be closed. + if _IS_SYNC: + raise unittest.SkipTest("the sync API cannot interrupt a callback") + + left, _right = self._socketpair() + + def hung_callback(context): + time.sleep(0.5) + return left + + with self.assertRaises(NetworkTimeout): + with pymongo.timeout(0.1): + _connect_kms(_KMS_ADDRESS, self._pool_options(), hung_callback, 10.0) + self.assertNotEqual(left.fileno(), -1) + # Let the shielded callback finish; the driver closes the late result. + time.sleep(0.75) + self.assertEqual(left.fileno(), -1) + + def test_cancelling_kms_connect_closes_the_callback_socket(self): + # Cancelling during the TLS handshake must close the callback's socket, + # so a TLS proxy's relay threads wind down. + if _IS_SYNC: + raise unittest.SkipTest("cancellation is an async-only behavior") + + server_ctx = _tls_server_context() + listener = self._listen() + gate = threading.Event() + eof = threading.Event() + + def stub_server(): + conn = None + try: + conn, _ = listener.accept() + # The cancel may land before or after the executor starts the + # handshake. Peek for the ClientHello without consuming it, or + # for EOF if the driver closed it, before wrap_socket detaches conn. + while True: + data = conn.recv(4096, socket.MSG_PEEK) + if not data: + eof.set() + return + if data[:1] == b"\x16": # TLS handshake record + break + # Hold the handshake open until the test has cancelled. + if not gate.wait(5): + return + tls = server_ctx.wrap_socket(conn, server_side=True, do_handshake_on_connect=False) + try: + tls.do_handshake() + # A discarded connection may end in a reset rather than a + # clean EOF; either proves the driver closed it. + while tls.recv(4096): + pass + except OSError: + pass + eof.set() + tls.close() + except OSError: + pass + finally: + if conn is not None: + conn.close() + + threading.Thread(target=stub_server, daemon=True).start() + + options = self._pool_options(_client_tls_context()) + socks = [] + + def connect(): + sock = socket.create_connection(listener.getsockname(), timeout=10) + socks.append(sock) + return sock + + def callback(context): + return _run_blocking(connect) + + # The sync version returns a socket instead of a coroutine, so both + # error codes are needed depending on the version being checked. + pending = _connect_kms(listener.getsockname(), options, callback, 10.0) + task = asyncio.ensure_future(pending) # type: ignore[type-var,arg-type] + for _ in range(100): + if socks: + break + time.sleep(0.01) + self.assertTrue(socks, "callback was never invoked") + # Bias the cancel to land mid-handshake; the stub handles the earlier + # window too. + time.sleep(0.1) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + task + # The late SSLSocket (or raw socket) must be closed; the stub sees EOF. + gate.set() + for _ in range(50): + if eof.is_set(): + break + time.sleep(0.1) + self.assertTrue(eof.is_set(), "driver never closed the callback socket") + + def test_client_encryption_accepts_callback(self): + def callback(context): + raise AssertionError("not called") + + client = self.simple_client() + encryption = ClientEncryption( + {"local": {"key": b"\x00" * 96}}, + "keyvault.datakeys", + client, + OPTS, + kms_connect_callback=callback, + ) + self.addCleanup(encryption.close) + self.assertIs(encryption._io_callbacks.opts._kms_connect_callback, callback) + + def test_client_encryption_rejects_non_callable(self): + client = self.simple_client() + with self.assertRaisesRegex(TypeError, "kms_connect_callback must be callable"): + ClientEncryption( + {"local": {"key": b"\x00" * 96}}, + "keyvault.datakeys", + client, + OPTS, + kms_connect_callback="not-callable", # type: ignore[arg-type] + ) diff --git a/test/test_pooling.py b/test/test_pooling.py index a3f0eaf589..d690de0ffb 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -34,13 +34,19 @@ from pymongo.hello import HelloCompat from pymongo.lock import _create_lock from pymongo.monitoring import _EventListeners +from pymongo.pool_shared import _wrap_socket_tls from test.utils import flaky, get_pool, joinall sys.path[0:0] = [""] from pymongo.socket_checker import SocketChecker from pymongo.synchronous.pool import Pool, PoolOptions -from test import IntegrationTest, client_context, unittest +from test import ( + IntegrationTest, + PyMongoTestCase, + client_context, + unittest, +) from test.helpers import ConcurrentRunner from test.utils_shared import CMAPListener, delay @@ -306,6 +312,7 @@ def notify(): # Accounting was applied exactly once. self.assertEqual(0, cx_pool.requests) self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool.operation_count) def test_checkout_error_accounting_on_kill_during_acquire(self): # PYTHON-6074: an exception delivered while the checkout error @@ -341,6 +348,98 @@ def __exit__(self, *args): # The fallback applied the accounting exactly once. self.assertEqual(0, cx_pool.requests) self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool.operation_count) + + def test_checkout_error_accounting_on_connect_keyboard_interrupt(self): + # PYTHON-6136: a KeyboardInterrupt from connect() must roll back the + # size gate, the maxConnecting gate, and active_sockets. + cx_pool = self.create_pool(max_pool_size=1) + + with patch.object(cx_pool, "connect", side_effect=KeyboardInterrupt()): + with self.assertRaises(KeyboardInterrupt): + with cx_pool.checkout(): + pass + + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) + + def test_checkout_error_accounting_on_kill_during_pending_cleanup(self): + # PYTHON-6136: an interruption during the pending-gate cleanup must + # not skip the counter restore, which must wake threads waiting at + # the maxConnecting gate. + cx_pool = self.create_pool(max_pool_size=1) + + class _InterruptOnSecondEnter(type(cx_pool._max_connecting_cond)): + def __init__(self, lock): + super().__init__(lock) + self.enters = 0 + self.notifies = 0 + + def __enter__(self): + self.enters += 1 + if self.enters == 2: + # First enter is the checkout wait, second is connect()'s + # cleanup. Simulate a kill delivered while waiting there. + raise KeyboardInterrupt() + return super().__enter__() + + def __exit__(self, *args): + return super().__exit__(*args) + + def notify(self, n=1): + # The counter restore wakes the maxConnecting gate, then is + # itself killed. + self.notifies += 1 + raise KeyboardInterrupt() + + cond = _InterruptOnSecondEnter(cx_pool._max_connecting_cond._lock) + cx_pool._max_connecting_cond = cond + + with patch.object(cx_pool, "connect", side_effect=asyncio.CancelledError()): + with self.assertRaises(KeyboardInterrupt): + with cx_pool.checkout(): + pass + + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) + # The restore notified the maxConnecting gate; the notify interrupted + # by the kill was retried. + self.assertEqual(2, cond.notifies) + + def test_checkout_error_accounting_on_kill_during_pending_notify(self): + # PYTHON-6136: a kill landing inside the cleanup notify must not + # strand a checkout waiting at the maxConnecting gate. + cx_pool = self.create_pool(max_pool_size=1) + + class _InterruptOnFirstNotify(type(cx_pool._max_connecting_cond)): + def __init__(self, lock): + super().__init__(lock) + self.notifies = 0 + + def notify(self, n=1): + self.notifies += 1 + if self.notifies == 1: + # Simulate a kill delivered inside notify(). + raise KeyboardInterrupt() + + cond = _InterruptOnFirstNotify(cx_pool._max_connecting_cond._lock) + cx_pool._max_connecting_cond = cond + + with patch.object(cx_pool, "connect", side_effect=asyncio.CancelledError()): + with self.assertRaises(KeyboardInterrupt): + with cx_pool.checkout(): + pass + + # The cleanup notify was interrupted, then retried. + self.assertEqual(2, cond.notifies) + self.assertEqual(0, cx_pool.requests) + self.assertEqual(0, cx_pool.active_sockets) + self.assertEqual(0, cx_pool._pending) + self.assertEqual(0, cx_pool.operation_count) def test_pool_removes_closed_socket(self): # Test that Pool removes explicitly closed socket. @@ -463,6 +562,8 @@ def test_wait_queue_timeout(self): 1, f"Waited {duration:.2f} seconds for a socket, expected {wait_queue_timeout:f}", ) + # The load metric must not be inflated by the failed checkout. + self.assertEqual(0, pool.operation_count) def test_no_wait_queue_timeout(self): # Verify get_socket() with no wait_queue_timeout blocks forever. @@ -822,5 +923,18 @@ def test_certificate_error_is_not_labeled_overloaded(self): self.assertFalse(err.has_error_label("SystemOverloadedError")) +class TestWrapSocketTLS(PyMongoTestCase): + def test_wrap_socket_tls_without_ssl_context_returns_same_socket(self): + options = PoolOptions(socket_timeout=7.5) + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + + result = _wrap_socket_tls(left, ("kms.example.com", 443), options) + + self.assertIs(result, left) + self.assertEqual(result.gettimeout(), 7.5) + + if __name__ == "__main__": unittest.main() diff --git a/test/test_srv_polling.py b/test/test_srv_polling.py index e01ae1da7f..11ea739a55 100644 --- a/test/test_srv_polling.py +++ b/test/test_srv_polling.py @@ -19,7 +19,9 @@ import asyncio import sys import time +from types import SimpleNamespace from typing import Any +from unittest.mock import patch from test.utils_shared import FunctionCallRecorder @@ -37,6 +39,19 @@ WAIT_TIME = 0.1 +class _FakeSrvAnswer(list): + """A minimal stand-in for the dnspython answer to an SRV query.""" + + def __init__(self, hosts, ttl=60): + import dns.name + + hosts = [(h, 27017) if isinstance(h, str) else h for h in hosts] + super().__init__( + SimpleNamespace(target=dns.name.from_text(h), port=port) for h, port in hosts + ) + self.rrset = SimpleNamespace(ttl=ttl) + + class SrvPollingKnobs: def __init__( self, @@ -64,9 +79,15 @@ def enable(self): def mock_get_hosts_and_min_ttl(resolver, *args): assert self.old_dns_resolver_response is not None - nodes, ttl = self.old_dns_resolver_response(resolver) - if self.nodelist_callback is not None: - nodes = self.nodelist_callback() + if self.nodelist_callback is None: + nodes, ttl = self.old_dns_resolver_response(resolver) + else: + + def mock_resolve_uri(is_polling): + return _FakeSrvAnswer(self.nodelist_callback()) + + with patch.object(resolver, "_resolve_uri", mock_resolve_uri): + nodes, ttl = self.old_dns_resolver_response(resolver) if self.ttl_time is not None: ttl = self.ttl_time return nodes, ttl @@ -357,8 +378,92 @@ def nodelist_callback(): ) client._connect() with SrvPollingKnobs(nodelist_callback=nodelist_callback): + expected = [(host.rstrip("."), port) for host, port in response] + self.assert_nodelist_change(expected, client) + + def test_14_the_validator_is_consulted_when_srv_records_are_rescanned(self): + seen = [] + + def validator(host): + seen.append(host) + return True + + response = self.BASE_SRV_RESPONSE[:] + response.append(("localhost.test.build.10gen.cc", 27019)) + + with SrvPollingKnobs(ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME): + client = self.simple_client(self.CONNECTION_STRING, srv_host_validator=validator) + client._connect() + self.assert_nodelist_change(self.BASE_SRV_RESPONSE, client) + seen.clear() + with SrvPollingKnobs(nodelist_callback=lambda: response): self.assert_nodelist_change(response, client) + self.assertIn("localhost.test.build.10gen.cc", seen) + + def test_15_a_rejecting_or_raising_validator_does_not_raise_or_stop_polling(self): + def rejecting(host): + return False + + def raising(host): + raise RuntimeError("boom") + + response = self.BASE_SRV_RESPONSE[:] + response.append(("localhost.test.build.10gen.cc", 27019)) + + for failing in (rejecting, raising): + with self.subTest(validator=failing.__name__): + # Accept everything until the client is connected, then start failing. + state = {"validator": lambda host: True} + + with SrvPollingKnobs(ttl_time=WAIT_TIME, min_srv_rescan_interval=WAIT_TIME): + client = self.simple_client( + self.CONNECTION_STRING, + srv_host_validator=lambda host: state["validator"](host), + ) + client._connect() + self.assert_nodelist_change(self.BASE_SRV_RESPONSE, client) + + # The rescan sees the new record but the validator fails: no + # error reaches the application and the topology is unchanged. + state["validator"] = failing + with SrvPollingKnobs( + nodelist_callback=lambda: response, count_resolver_calls=True + ): + self.assert_nodelist_nochange(self.BASE_SRV_RESPONSE, client) + + # Polling was not stopped by the failures: once the validator + # accepts again, the new host is picked up. + state["validator"] = lambda host: True + with SrvPollingKnobs(nodelist_callback=lambda: response): + self.assert_nodelist_change(response, client) + + client.close() + + def test_rescan_skips_only_the_hosts_that_fail_verification(self): + # Per the polling spec, a host that fails verification is left out of + # the topology without discarding the other hosts from the same rescan. + good_host = "localhost.test.build.10gen.cc" + bad_host = "bad.evil.com" + + def raising(host): + if host == bad_host: + raise RuntimeError("some error") + return True + + def rejecting(host): + return host != bad_host + + for validator in (rejecting, raising): + with self.subTest(validator=validator.__name__): + resolver = pymongo.synchronous.srv_resolver._SrvResolver( + "test1.test.build.10gen.cc", None, "mongodb", srv_host_validator=validator + ) + answer = _FakeSrvAnswer([good_host, bad_host]) + with patch("dns.resolver.resolve", return_value=answer): + nodes, _ = resolver.get_hosts_and_min_ttl() + self.assertEqual([(good_host, 27017)], nodes) + def test_srv_waits_to_poll(self): modified = [("localhost.test.build.10gen.cc", 27019)] diff --git a/test/unified_format.py b/test/unified_format.py index 56ea563e6d..657d7c0f22 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -23,6 +23,7 @@ import binascii import copy import functools +import gc import os import platform import re @@ -538,6 +539,11 @@ def setUpClass(cls) -> None: @classmethod def tearDownClass(cls) -> None: cls.knobs.disable() + # Workaround for sockets leaked by tests (PYTHON-3923): force garbage + # collection so they are closed here, rather than during a later test + # where their ResourceWarnings could cause failures. Remove once the + # leaks are fixed. + gc.collect() def setUp(self): # super call creates internal client cls.client diff --git a/test/utils_shared.py b/test/utils_shared.py index 96f3b1f6ed..fb209d69a0 100644 --- a/test/utils_shared.py +++ b/test/utils_shared.py @@ -31,9 +31,11 @@ from collections import abc, defaultdict from functools import partial from inspect import iscoroutinefunction +from typing import Any from bson.objectid import ObjectId from pymongo import monitoring, operations, read_preferences +from pymongo.common import has_c from pymongo.cursor_shared import CursorType from pymongo.errors import OperationFailure from pymongo.helpers_shared import _SENSITIVE_COMMANDS @@ -51,6 +53,7 @@ PoolCreatedEvent, PoolReadyEvent, ) +from pymongo.pool_options import _METADATA from pymongo.pool_shared import _CancellationContext, _PoolGeneration from pymongo.read_concern import ReadConcern from pymongo.server_type import SERVER_TYPE @@ -791,3 +794,36 @@ def pack_msg_header(length: int, request_id: int, response_to: int, op_code: int production header-packing never does. """ return struct.pack(" str: + """Build a metadata driver version aligned 1:1 with ``name`` segments. + + The ``|c`` and ``|async`` name segments always have an empty version entry, + so the version string has one delimiter per name delimiter. ``last_version`` + is used when the final segment carries a wrapped driver's version. + """ + segments = [""] * name.count("|") + if last_version is not None: + segments[-1] = last_version + return "|".join([base_version, *segments]) + + +def _metadata_with_appended_driver( + is_sync: bool, name: str, version: str, platform: str | None = None +) -> dict[str, Any]: + """Build the expected client metadata after appending a driver.""" + driver_name = "PyMongo" + if has_c(): + driver_name += "|c" + if not is_sync: + driver_name += "|async" + metadata = copy.deepcopy(_METADATA) + metadata["driver"]["name"] = driver_name + f"|{name}" + metadata["driver"]["version"] = _driver_version( + _METADATA["driver"]["version"], metadata["driver"]["name"], last_version=version + ) + metadata["application"] = {"name": "foobar"} + if platform is not None: + metadata["platform"] = "{}|{}".format(_METADATA["platform"], platform) + return metadata diff --git a/tools/generate_sbom_requirements.py b/tools/generate_sbom_requirements.py index 59389bee5a..aa8d5579ca 100644 --- a/tools/generate_sbom_requirements.py +++ b/tools/generate_sbom_requirements.py @@ -3,7 +3,7 @@ - Includes repo_root/requirements.txt if present - Includes all files matching repo_root/requirements/**/*.txt -- Excludes docs.txt and test.txt in the requirements folder +- Excludes docs.txt, test.txt, and mod_wsgi.txt in the requirements folder - Writes output to sbom-requirements.txt (overwrites) """ @@ -11,7 +11,7 @@ from pathlib import Path -EXCLUDED_NAMES = {"docs.txt", "test.txt"} +EXCLUDED_NAMES = {"docs.txt", "test.txt", "mod_wsgi.txt"} def collect_files(root: Path) -> list[Path]: diff --git a/tools/synchro.py b/tools/synchro.py index bebf92c005..b5c6b2873a 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -72,6 +72,7 @@ "_a_grid_out_property": "_grid_out_property", "AsyncClientEncryption": "ClientEncryption", "AsyncMongoCryptCallback": "MongoCryptCallback", + "AsyncKMSConnectCallback": "KMSConnectCallback", "AsyncExplicitEncrypter": "ExplicitEncrypter", "AsyncAutoEncrypter": "AutoEncrypter", "AsyncContextManager": "ContextManager", @@ -127,6 +128,7 @@ "AsyncNetworkingInterface": "NetworkingInterface", "_configured_protocol_interface": "_configured_socket_interface", "_async_configured_socket": "_configured_socket", + "_async_wrap_socket_tls": "_wrap_socket_tls", "SpecRunnerTask": "SpecRunnerThread", "AsyncMockConnection": "MockConnection", "AsyncMockPool": "MockPool", @@ -297,6 +299,13 @@ def translate_docstrings(lines: list[str]) -> list[str]: lines[i] = lines[i].replace("an asynchronous", "a") if "An asynchronous" in lines[i]: lines[i] = lines[i].replace("An asynchronous", "A") + # This sentence states the callback contract, whose meaning + # would invert under the async -> sync word replacements. + if "Must be a coroutine function for the asynchronous API." in lines[i]: + lines[i] = lines[i].replace( + "Must be a coroutine function for the asynchronous API.", + "Must be a regular function.", + ) # This ensures docstring links are for `pymongo.X` instead of `pymongo.synchronous.X` if "pymongo.asynchronous" in lines[i] and "import" not in lines[i]: lines[i] = lines[i].replace("pymongo.asynchronous", "pymongo") diff --git a/uv.lock b/uv.lock index f2ae10151d..ee008e88d4 100644 --- a/uv.lock +++ b/uv.lock @@ -58,9 +58,9 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, - { name = "idna", marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "exceptiongroup" }, + { name = "idna" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/96/f0/5eb65b2bb0d09ac6776f2eb54adee6abe8228ea05b20a5ad0e4945de8aac/anyio-4.12.1.tar.gz", hash = "sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703", size = 228685, upload-time = "2026-01-06T11:45:21.246Z" } wheels = [ @@ -76,9 +76,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, - { name = "idna", marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.15'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" } wheels = [ @@ -358,9 +358,9 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "botocore", version = "1.42.97", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "jmespath", marker = "python_full_version < '3.10'" }, - { name = "s3transfer", version = "0.16.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "botocore", version = "1.42.97", source = { registry = "https://pypi.org/simple" } }, + { name = "jmespath" }, + { name = "s3transfer", version = "0.16.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/55/7d/5c6fa0bb9fd5caf865b9356411793900304328bcd0bc1eda96a32a1368a6/boto3-1.42.97.tar.gz", hash = "sha256:2833dbeda3670ea610ad48dff7d27cdc829dbbfcdfbc6b750b673948e949b6f0", size = 113217, upload-time = "2026-04-27T20:39:17.646Z" } wheels = [ @@ -376,9 +376,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "botocore", version = "1.43.94", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "jmespath", marker = "python_full_version >= '3.10'" }, - { name = "s3transfer", version = "0.19.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "botocore", version = "1.43.94", source = { registry = "https://pypi.org/simple" } }, + { name = "jmespath" }, + { name = "s3transfer", version = "0.19.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/d5/0f/a1d691e9c51ae5dce5a7394e550d7e36a995494e1c2795ce210d3520b257/boto3-1.43.94.tar.gz", hash = "sha256:aa832d88e85035cebb83f04700c65623328b88fe828ff7ed9539483bd9d608c5", size = 112663, upload-time = "2026-09-14T20:10:05.902Z" } wheels = [ @@ -394,9 +394,9 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "jmespath", marker = "python_full_version < '3.10'" }, - { name = "python-dateutil", marker = "python_full_version < '3.10'" }, - { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/c6/95/c37edb602948fad2253ffd1bb3dba5b938645bd1845ee4160350136a0f41/botocore-1.42.97.tar.gz", hash = "sha256:5c0bb00e32d16ff6d278cc8c9e10dc3672d9c1d569031635ac3c908a60de8310", size = 15269348, upload-time = "2026-04-27T20:39:05.625Z" } wheels = [ @@ -412,9 +412,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "jmespath", marker = "python_full_version >= '3.10'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.10'" }, - { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/5b/5a7cc096db8ddb5f6c221112d5a4a46888d9e38c1f5d5a06ae7e3e0755f2/botocore-1.43.94.tar.gz", hash = "sha256:cb97cedec5fceee3261f7fff75d0b0aa2cf68b32eeb2e8c2649e4657419773a6", size = 16104907, upload-time = "2026-09-14T20:10:00.315Z" } wheels = [ @@ -439,7 +439,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "pycparser", version = "2.23", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and implementation_name != 'PyPy'" }, + { name = "pycparser", version = "2.23", source = { registry = "https://pypi.org/simple" }, marker = "implementation_name != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ @@ -537,7 +537,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "pycparser", version = "3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and implementation_name != 'PyPy'" }, + { name = "pycparser", version = "3.0", source = { registry = "https://pypi.org/simple" }, marker = "implementation_name != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } wheels = [ @@ -830,7 +830,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } wheels = [ @@ -976,7 +976,7 @@ wheels = [ [package.optional-dependencies] toml = [ - { name = "tomli", marker = "python_full_version < '3.10'" }, + { name = "tomli" }, ] [[package]] @@ -1083,7 +1083,7 @@ wheels = [ [package.optional-dependencies] toml = [ - { name = "tomli", marker = "python_full_version >= '3.10' and python_full_version <= '3.11'" }, + { name = "tomli", marker = "python_full_version <= '3.11'" }, ] [[package]] @@ -1353,8 +1353,8 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version <= '3.9' and platform_python_implementation != 'PyPy'" }, - { name = "typing-extensions", marker = "python_full_version <= '3.9'" }, + { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation != 'PyPy'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/b2/7ffa7fe8207a8c42147ffe70c3e360b228160c1d85dc3faff16aaa3244c0/cryptography-47.0.0.tar.gz", hash = "sha256:9f8e55fe4e63613a5e1cc5819030f27b97742d720203a087802ce4ce9ceb52bb", size = 830863, upload-time = "2026-04-24T19:54:57.056Z" } wheels = [ @@ -1418,9 +1418,9 @@ resolution-markers = [ "python_full_version > '3.9' and python_full_version < '3.10'", ] dependencies = [ - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version > '3.9' and python_full_version < '3.10' and platform_python_implementation != 'PyPy'" }, + { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and platform_python_implementation != 'PyPy'" }, { name = "cffi", version = "2.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and platform_python_implementation != 'PyPy'" }, - { name = "typing-extensions", marker = "python_full_version > '3.9' and python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bb/ad/5d6702db60b1e40b41ef513b6967ff5848f307d50f8449baf1634f5908f1/cryptography-50.0.1.tar.gz", hash = "sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20", size = 880381, upload-time = "2026-08-25T19:45:45.499Z" } wheels = [ @@ -1520,7 +1520,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1554,10 +1554,10 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, - { name = "greenlet", version = "3.2.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and platform_python_implementation == 'CPython'" }, - { name = "zope-event", version = "6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "zope-interface", version = "8.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, + { name = "greenlet", version = "3.2.5", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation == 'CPython'" }, + { name = "zope-event", version = "6.0", source = { registry = "https://pypi.org/simple" } }, + { name = "zope-interface", version = "8.0.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/9f/5c/92002455a57cb3634383e2b822e3bccf409f43cde34528e46428971475cf/gevent-26.7.0.tar.gz", hash = "sha256:5b333a556e38a302b1b8c80525bef16d437e16f1e7767947789406841856a102", size = 6729213, upload-time = "2026-07-22T20:16:04.713Z" } wheels = [ @@ -1617,10 +1617,10 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "cffi", version = "2.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, - { name = "greenlet", version = "3.5.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and platform_python_implementation == 'CPython'" }, - { name = "zope-event", version = "6.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "zope-interface", version = "8.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "cffi", version = "2.1.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, + { name = "greenlet", version = "3.5.6", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation == 'CPython'" }, + { name = "zope-event", version = "6.2", source = { registry = "https://pypi.org/simple" } }, + { name = "zope-interface", version = "8.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/b8/eb/5f2db8013f1a4a6df2c23201f384a066f13ff5764a9f62a608c8a50ac8cc/gevent-26.8.0.tar.gz", hash = "sha256:96039f41bbde6dcd72559e5ffbd408a04f46774b47d991d4cf032da8fa79e5a0", size = 6625998, upload-time = "2026-08-10T18:02:28.038Z" } wheels = [ @@ -1903,7 +1903,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "zipp", version = "3.23.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "zipp", version = "3.23.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" } wheels = [ @@ -1919,7 +1919,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "zipp", version = "4.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "zipp", version = "4.1.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/7e/1e7e8dc30634b93ebb3d58a3dea569ad146e656218d3960ab04f62047b29/importlib_metadata-9.0.1.tar.gz", hash = "sha256:ab830580bc0ef3db61ce8fae716389e5462b67e033018bab6d8f80ef17172f99", size = 59124, upload-time = "2026-08-28T15:30:34.646Z" } wheels = [ @@ -2572,8 +2572,8 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "cryptography", version = "47.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version <= '3.9'" }, - { name = "typing-extensions", marker = "python_full_version <= '3.9'" }, + { name = "cryptography", version = "47.0.0", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1a/51/27a5ad5f939d08f690a326ef9582cda7140555180db71695f6fb747d6a36/pyopenssl-26.2.0.tar.gz", hash = "sha256:8c6fcecd1183a7fc897548dfe388b0cdb7f37e018200d8409cf33959dbe35387", size = 182195, upload-time = "2026-05-04T23:06:09.72Z" } wheels = [ @@ -2590,8 +2590,8 @@ resolution-markers = [ "python_full_version > '3.9' and python_full_version < '3.10'", ] dependencies = [ - { name = "cryptography", version = "50.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version > '3.9'" }, - { name = "typing-extensions", marker = "python_full_version > '3.9' and python_full_version < '3.13'" }, + { name = "cryptography", version = "50.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3f/e8/7325d258199b159eb2c03fe32107533e2832e70e63f4fb88a6aa00023201/pyopenssl-26.4.0.tar.gz", hash = "sha256:28dfcce0162b9211413e26dfbfdf1d24317fbeba18fc93c12400a1856b2a0bc7", size = 182046, upload-time = "2026-08-01T19:50:50.512Z" } wheels = [ @@ -2620,13 +2620,13 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, - { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "packaging", marker = "python_full_version < '3.10'" }, - { name = "pluggy", marker = "python_full_version < '3.10'" }, - { name = "pygments", marker = "python_full_version < '3.10'" }, - { name = "tomli", marker = "python_full_version < '3.10'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup" }, + { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } wheels = [ @@ -2642,13 +2642,13 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, - { name = "iniconfig", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "packaging", marker = "python_full_version >= '3.10'" }, - { name = "pluggy", marker = "python_full_version >= '3.10'" }, - { name = "pygments", marker = "python_full_version >= '3.10'" }, - { name = "tomli", marker = "python_full_version == '3.10.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig", version = "2.3.0", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ @@ -2664,9 +2664,9 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "backports-asyncio-runner", marker = "python_full_version < '3.10'" }, - { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "backports-asyncio-runner" }, + { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/86/9e3c5f48f7b7b638b216e4b9e645f54d199d7abbbab7a64a13b4e12ba10f/pytest_asyncio-1.2.0.tar.gz", hash = "sha256:c609a64a2a8768462d0c99811ddb8bd2583c33fd33cf7f21af1c142e824ffb57", size = 50119, upload-time = "2025-09-12T07:33:53.816Z" } wheels = [ @@ -2682,9 +2682,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "backports-asyncio-runner", marker = "python_full_version == '3.10.*'" }, - { name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.13'" }, + { name = "backports-asyncio-runner", marker = "python_full_version < '3.11'" }, + { name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } wheels = [ @@ -2734,10 +2734,10 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "certifi", marker = "python_full_version < '3.10'" }, - { name = "charset-normalizer", marker = "python_full_version < '3.10'" }, - { name = "idna", marker = "python_full_version < '3.10'" }, - { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } wheels = [ @@ -2753,10 +2753,10 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "certifi", marker = "python_full_version >= '3.10'" }, - { name = "charset-normalizer", marker = "python_full_version >= '3.10'" }, - { name = "idna", marker = "python_full_version >= '3.10'" }, - { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ @@ -2777,7 +2777,7 @@ name = "roman-numerals-py" version = "4.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "roman-numerals", marker = "python_full_version >= '3.11'" }, + { name = "roman-numerals" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/b5/de96fca640f4f656eb79bbee0e79aeec52e3e0e359f8a3e6a0d366378b64/roman_numerals_py-4.1.0.tar.gz", hash = "sha256:f5d7b2b4ca52dd855ef7ab8eb3590f428c0b1ea480736ce32b01fef2a5f8daf9", size = 4274, upload-time = "2025-12-17T18:25:41.153Z" } wheels = [ @@ -2818,7 +2818,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "botocore", version = "1.42.97", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "botocore", version = "1.42.97", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/46/29/af14f4ef3c11a50435308660e2cc68761c9a7742475e0585cd4396b91777/s3transfer-0.16.1.tar.gz", hash = "sha256:8e424355754b9ccb32467bdc568edf55be82692ef2002d934b1311dbb3b9e524", size = 154801, upload-time = "2026-04-22T20:36:06.475Z" } wheels = [ @@ -2834,7 +2834,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "botocore", version = "1.43.94", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "botocore", version = "1.43.94", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/76/43/35e4d8aa320bffe8287fe8f65f578fa2d2db0a64212f0e710dce58267854/s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993", size = 165592, upload-time = "2026-07-22T19:30:44.432Z" } wheels = [ @@ -3020,24 +3020,24 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "alabaster", version = "0.7.16", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "babel", marker = "python_full_version < '3.10'" }, - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version < '3.10'" }, - { name = "imagesize", version = "1.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "importlib-metadata", version = "8.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "jinja2", marker = "python_full_version < '3.10'" }, - { name = "packaging", marker = "python_full_version < '3.10'" }, - { name = "pygments", marker = "python_full_version < '3.10'" }, - { name = "requests", version = "2.32.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "snowballstemmer", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version < '3.10'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version < '3.10'" }, - { name = "tomli", marker = "python_full_version < '3.10'" }, + { name = "alabaster", version = "0.7.16", source = { registry = "https://pypi.org/simple" } }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize", version = "1.5.0", source = { registry = "https://pypi.org/simple" } }, + { name = "importlib-metadata", version = "8.7.1", source = { registry = "https://pypi.org/simple" } }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests", version = "2.32.5", source = { registry = "https://pypi.org/simple" } }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, + { name = "tomli" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5b/be/50e50cb4f2eff47df05673d361095cafd95521d2a22521b920c67a372dcb/sphinx-7.4.7.tar.gz", hash = "sha256:242f92a7ea7e6c5b406fdc2615413890ba9f699114a9c09192d7dfead2ee9cfe", size = 8067911, upload-time = "2024-07-20T14:46:56.059Z" } wheels = [ @@ -3052,23 +3052,23 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "alabaster", version = "1.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "babel", marker = "python_full_version == '3.10.*'" }, - { name = "colorama", marker = "python_full_version == '3.10.*' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version == '3.10.*'" }, - { name = "imagesize", version = "2.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "jinja2", marker = "python_full_version == '3.10.*'" }, - { name = "packaging", marker = "python_full_version == '3.10.*'" }, - { name = "pygments", marker = "python_full_version == '3.10.*'" }, - { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "snowballstemmer", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version == '3.10.*'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version == '3.10.*'" }, - { name = "tomli", marker = "python_full_version == '3.10.*'" }, + { name = "alabaster", version = "1.0.0", source = { registry = "https://pypi.org/simple" } }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize", version = "2.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" } }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, + { name = "tomli" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/be0b61178fe2cdcb67e2a92fc9ebb488e3c51c4f74a36a7824c0adf23425/sphinx-8.1.3.tar.gz", hash = "sha256:43c1911eecb0d3e161ad78611bc905d1ad0e523e4ddc202a58a821773dc4c927", size = 8184611, upload-time = "2024-10-13T20:27:13.93Z" } wheels = [ @@ -3083,23 +3083,23 @@ resolution-markers = [ "python_full_version >= '3.11'", ] dependencies = [ - { name = "alabaster", version = "1.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "babel", marker = "python_full_version >= '3.11'" }, - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version >= '3.11'" }, - { name = "imagesize", version = "2.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "jinja2", marker = "python_full_version >= '3.11'" }, - { name = "packaging", marker = "python_full_version >= '3.11'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "roman-numerals-py", marker = "python_full_version >= '3.11'" }, - { name = "snowballstemmer", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version >= '3.11'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version >= '3.11'" }, + { name = "alabaster", version = "1.0.0", source = { registry = "https://pypi.org/simple" } }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize", version = "2.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" } }, + { name = "roman-numerals-py" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/38/ad/4360e50ed56cb483667b8e6dadf2d3fda62359593faabbe749a27c4eaca6/sphinx-8.2.3.tar.gz", hash = "sha256:398ad29dee7f63a75888314e9424d40f52ce5a6a87ae88e7071e80af296ec348", size = 8321876, upload-time = "2025-03-02T22:31:59.658Z" } wheels = [ @@ -3116,16 +3116,16 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "colorama" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "starlette", version = "0.49.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "starlette", version = "0.49.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "starlette", version = "1.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "uvicorn", version = "0.39.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "uvicorn", version = "0.39.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "uvicorn", version = "0.53.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "watchfiles", version = "1.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "watchfiles", version = "1.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "watchfiles", version = "1.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "websockets", version = "15.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "websockets", version = "15.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "websockets", version = "16.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a5/2c/155e1de2c1ba96a72e5dba152c509a8b41e047ee5c2def9e9f0d812f8be7/sphinx_autobuild-2024.10.3.tar.gz", hash = "sha256:248150f8f333e825107b6d4b86113ab28fa51750e5f9ae63b59dc339be951fb1", size = 14023, upload-time = "2024-10-02T23:15:30.172Z" } @@ -3141,12 +3141,12 @@ resolution-markers = [ "python_full_version >= '3.11'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11'" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "starlette", version = "1.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "uvicorn", version = "0.53.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "watchfiles", version = "1.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "websockets", version = "17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "colorama" }, + { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" } }, + { name = "starlette", version = "1.6.0", source = { registry = "https://pypi.org/simple" } }, + { name = "uvicorn", version = "0.53.0", source = { registry = "https://pypi.org/simple" } }, + { name = "watchfiles", version = "1.2.0", source = { registry = "https://pypi.org/simple" } }, + { name = "websockets", version = "17.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/e0/3c/a59a3a453d4133777f7ed2e83c80b7dc817d43c74b74298ca0af869662ad/sphinx_autobuild-2025.8.25.tar.gz", hash = "sha256:9cf5aab32853c8c31af572e4fecdc09c997e2b8be5a07daf2a389e270e85b213", size = 15200, upload-time = "2025-08-25T18:44:55.436Z" } wheels = [ @@ -3280,8 +3280,8 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "anyio", version = "4.12.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "anyio", version = "4.12.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/de/1a/608df0b10b53b0beb96a37854ee05864d182ddd4b1156a22f1ad3860425a/starlette-0.49.3.tar.gz", hash = "sha256:1c14546f299b5901a1ea0e34410575bc33bbd741377a10484a54445588d00284", size = 2655031, upload-time = "2025-11-01T15:12:26.13Z" } wheels = [ @@ -3297,8 +3297,8 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "anyio", version = "4.15.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.13'" }, + { name = "anyio", version = "4.15.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b5/b4/205b0d5241d934e8add0c38aa924c4f9fb7330834ff11e5444db964ec3f9/starlette-1.6.0.tar.gz", hash = "sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b", size = 2716969, upload-time = "2026-08-08T18:27:57.512Z" } wheels = [ @@ -3426,9 +3426,9 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "h11", marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" } }, + { name = "h11" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ae/4f/f9fdac7cf6dd79790eb165639b5c452ceeabc7bbabbba4569155470a287d/uvicorn-0.39.0.tar.gz", hash = "sha256:610512b19baa93423d2892d7823741f6d27717b642c8964000d7194dded19302", size = 82001, upload-time = "2025-12-21T13:05:17.973Z" } wheels = [ @@ -3444,9 +3444,9 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "click", version = "8.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "h11", marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version == '3.10.*'" }, + { name = "click", version = "8.5.0", source = { registry = "https://pypi.org/simple" } }, + { name = "h11" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5d/ad/04bbb797c84fc1f26cb171f7394716f4865ffb8d8c5e1eef42565c2dfa6b/uvicorn-0.53.0.tar.gz", hash = "sha256:a9356f0cb89b3b8621529c5d5eebd69bfe154f4c3f68b4cf2de47e45fa855c2e", size = 110881, upload-time = "2026-09-14T07:44:23.815Z" } wheels = [ @@ -3462,7 +3462,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "anyio", version = "4.12.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "anyio", version = "4.12.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/c2/c9/8869df9b2a2d6c59d79220a4db37679e74f807c559ffe5265e08b227a210/watchfiles-1.1.1.tar.gz", hash = "sha256:a173cb5c16c4f40ab19cecf48a534c409f7ea983ab8fed0741304a1c0a31b3f2", size = 94440, upload-time = "2025-10-14T15:06:21.08Z" } wheels = [ @@ -3585,7 +3585,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "anyio", version = "4.15.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "anyio", version = "4.15.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/41/5e1a4bb12aac5f1493fa1bdc11154eca3b258ca4eba65d39c473fe19d8e9/watchfiles-1.2.0.tar.gz", hash = "sha256:c995fba777f1ea992f090f9236e9284cf7a5d1a0130dd5a3d82c598cacd76838", size = 108252, upload-time = "2026-05-18T04:32:04.251Z" } wheels = [ @@ -4135,7 +4135,7 @@ resolution-markers = [ "python_full_version <= '3.9'", ] dependencies = [ - { name = "setuptools", version = "82.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "setuptools", version = "82.0.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/c2/d8/9c8b0c6bb1db09725395618f68d3b8a08089fca0aed28437500caaf713ee/zope_event-6.0.tar.gz", hash = "sha256:0ebac894fa7c5f8b7a89141c272133d8c1de6ddc75ea4b1f327f00d1f890df92", size = 18731, upload-time = "2025-09-12T07:10:13.551Z" } wheels = [