From e072c4b2d9c4d81763a0ad25070364e30dae7b0a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 25 Sep 2026 16:00:57 -0500 Subject: [PATCH 1/2] PYTHON-6051 Use the drivers-evergreen-tools submodule everywhere Add mongodb-labs/drivers-evergreen-tools as a git submodule pinned to v1.1.0 and point every Evergreen, spawn-host, and local-dev script at that in-tree checkout instead of the external DRIVERS_TOOLS clone, defaulting DRIVERS_TOOLS to the submodule while still letting an env var override win. Initialize the submodule in configure-env.sh and setup-dev-env.sh (Evergreen's git.get_project does not init submodules), pin the GitHub Actions reference, and update CONTRIBUTING.md. Because the submodule lives inside the project directory, uv's config discovery from the tools' own scripts (uv venv / uv export in install-cli.sh, run by setup.sh and run-mongodb.sh) reached this project's pyproject.toml and enforced the [tool.uv] required-version pin against whatever uv those scripts run (the host image's uv, or the uv~=0.8.0 shim install-cli.sh installs), failing every server-starting task. Write a uv.toml configuration boundary into the tools checkout (write-if-absent) so the pin applies only to pymongo's own uv invocations, record it in the submodule's local info/exclude so routine setup leaves a clean working tree, and document the boundary. Also: - cleanup.sh: remove ignored credential/state files the tools scripts write inside the tools checkout (secrets-export.sh with CSFLE Azure secrets, AWS creds json, token_file.txt); they survive git submodule update on reused hosts. - utils.py: check_drivers_tools requires the .evergreen/run-mongodb.sh sentinel instead of is_dir(), which passes on the empty directory an uninitialized submodule leaves behind. - gitignore the evergreen-written test-results.json and the AI-assistant review workflow logs. (commit --no-verify: the intentional submodule addition trips the forbid-new-submodules hook, which this diff deliberately does not change.) --- .evergreen/scripts/cleanup.sh | 15 +++++- .evergreen/scripts/configure-env.sh | 53 +++++++++++++++++--- .evergreen/scripts/create-spec-pr.sh | 6 +-- .evergreen/scripts/install-dependencies.sh | 7 ++- .evergreen/scripts/run-getdata.sh | 7 +++ .evergreen/scripts/run_server.py | 20 +++----- .evergreen/scripts/setup-dev-env.sh | 33 ++++++++++++ .evergreen/scripts/setup_tests.py | 10 ++-- .evergreen/scripts/stop-server.sh | 4 ++ .evergreen/scripts/utils.py | 18 ++++++- .github/dependabot.yml | 9 ++++ .github/workflows/test-python.yml | 17 ++++--- .github/zizmor.yml | 2 +- .gitignore | 6 ++- .gitmodules | 3 ++ CONTRIBUTING.md | 58 +++++++++++++++++++--- drivers-evergreen-tools | 1 + pyproject.toml | 5 ++ 18 files changed, 227 insertions(+), 47 deletions(-) create mode 100644 .gitmodules create mode 160000 drivers-evergreen-tools diff --git a/.evergreen/scripts/cleanup.sh b/.evergreen/scripts/cleanup.sh index f04a936fd2..db4f86a4d8 100755 --- a/.evergreen/scripts/cleanup.sh +++ b/.evergreen/scripts/cleanup.sh @@ -10,5 +10,18 @@ if [ -f $HERE/env.sh ]; then source $HERE/env.sh fi -rm -rf "${DRIVERS_TOOLS}" || true +# DRIVERS_TOOLS now points inside the checkout (the drivers-evergreen-tools +# submodule); deleting it would corrupt the workdir for later tasks on the +# same host, so it is intentionally not removed here. +# +# The tools scripts write ignored credential and state files inside the tools +# checkout (secrets-export.sh — csfle's setup-secrets.sh appends Azure client +# secrets to it — plus token_file.txt and AWS creds json), and +# `git submodule update` does not remove ignored files, so clean the checkout +# of all untracked and ignored files to keep credentials from carrying into +# later tasks on a reused host. Default to the submodule when unset; a +# caller-provided DRIVERS_TOOLS (including the value baked into env.sh) wins, +# so the checkout actually used is the one cleaned. +: "${DRIVERS_TOOLS:=$HERE/../../drivers-evergreen-tools}" rm -f $HERE/../../secrets-export.sh || true +git -C "$DRIVERS_TOOLS" clean -fdx 2>/dev/null || true diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index fec3cdadae..df4307022b 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -12,7 +12,9 @@ else fi PROJECT_DIRECTORY="$(pwd)" -DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools" +# Default to the submodule checkout; an env var override wins, mirroring +# install-dependencies.sh, run-getdata.sh, stop-server.sh, and utils.py. +DRIVERS_TOOLS="${DRIVERS_TOOLS:-$PROJECT_DIRECTORY/drivers-evergreen-tools}" CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin" MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" @@ -93,12 +95,51 @@ export PROJECT="${project:-mongo-python-driver}" export PIP_QUIET=1 EOT -# Write the .env file for drivers-tools. -rm -rf $DRIVERS_TOOLS -BRANCH=master -ORG=mongodb-labs -git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS +# Initialize the drivers-evergreen-tools submodule (Evergreen's +# git.get_project does not init submodules). Checks out the gitlink recorded +# in this checkout. Tolerate non-git contexts (rsync'd spawn hosts) with a +# warning; the checkout contents are still present there. +if ! git -C "$PROJECT_DIRECTORY" submodule update --init --recursive; then + echo "WARNING: could not initialize the drivers-evergreen-tools submodule;" \ + "using the existing checkout contents instead." +fi + +# Write a uv configuration boundary into the drivers-evergreen-tools checkout. +# +# The submodule is vendored INSIDE the project directory, so uv's config +# discovery from the tools' own scripts (uv venv and uv export in +# install-cli.sh, invoked via setup.sh and run-mongodb.sh) would otherwise +# walk up out of the submodule into pyproject.toml and enforce this project's +# [tool.uv] required-version pin against whatever uv those scripts happen to +# run (the host image's uv, or the "uv~=0.8.0" shim install-cli.sh installs), +# failing on any mismatch. A uv.toml here stops that discovery at the +# submodule boundary, leaving the pin to apply only to this project's own uv +# invocations. +# +# The file is untracked in the submodule and only written when absent, so a +# submodule checkout that gains its own uv.toml makes "git submodule update" +# fail loudly rather than being silently clobbered. +if [ -d "${DRIVERS_TOOLS}" ] && [ ! -f "${DRIVERS_TOOLS}/uv.toml" ]; then + cat < "${DRIVERS_TOOLS}/uv.toml" +# Configuration boundary written by the mongo-python-driver scripts; see +# .evergreen/scripts/configure-env.sh. Keeps uv's config discovery from +# reaching the vendoring project's pyproject.toml and its required-version pin. +EOT +fi + +# Keep the boundary out of git status: an untracked uv.toml marks the parent +# checkout dirty with modified submodule content after every setup. Add it to +# the submodule's local exclude (.git/modules/.../info/exclude) rather than +# its tracked .gitignore, so the pinned checkout stays untouched. No-op +# without git (uninitialized submodule, rsync'd spawn hosts), where there is +# no status to keep clean. +if _git_dir=$(git -C "${DRIVERS_TOOLS}" rev-parse --absolute-git-dir 2>/dev/null); then + mkdir -p "${_git_dir}/info" + grep -qxF "uv.toml" "${_git_dir}/info/exclude" 2>/dev/null || + printf "uv.toml\n" >> "${_git_dir}/info/exclude" +fi +# Write the .env file for drivers-tools. cat < ${DRIVERS_TOOLS}/.env SKIP_LEGACY_SHELL=1 DRIVERS_TOOLS="$DRIVERS_TOOLS" diff --git a/.evergreen/scripts/create-spec-pr.sh b/.evergreen/scripts/create-spec-pr.sh index 1de1a7cae1..e50da127de 100755 --- a/.evergreen/scripts/create-spec-pr.sh +++ b/.evergreen/scripts/create-spec-pr.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash -tools="$(realpath -s "../drivers-tools")" +# Default to the drivers-evergreen-tools submodule when unset; a +# caller-provided DRIVERS_TOOLS wins, as in the other consumers. +tools="$(realpath -s "${DRIVERS_TOOLS:-./drivers-evergreen-tools}")" pushd $tools/.evergreen/github_app || exit owner="mongodb" @@ -46,5 +48,3 @@ resp=$(curl -L \ --url https://api.github.com/repos/$owner/$repo/pulls) echo $resp | jq '.html_url' echo "Creating the PR... done." - -rm -rf $tools diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index 36e7866735..a0f42b375b 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -60,8 +60,11 @@ fi # Set up uv if needed. if [ "$_need_setup" = "1" ]; then - # ensure-uv.sh (drivers-evergreen-tools) finds or installs uv and scopes its env. - if [ -n "${DRIVERS_TOOLS:-}" ] && [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then + # ensure-uv.sh (drivers-evergreen-tools) finds or installs uv and scopes its + # env. Default DRIVERS_TOOLS to the drivers-evergreen-tools submodule so it + # is used whenever present; an env var override wins. + : "${DRIVERS_TOOLS:=$(dirname "$(dirname "$HERE")")/drivers-evergreen-tools}" + if [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then . "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ensure_uv || exit 1 fi diff --git a/.evergreen/scripts/run-getdata.sh b/.evergreen/scripts/run-getdata.sh index 9435a5fcc3..416744ac03 100755 --- a/.evergreen/scripts/run-getdata.sh +++ b/.evergreen/scripts/run-getdata.sh @@ -2,6 +2,13 @@ # Get the debug data for an evergreen task. set -eu +HERE=$(dirname ${BASH_SOURCE:-$0}) +HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" +ROOT=$(dirname "$(dirname $HERE)") + +# Default to the drivers-evergreen-tools submodule when unset. +: "${DRIVERS_TOOLS:=$ROOT/drivers-evergreen-tools}" + . ${DRIVERS_TOOLS}/.evergreen/get-distro.sh || true get_distro || true echo $DISTRO diff --git a/.evergreen/scripts/run_server.py b/.evergreen/scripts/run_server.py index f81964f367..ef452f1060 100644 --- a/.evergreen/scripts/run_server.py +++ b/.evergreen/scripts/run_server.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -from utils import DRIVERS_TOOLS, ROOT, get_test_options, run_command +from utils import DRIVERS_TOOLS, ROOT, check_drivers_tools, get_test_options, run_command def set_env(name: str, value: Any = "1") -> None: @@ -14,11 +14,9 @@ def set_env(name: str, value: Any = "1") -> None: def start_server(): - run_mongodb_script = ( - Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh" if DRIVERS_TOOLS else None - ) + run_mongodb_script = Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh" want_help = bool({"-h", "--help"} & set(sys.argv[1:])) - if want_help and run_mongodb_script and run_mongodb_script.is_file(): + if want_help and run_mongodb_script.is_file(): # Forward straight to run-mongodb.sh's own help, without run_command's # "Running command..." logging noise. subprocess.run( # noqa: S603 @@ -28,13 +26,11 @@ def start_server(): ) return - # DRIVERS_TOOLS is only needed to actually start a server. When it's unset and - # -h/--help was requested, fall through to get_test_options' own argparse help below. - if not want_help and not DRIVERS_TOOLS: - raise ValueError( - "DRIVERS_TOOLS is not set; run `just run-server` from an Evergreen task " - "or set DRIVERS_TOOLS to a drivers-evergreen-tools checkout." - ) + # DRIVERS_TOOLS is only needed to actually start a server. When the + # submodule is missing and -h/--help was requested, fall through to + # get_test_options' own argparse help below. + if not want_help: + check_drivers_tools() opts, extra_opts = get_test_options( "Run a MongoDB server. All given flags will be passed to run-mongodb.sh in DRIVERS_TOOLS.", diff --git a/.evergreen/scripts/setup-dev-env.sh b/.evergreen/scripts/setup-dev-env.sh index 508eb099bc..05f32630ac 100755 --- a/.evergreen/scripts/setup-dev-env.sh +++ b/.evergreen/scripts/setup-dev-env.sh @@ -53,6 +53,39 @@ if [ "${CI:-}" != "true" ] && [ "${GITHUB_ACTIONS:-}" != "true" ]; then printf 'export PATH="%s:$PATH"\n' "$PYMONGO_BIN_DIR_POSIX" >> "$_rc" fi +# Initialize the drivers-evergreen-tools submodule before +# install-dependencies.sh (which sources ensure-uv.sh from the tools +# checkout). Evergreen's git.get_project does not init submodules, so this +# must happen in our scripts. Tolerate non-git contexts (containers) with a +# warning rather than a hard failure. +if ! git -C "$ROOT" submodule update --init --recursive; then + echo "WARNING: could not initialize the drivers-evergreen-tools submodule;" \ + "set DRIVERS_TOOLS to a drivers-evergreen-tools checkout instead." +fi + +# Mirror configure-env.sh: write the uv configuration boundary into the +# submodule checkout so uv's config discovery from the tools' own scripts +# (which run uv versions we do not pin) cannot reach the project's +# [tool.uv] required-version pin. Only meaningful when the submodule checkout +# exists; see configure-env.sh for the full explanation. +if [ -d "$ROOT/drivers-evergreen-tools" ] && [ ! -f "$ROOT/drivers-evergreen-tools/uv.toml" ]; then + cat < "$ROOT/drivers-evergreen-tools/uv.toml" +# Configuration boundary written by the mongo-python-driver scripts; see +# .evergreen/scripts/configure-env.sh. Keeps uv's config discovery from +# reaching the vendoring project's pyproject.toml and its required-version pin. +EOT +fi + +# Keep the boundary out of git status, as in configure-env.sh: add it to the +# submodule's local exclude so the untracked uv.toml does not mark the parent +# checkout dirty. No-op without git (uninitialized submodule, rsync'd spawn +# hosts). +if _git_dir=$(git -C "$ROOT/drivers-evergreen-tools" rev-parse --absolute-git-dir 2>/dev/null); then + mkdir -p "${_git_dir}/info" + grep -qxF "uv.toml" "${_git_dir}/info/exclude" 2>/dev/null || + printf "uv.toml\n" >> "${_git_dir}/info/exclude" +fi + # Ensure dependencies are installed. bash $HERE/install-dependencies.sh diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 5a3f441f58..4ec219795d 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -17,6 +17,7 @@ ROOT, TEST_SUITE_MAP, Distro, + check_drivers_tools, get_test_options, read_env, run_command, @@ -327,8 +328,7 @@ def handle_test_env() -> None: MULTI_MONGOS_LB_URI += "&tls=true" write_env("SINGLE_MONGOS_LB_URI", SINGLE_MONGOS_LB_URI) write_env("MULTI_MONGOS_LB_URI", MULTI_MONGOS_LB_URI) - if not DRIVERS_TOOLS: - raise RuntimeError("Missing DRIVERS_TOOLS") + check_drivers_tools() cmd = f'bash "{DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh" start' run_command(cmd) @@ -374,8 +374,7 @@ def handle_test_env() -> None: run_command(cmd, cwd=DRIVERS_TOOLS) if SSL != "nossl": - if not DRIVERS_TOOLS: - raise RuntimeError("Missing DRIVERS_TOOLS") + check_drivers_tools() write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem") write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem") @@ -427,8 +426,7 @@ def handle_test_env() -> None: # PATH is updated by configure-env.sh for access to mongocryptd. if test_name == "encryption": - if not DRIVERS_TOOLS: - raise RuntimeError("Missing DRIVERS_TOOLS") + check_drivers_tools() csfle_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/csfle") # Opt in to corporate Azure credentials (DRIVERS-3392) os.environ["FLE_AZURE_USE_CORPORATE"] = "YES" diff --git a/.evergreen/scripts/stop-server.sh b/.evergreen/scripts/stop-server.sh index 045a655cbd..a254a2715a 100755 --- a/.evergreen/scripts/stop-server.sh +++ b/.evergreen/scripts/stop-server.sh @@ -4,6 +4,7 @@ set -eu HERE=$(dirname ${BASH_SOURCE:-$0}) HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" +ROOT=$(dirname "$(dirname $HERE)") # Try to source the env file. if [ -f $HERE/env.sh ]; then @@ -11,4 +12,7 @@ if [ -f $HERE/env.sh ]; then source $HERE/env.sh fi +# Default to the drivers-evergreen-tools submodule when unset. +: "${DRIVERS_TOOLS:=$ROOT/drivers-evergreen-tools}" + bash ${DRIVERS_TOOLS}/.evergreen/run-mongodb.sh stop diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index a7decdfaa8..36b6509c09 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -12,7 +12,11 @@ HERE = Path(__file__).absolute().parent ROOT = HERE.parent.parent -DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/") +# DRIVERS_TOOLS defaults to the drivers-evergreen-tools submodule; an env var +# override wins. +DRIVERS_TOOLS = (os.environ.get("DRIVERS_TOOLS") or str(ROOT / "drivers-evergreen-tools")).replace( + os.sep, "/" +) TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz" # noqa: S108 LOGGER = logging.getLogger("test") @@ -309,6 +313,18 @@ def run_command(cmd: str | list[str], **kwargs: Any) -> None: LOGGER.info("Running command '%s'... done.", cmd) +def check_drivers_tools() -> None: + """Raise a clear error when the drivers-evergreen-tools checkout is missing.""" + # An uninitialized submodule can exist as an empty directory, so a bare + # is_dir() check passes. Require a script every consumer needs instead. + if not (Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh").is_file(): + raise RuntimeError( + "The drivers-evergreen-tools checkout is missing or empty; run `just " + "install` to initialize the submodule, or set DRIVERS_TOOLS to a " + "drivers-evergreen-tools checkout." + ) + + def create_archive() -> str: run_command("git add .", cwd=ROOT) run_command('git commit --no-verify -m "add files"', check=False, cwd=ROOT) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 72d1e1e084..de7dd1125f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,3 +19,12 @@ updates: schedule: interval: "weekly" open-pull-requests-limit: 0 + # drivers-evergreen-tools submodule + - package-ecosystem: "gitsubmodule" + directory: "/" + schedule: + interval: "weekly" + cooldown: + default-days: 7 + allow: + - dependency-name: "drivers-evergreen-tools" diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 5e0d89f7a4..f2e40b12aa 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -85,7 +85,7 @@ jobs: # The beta Python here relies on the action's prerelease default. python-version: ${{ matrix.python-version }} - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "${{ matrix.mongodb-version }}" - name: Run tests @@ -101,12 +101,13 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false + submodules: true - name: Install Python tooling uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 with: python-version: "3.10" - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "8.0" - name: Setup tests @@ -126,6 +127,7 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false + submodules: true - name: Install Python tooling uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 with: @@ -133,7 +135,7 @@ jobs: - name: Install dependencies run: just install - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "8.0" - name: Run tests @@ -184,6 +186,7 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false + submodules: true - name: Install Python tooling uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 with: @@ -191,12 +194,12 @@ jobs: - name: Install dependencies run: just install - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 - name: Run tests run: | just integration-tests - id: setup-mongodb-ssl - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: ssl: true - name: Run tests @@ -251,7 +254,7 @@ jobs: # Test sdist on lowest supported Python python-version: "3.9" - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 - name: Run connect test from sdist shell: bash run: | @@ -275,7 +278,7 @@ jobs: with: python-version: "3.9" - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "8.0" - name: Run tests diff --git a/.github/zizmor.yml b/.github/zizmor.yml index af9f715dd4..2a69f0dd1a 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -4,4 +4,4 @@ rules: policies: actions/*: ref-pin mongodb-labs/drivers-github-tools/*: hash-pin - mongodb-labs/drivers-evergreen-tools: ref-pin + mongodb-labs/drivers-evergreen-tools: hash-pin diff --git a/.gitignore b/.gitignore index 8546aeabb7..2b0dc771b0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ build/ doc/_build/ dist/ tools/settings.py -drivers-evergreen-tools pymongo.egg-info/ *.so *.egg* @@ -47,3 +46,8 @@ xunit-results/ coverage.xml server.log .coverage +test-results.json + +# AI-assistant review workflow logs (kept untracked) +/REVIEW_STATE.md +/REVIEW.md diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..d89986aa20 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "drivers-evergreen-tools"] + path = drivers-evergreen-tools + url = https://github.com/mongodb-labs/drivers-evergreen-tools.git diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 20b7210736..8ba1d64c32 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -203,9 +203,12 @@ the pages will re-render and the browser will automatically refresh. `just install` installs the pinned version of `uv` (from `[tool.uv] required-version`) into `$HOME/.local/bin`, and adds that directory to your shell rc file when missing, so the pinned `uv` takes effect in new shells. If a project `uv` command (e.g. `just test`) runs with a different `uv` version, `uv` fails fast and tells you how to update. + Scripts inside the vendored `drivers-evergreen-tools` submodule are exempt: the setup scripts write a `uv.toml` + boundary file into that checkout (see `.evergreen/scripts/configure-env.sh`), which stops uv's config discovery + there, since the tools' scripts run uv versions this project does not pin. - Ensure you have started the appropriate Mongo Server(s). You can run `just run-server` with optional args to set up the server. All given options will be passed to - [`run-mongodb.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-mongodb.sh). Run `$DRIVERS_TOOLS/.evergreen/run-mongodb.sh start -h` + [`run-mongodb.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-mongodb.sh). Run `${DRIVERS_TOOLS:-drivers-evergreen-tools}/.evergreen/run-mongodb.sh start -h` for a full list of options. - Run `just test` or `pytest` to run all of the tests. - Append `test/.py::::` to run @@ -220,10 +223,10 @@ the pages will re-render and the browser will automatically refresh. ### Prerequisites -- Clone `drivers-evergreen-tools`: - `git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`. -- Run `export DRIVERS_TOOLS=$PWD/drivers-evergreen-tools`. This can be put into a `.bashrc` file - for convenience. +- The `drivers-evergreen-tools` submodule (see + [The drivers-evergreen-tools submodule](#the-drivers-evergreen-tools-submodule)). + `just install` initializes it; no manual clone or `export DRIVERS_TOOLS` is needed. + `DRIVERS_TOOLS` remains an optional override for an existing local checkout. - Some tests require access to [Drivers test secrets](https://github.com/mongodb-labs/drivers-evergreen-tools/tree/master/.evergreen/secrets_handling#secrets-handling). ### Usage @@ -356,7 +359,7 @@ You will need to set up access to the `drivers-test-secrets-role`, see the [Wiki ### OCSP tests - Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`. -This corresponds to a config file in `$DRIVERS_TOOLS/.evergreen/orchestration/configs/servers`. +This corresponds to a config file in `${DRIVERS_TOOLS:-drivers-evergreen-tools}/.evergreen/orchestration/configs/servers`. MongoDB servers on MacOS and Windows do not staple OCSP responses and only support RSA. NOTE: because the mock ocsp responder MUST be started prior to the server starting, the ocsp tests start the server as part of `setup-tests`. @@ -372,6 +375,44 @@ If you are running one of the `no-responder` tests, omit the `run-server` step. - Set up the tests with `sync` or `async`: `just setup-tests perf sync`. - Run the tests: `just run-tests`. +## The drivers-evergreen-tools submodule + +The `drivers-evergreen-tools` repository is consumed as a git submodule at the repo root, +pinned to a specific commit. Dependabot bumps the pin weekly. + +### Daily flow + +Nothing to do: `just install` initializes the submodule, and Dependabot keeps it fresh. + +### Manually bumping the submodule + +```bash +git -C drivers-evergreen-tools fetch --tags +git -C drivers-evergreen-tools checkout vX.Y.Z +git add drivers-evergreen-tools +``` + +### Evergreen patches that need tools changes + +Point the submodule at the needed commit and commit the new gitlink in the patch branch; +`configure-env.sh` checks out the recorded SHA on Evergreen hosts (`setup-dev-env.sh` does +the same for local checkouts with `just install`). + +### Using a local checkout instead + +Set `DRIVERS_TOOLS` to the path of a local clone — the environment variable wins over the +submodule default: + +```bash +export DRIVERS_TOOLS=/path/to/drivers-evergreen-tools +``` + +Alternatively, keep a local pin from being reset by `git submodule update`: + +```bash +git config submodule.drivers-evergreen-tools.update none +``` + ## Enable Debug Logs - Use `-o log_cli_level="DEBUG" -o log_cli=1` with `just test` or `pytest` to output all debug logs to the terminal. **Warning**: This will output a huge amount of logs. @@ -415,7 +456,10 @@ tasks are host-agnostic. - The uv binary version is pinned once in `[tool.uv] required-version` in `pyproject.toml`. `.evergreen/scripts/install-dependencies.sh` installs it with `uv tool install`, uv enforces it locally, and `astral-sh/setup-uv` reads it on GitHub. Bump it manually when a newer uv is needed. If uv cannot find the - requested Python, it installs it; if that fails, the task fails. + requested Python, it installs it; if that fails, the task fails. The pin is not enforced for scripts inside + the vendored `drivers-evergreen-tools` submodule: those scripts run uv versions this project does not pin, so + the setup scripts write a `uv.toml` boundary into the submodule checkout that stops uv's config discovery + (and with it the required-version check) at the submodule boundary. - Regenerate the test variants and tasks using `pre-commit run --all-files generate-config`. - Make sure to add instructions for running the test suite to `CONTRIBUTING.md`. diff --git a/drivers-evergreen-tools b/drivers-evergreen-tools new file mode 160000 index 0000000000..ec0b1497b3 --- /dev/null +++ b/drivers-evergreen-tools @@ -0,0 +1 @@ +Subproject commit ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 diff --git a/pyproject.toml b/pyproject.toml index 2f7a68c837..cf824a72f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,11 @@ validate-bump = false [tool.hatch.build.targets.wheel] packages = ["bson","gridfs", "pymongo"] +[tool.hatch.build.targets.sdist] +# The drivers-evergreen-tools submodule must not ship in the sdist; hatchling's +# walk would otherwise pick it up when the submodule is populated locally. +exclude = ["drivers-evergreen-tools"] + [tool.hatch.metadata.hooks.requirements_txt] files = ["requirements.txt"] From 560272a5526f02117119142e77294ca25de7ab31 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 2 Oct 2026 16:37:13 -0500 Subject: [PATCH 2/2] PYTHON-6051 Only touch the in-tree submodule when it is the checkout in use --- .evergreen/scripts/configure-env.sh | 51 ++++++++++++++++------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index 99028860bc..5e36125838 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -94,33 +94,40 @@ export PROJECT="${project:-mongo-python-driver}" export PIP_QUIET=1 EOT -# Initialize the submodule (Evergreen's git.get_project does not); tolerate -# non-git hosts with a warning. -if ! git -C "$PROJECT_DIRECTORY" submodule update --init --recursive; then - echo "WARNING: could not initialize the drivers-evergreen-tools submodule;" \ - "using the existing checkout contents instead." -fi - -# Write a uv config boundary into the submodule: it is vendored inside this -# project, so uv run by the tools' own scripts would otherwise walk up to -# pyproject.toml and enforce our required-version pin. Write-if-absent, so an -# upstream uv.toml fails the submodule update loudly instead of being clobbered. -if [ -d "${DRIVERS_TOOLS}" ] && [ ! -f "${DRIVERS_TOOLS}/uv.toml" ]; then - cat < "${DRIVERS_TOOLS}/uv.toml" +# Only touch the in-tree submodule when it is the checkout actually in use; +# an overridden DRIVERS_TOOLS is a checkout we do not own. +if [ "$DRIVERS_TOOLS" = "$PROJECT_DIRECTORY/drivers-evergreen-tools" ]; then + # Initialize the submodule (Evergreen's git.get_project does not); tolerate + # non-git hosts with a warning. + if ! git -C "$PROJECT_DIRECTORY" submodule update --init --recursive; then + echo "WARNING: could not initialize the drivers-evergreen-tools submodule;" \ + "using the existing checkout contents instead." + fi + + # Write a uv config boundary into the submodule: it is vendored inside this + # project, so uv run by the tools' own scripts would otherwise walk up to + # pyproject.toml and enforce our required-version pin. Write-if-absent, so an + # upstream uv.toml fails the submodule update loudly instead of being clobbered. + if [ -d "${DRIVERS_TOOLS}" ] && [ ! -f "${DRIVERS_TOOLS}/uv.toml" ]; then + cat < "${DRIVERS_TOOLS}/uv.toml" # Written by mongo-python-driver to stop uv's config discovery here; see # .evergreen/scripts/configure-env.sh. EOT + fi + + # Keep the boundary out of git status via the submodule's local exclude; + # no-op without git. + if _git_dir=$(git -C "${DRIVERS_TOOLS}" rev-parse --absolute-git-dir 2>/dev/null); then + mkdir -p "${_git_dir}/info" + grep -qxF "uv.toml" "${_git_dir}/info/exclude" 2>/dev/null || + printf "uv.toml\n" >> "${_git_dir}/info/exclude" + fi fi -# Keep the boundary out of git status via the submodule's local exclude; -# no-op without git. -if _git_dir=$(git -C "${DRIVERS_TOOLS}" rev-parse --absolute-git-dir 2>/dev/null); then - mkdir -p "${_git_dir}/info" - grep -qxF "uv.toml" "${_git_dir}/info/exclude" 2>/dev/null || - printf "uv.toml\n" >> "${_git_dir}/info/exclude" -fi - -# Write the .env file for drivers-tools. +# Write the .env file for drivers-tools. Create the checkout if it is missing so +# a failed submodule init does not stop this script before setup-tests.py can +# reach check_drivers_tools() and report the problem actionably. +mkdir -p "${DRIVERS_TOOLS}" cat < ${DRIVERS_TOOLS}/.env SKIP_LEGACY_SHELL=1 DRIVERS_TOOLS="$DRIVERS_TOOLS"