PYTHON-6051 Convert drivers-evergreen-tools to a git submodule and pin the GitHub Actions reference

.evergreen/scripts/cleanup.sh

@@ -10,5 +10,10 @@ if [ -f $HERE/env.sh ]; then
   source $HERE/env.sh
 fi
 
-rm -rf "${DRIVERS_TOOLS}" || true
+# Don't delete the in-tree submodule; clean the ignored credential and state
+# files (secrets-export.sh, AWS creds, token files) that `git submodule update`
+# leaves behind, so they can't carry into later tasks on a reused host. A
+# caller-provided DRIVERS_TOOLS wins, so the checkout actually used is cleaned.
+: "${DRIVERS_TOOLS:=$HERE/../../drivers-evergreen-tools}"
 rm -f $HERE/../../secrets-export.sh || true
+git -C "$DRIVERS_TOOLS" clean -fdx 2>/dev/null || true

.evergreen/scripts/configure-env.sh

@@ -12,7 +12,8 @@ else
 fi
 
 PROJECT_DIRECTORY="$(pwd)"
-DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools"
+# Default to the submodule; an env var override wins.
+DRIVERS_TOOLS="${DRIVERS_TOOLS:-$PROJECT_DIRECTORY/drivers-evergreen-tools}"
 CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo}
 DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin"
 MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
@@ -93,12 +94,40 @@ export PROJECT="${project:-mongo-python-driver}"
 export PIP_QUIET=1
 EOT
 
-# Write the .env file for drivers-tools.
-rm -rf $DRIVERS_TOOLS
-BRANCH=master
-ORG=mongodb-labs
-git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS
+# Only touch the in-tree submodule when it is the checkout actually in use;
+# an overridden DRIVERS_TOOLS is a checkout we do not own.
+if [ "$DRIVERS_TOOLS" = "$PROJECT_DIRECTORY/drivers-evergreen-tools" ]; then
+  # Initialize the submodule (Evergreen's git.get_project does not); tolerate
+  # non-git hosts with a warning.
+  if ! git -C "$PROJECT_DIRECTORY" submodule update --init --recursive; then
+    echo "WARNING: could not initialize the drivers-evergreen-tools submodule;" \
+      "using the existing checkout contents instead."
+  fi
 
+  # Write a uv config boundary into the submodule: it is vendored inside this
+  # project, so uv run by the tools' own scripts would otherwise walk up to
+  # pyproject.toml and enforce our required-version pin. Write-if-absent, so an
+  # upstream uv.toml fails the submodule update loudly instead of being clobbered.
+  if [ -d "${DRIVERS_TOOLS}" ] && [ ! -f "${DRIVERS_TOOLS}/uv.toml" ]; then
+    cat <<EOT > "${DRIVERS_TOOLS}/uv.toml"
+# Written by mongo-python-driver to stop uv's config discovery here; see
+# .evergreen/scripts/configure-env.sh.
+EOT
+  fi
+
+  # Keep the boundary out of git status via the submodule's local exclude;
+  # no-op without git.
+  if _git_dir=$(git -C "${DRIVERS_TOOLS}" rev-parse --absolute-git-dir 2>/dev/null); then
+    mkdir -p "${_git_dir}/info"
+    grep -qxF "uv.toml" "${_git_dir}/info/exclude" 2>/dev/null ||
+      printf "uv.toml\n" >> "${_git_dir}/info/exclude"
+  fi
+fi
+
+# Write the .env file for drivers-tools. Create the checkout if it is missing so
+# a failed submodule init does not stop this script before setup-tests.py can
+# reach check_drivers_tools() and report the problem actionably.
+mkdir -p "${DRIVERS_TOOLS}"
 cat <<EOT > ${DRIVERS_TOOLS}/.env
 SKIP_LEGACY_SHELL=1
 DRIVERS_TOOLS="$DRIVERS_TOOLS"

.evergreen/scripts/create-spec-pr.sh

@@ -1,7 +1,9 @@
 #!/usr/bin/env bash
 set -eu
 
-tools="$(realpath -s "../drivers-tools")"
+# Default to the drivers-evergreen-tools submodule when unset; a
+# caller-provided DRIVERS_TOOLS wins, as in the other consumers.
+tools="$(realpath -s "${DRIVERS_TOOLS:-./drivers-evergreen-tools}")"
 pushd $tools/.evergreen/github_app || exit
 
 owner="mongodb"
@@ -116,5 +118,3 @@ if [ -z "$pr_url" ]; then
 fi
 echo "$pr_url"
 echo "Creating the PR... done."
-
-rm -rf $tools

.evergreen/scripts/install-dependencies.sh

@@ -67,8 +67,9 @@ fi
 
 # Set up uv if needed.
 if [ "$_need_setup" = "1" ]; then
-  # ensure-uv.sh (drivers-evergreen-tools) finds or installs uv and scopes its env.
-  if [ -n "${DRIVERS_TOOLS:-}" ] && [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then
+  # Default DRIVERS_TOOLS to the submodule; an env var override wins.
+  : "${DRIVERS_TOOLS:=$(dirname "$(dirname "$HERE")")/drivers-evergreen-tools}"
+  if [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then
     . "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh"
     ensure_uv || exit 1
   fi

.evergreen/scripts/run-getdata.sh

@@ -2,6 +2,13 @@
 # Get the debug data for an evergreen task.
 set -eu
 
+HERE=$(dirname ${BASH_SOURCE:-$0})
+HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )"
+ROOT=$(dirname "$(dirname $HERE)")
+
+# Default to the drivers-evergreen-tools submodule when unset.
+: "${DRIVERS_TOOLS:=$ROOT/drivers-evergreen-tools}"
+
 . ${DRIVERS_TOOLS}/.evergreen/get-distro.sh || true
 get_distro || true
 echo $DISTRO

.evergreen/scripts/run_server.py

@@ -6,19 +6,17 @@
 from pathlib import Path
 from typing import Any
 
-from utils import DRIVERS_TOOLS, ROOT, get_test_options, run_command
+from utils import DRIVERS_TOOLS, ROOT, check_drivers_tools, get_test_options, run_command
 
 
 def set_env(name: str, value: Any = "1") -> None:
     os.environ[name] = str(value)
 
 
 def start_server():
-    run_mongodb_script = (
-        Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh" if DRIVERS_TOOLS else None
-    )
+    run_mongodb_script = Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh"
     want_help = bool({"-h", "--help"} & set(sys.argv[1:]))
-    if want_help and run_mongodb_script and run_mongodb_script.is_file():
+    if want_help and run_mongodb_script.is_file():
         # Forward straight to run-mongodb.sh's own help, without run_command's
         # "Running command..." logging noise.
         subprocess.run(  # noqa: S603
@@ -28,13 +26,10 @@ def start_server():
         )
         return
 
-    # DRIVERS_TOOLS is only needed to actually start a server. When it's unset and
-    # -h/--help was requested, fall through to get_test_options' own argparse help below.
-    if not want_help and not DRIVERS_TOOLS:
-        raise ValueError(
-            "DRIVERS_TOOLS is not set; run `just run-server` from an Evergreen task "
-            "or set DRIVERS_TOOLS to a drivers-evergreen-tools checkout."
-        )
+    # DRIVERS_TOOLS is only needed to start a server; allow -h/--help to fall
+    # through to argparse when the submodule is missing.
+    if not want_help:
+        check_drivers_tools()
 
     opts, extra_opts = get_test_options(
         "Run a MongoDB server.  All given flags will be passed to run-mongodb.sh in DRIVERS_TOOLS.",

.evergreen/scripts/setup-dev-env.sh

@@ -53,6 +53,29 @@ if [ "${CI:-}" != "true" ] && [ "${GITHUB_ACTIONS:-}" != "true" ]; then
     printf 'export PATH="%s:$PATH"\n' "$PYMONGO_BIN_DIR_POSIX" >> "$_rc"
 fi
 
+# Initialize the submodule (Evergreen's git.get_project does not); tolerate
+# non-git hosts with a warning.
+if ! git -C "$ROOT" submodule update --init --recursive; then
+  echo "WARNING: could not initialize the drivers-evergreen-tools submodule;" \
+    "set DRIVERS_TOOLS to a drivers-evergreen-tools checkout instead."
+fi
+
+# Mirror configure-env.sh's uv config boundary; see it for the full explanation.
+if [ -d "$ROOT/drivers-evergreen-tools" ] && [ ! -f "$ROOT/drivers-evergreen-tools/uv.toml" ]; then
+  cat <<EOT > "$ROOT/drivers-evergreen-tools/uv.toml"
+# Written by mongo-python-driver to stop uv's config discovery here; see
+# .evergreen/scripts/configure-env.sh.
+EOT
+fi
+
+# Keep the boundary out of git status via the submodule's local exclude;
+# no-op without git.
+if _git_dir=$(git -C "$ROOT/drivers-evergreen-tools" rev-parse --absolute-git-dir 2>/dev/null); then
+  mkdir -p "${_git_dir}/info"
+  grep -qxF "uv.toml" "${_git_dir}/info/exclude" 2>/dev/null ||
+    printf "uv.toml\n" >> "${_git_dir}/info/exclude"
+fi
+
 # Ensure dependencies are installed.
 bash $HERE/install-dependencies.sh
 

.evergreen/scripts/setup_tests.py

@@ -18,6 +18,7 @@
     ROOT,
     TEST_SUITE_MAP,
     Distro,
+    check_drivers_tools,
     get_test_options,
     read_env,
     run_command,
@@ -327,8 +328,7 @@ def handle_test_env() -> None:
             MULTI_MONGOS_LB_URI += "&tls=true"
         write_env("SINGLE_MONGOS_LB_URI", SINGLE_MONGOS_LB_URI)
         write_env("MULTI_MONGOS_LB_URI", MULTI_MONGOS_LB_URI)
-        if not DRIVERS_TOOLS:
-            raise RuntimeError("Missing DRIVERS_TOOLS")
+        check_drivers_tools()
         cmd = f'bash "{DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh" start'
         run_command(cmd)
 
@@ -376,8 +376,7 @@ def handle_test_env() -> None:
         run_command(cmd, cwd=DRIVERS_TOOLS)
 
     if SSL != "nossl":
-        if not DRIVERS_TOOLS:
-            raise RuntimeError("Missing DRIVERS_TOOLS")
+        check_drivers_tools()
         write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem")
         write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem")
 
@@ -429,8 +428,7 @@ def handle_test_env() -> None:
         # PATH is updated by configure-env.sh for access to mongocryptd.
 
     if test_name == "encryption":
-        if not DRIVERS_TOOLS:
-            raise RuntimeError("Missing DRIVERS_TOOLS")
+        check_drivers_tools()
         csfle_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/csfle")
         # Opt in to corporate Azure credentials (DRIVERS-3392)
         os.environ["FLE_AZURE_USE_CORPORATE"] = "YES"

.evergreen/scripts/stop-server.sh

@@ -4,11 +4,15 @@ set -eu
 
 HERE=$(dirname ${BASH_SOURCE:-$0})
 HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )"
+ROOT=$(dirname "$(dirname $HERE)")
 
 # Try to source the env file.
 if [ -f $HERE/env.sh ]; then
   echo "Sourcing env file"
   source $HERE/env.sh
 fi
 
+# Default to the drivers-evergreen-tools submodule when unset.
+: "${DRIVERS_TOOLS:=$ROOT/drivers-evergreen-tools}"
+
 bash ${DRIVERS_TOOLS}/.evergreen/run-mongodb.sh stop

.evergreen/scripts/utils.py

@@ -12,7 +12,11 @@
 
 HERE = Path(__file__).absolute().parent
 ROOT = HERE.parent.parent
-DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/")
+# DRIVERS_TOOLS defaults to the drivers-evergreen-tools submodule; an env var
+# override wins.
+DRIVERS_TOOLS = (os.environ.get("DRIVERS_TOOLS") or str(ROOT / "drivers-evergreen-tools")).replace(
+    os.sep, "/"
+)
 TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz"  # noqa: S108
 
 LOGGER = logging.getLogger("test")
@@ -312,6 +316,18 @@ def run_command(cmd: str | list[str], **kwargs: Any) -> None:
     LOGGER.info("Running command '%s'... done.", cmd)
 
 
+def check_drivers_tools() -> None:
+    """Raise a clear error when the drivers-evergreen-tools checkout is missing."""
+    # An uninitialized submodule can exist as an empty directory, so a bare
+    # is_dir() check passes. Require a script every consumer needs instead.
+    if not (Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh").is_file():
+        raise RuntimeError(
+            "The drivers-evergreen-tools checkout is missing or empty; run `just "
+            "install` to initialize the submodule, or set DRIVERS_TOOLS to a "
+            "drivers-evergreen-tools checkout."
+        )
+
+
 def create_archive() -> str:
     run_command("git add .", cwd=ROOT)
     run_command('git commit --no-verify -m "add files"', check=False, cwd=ROOT)

.github/dependabot.yml

@@ -19,6 +19,15 @@ updates:
     schedule:
       interval: "weekly"
     open-pull-requests-limit: 0
+  # drivers-evergreen-tools submodule
+  - package-ecosystem: "gitsubmodule"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    cooldown:
+      default-days: 7
+    allow:
+      - dependency-name: "drivers-evergreen-tools"
   # Python (pip). Manages requirements/mod_wsgi.txt only; the other
   # requirements files are resolved in uv.lock.
   - package-ecosystem: "pip"

.github/workflows/test-python.yml

@@ -85,7 +85,7 @@ jobs:
           # The beta Python here relies on the action's prerelease default.
           python-version: ${{ matrix.python-version }}
       - id: setup-mongodb
-        uses: mongodb-labs/drivers-evergreen-tools@master
+        uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0
         with:
           version: "${{ matrix.mongodb-version }}"
       - name: Run tests
@@ -101,12 +101,13 @@ jobs:
       - uses: actions/checkout@v7.0.1
         with:
           persist-credentials: false
+          submodules: true
       - name: Install Python tooling
         uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218  # v3.0.5
         with:
           python-version: "3.10"
       - id: setup-mongodb
-        uses: mongodb-labs/drivers-evergreen-tools@master
+        uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0
         with:
           version: "8.0"
       - name: Setup tests
@@ -126,14 +127,15 @@ jobs:
       - uses: actions/checkout@v7.0.1
         with:
           persist-credentials: false
+          submodules: true
       - name: Install Python tooling
         uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218  # v3.0.5
         with:
           python-version: "3.10"
       - name: Install dependencies
         run: just install
       - id: setup-mongodb
-        uses: mongodb-labs/drivers-evergreen-tools@master
+        uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0
         with:
           version: "8.0"
       - name: Run tests
@@ -184,19 +186,20 @@ jobs:
       - uses: actions/checkout@v7.0.1
         with:
           persist-credentials: false
+          submodules: true
       - name: Install Python tooling
         uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218  # v3.0.5
         with:
           python-version: "3.10"
       - name: Install dependencies
         run: just install
       - id: setup-mongodb
-        uses: mongodb-labs/drivers-evergreen-tools@master
+        uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0
       - name: Run tests
         run:  |
           just integration-tests
       - id: setup-mongodb-ssl
-        uses: mongodb-labs/drivers-evergreen-tools@master
+        uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0
         with:
           ssl: true
       - name: Run tests
@@ -251,7 +254,7 @@ jobs:
             # Test sdist on lowest supported Python
             python-version: "3.9"
         - id: setup-mongodb
-          uses: mongodb-labs/drivers-evergreen-tools@master
+          uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0
         - name: Run connect test from sdist
           shell: bash
           run: |
@@ -275,7 +278,7 @@ jobs:
           with:
             python-version: "3.9"
         - id: setup-mongodb
-          uses: mongodb-labs/drivers-evergreen-tools@master
+          uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0
           with:
             version: "8.0"
         - name: Run tests

.github/zizmor.yml

@@ -4,4 +4,4 @@ rules:
       policies:
         actions/*: ref-pin
         mongodb-labs/drivers-github-tools/*: hash-pin
-        mongodb-labs/drivers-evergreen-tools: ref-pin
+        mongodb-labs/drivers-evergreen-tools: hash-pin

.gitignore

@@ -9,7 +9,6 @@ build/
 doc/_build/
 dist/
 tools/settings.py
-drivers-evergreen-tools
 pymongo.egg-info/
 *.so
 *.egg*
@@ -47,5 +46,10 @@ xunit-results/
 coverage.xml
 server.log
 .coverage
+test-results.json
 access_log
 error_log
+
+# AI-assistant review workflow logs (kept untracked)
+/REVIEW_STATE.md
+/REVIEW.md

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "drivers-evergreen-tools"]
+	path = drivers-evergreen-tools
+	url = https://github.com/mongodb-labs/drivers-evergreen-tools.git