PYTHON-6051 Convert drivers-evergreen-tools to a git submodule and pin the GitHub Actions reference

.evergreen/scripts/cleanup.sh

@@ -10,5 +10,7 @@ if [ -f $HERE/env.sh ]; then
   source $HERE/env.sh
 fi
 
-rm -rf "${DRIVERS_TOOLS}" || true
+# DRIVERS_TOOLS now points inside the checkout (the drivers-evergreen-tools
+# submodule); deleting it would corrupt the workdir for later tasks on the
+# same host, so it is intentionally not removed here.
 rm -f $HERE/../../secrets-export.sh || true

.evergreen/scripts/configure-env.sh

@@ -12,7 +12,7 @@ else
 fi
 
 PROJECT_DIRECTORY="$(pwd)"
-DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools"
+DRIVERS_TOOLS="$PROJECT_DIRECTORY/drivers-evergreen-tools"
 CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo}
 DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin"
 MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
@@ -93,12 +93,12 @@ export PROJECT="${project:-mongo-python-driver}"
 export PIP_QUIET=1
 EOT
 
-# Write the .env file for drivers-tools.
-rm -rf $DRIVERS_TOOLS
-BRANCH=master
-ORG=mongodb-labs
-git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS
+# Initialize the drivers-evergreen-tools submodule (Evergreen's
+# git.get_project does not init submodules). Checks out the gitlink recorded
+# in this checkout.
+git -C "$PROJECT_DIRECTORY" submodule update --init --recursive
 
+# Write the .env file for drivers-tools.
 cat <<EOT > ${DRIVERS_TOOLS}/.env
 SKIP_LEGACY_SHELL=1
 DRIVERS_TOOLS="$DRIVERS_TOOLS"

.evergreen/scripts/create-spec-pr.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-tools="$(realpath -s "../drivers-tools")"
+tools="$(realpath -s "./drivers-evergreen-tools")"
 pushd $tools/.evergreen/github_app || exit
 
 owner="mongodb"
@@ -46,5 +46,3 @@ resp=$(curl -L \
     --url https://api.github.com/repos/$owner/$repo/pulls)
 echo $resp | jq '.html_url'
 echo "Creating the PR... done."
-
-rm -rf $tools

.evergreen/scripts/install-dependencies.sh

@@ -60,8 +60,11 @@ fi
 
 # Set up uv if needed.
 if [ "$_need_setup" = "1" ]; then
-  # ensure-uv.sh (drivers-evergreen-tools) finds or installs uv and scopes its env.
-  if [ -n "${DRIVERS_TOOLS:-}" ] && [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then
+  # ensure-uv.sh (drivers-evergreen-tools) finds or installs uv and scopes its
+  # env. Default DRIVERS_TOOLS to the drivers-evergreen-tools submodule so it
+  # is used whenever present; an env var override wins.
+  : "${DRIVERS_TOOLS:=$(dirname "$(dirname "$HERE")")/drivers-evergreen-tools}"
+  if [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then
     . "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh"
     ensure_uv || exit 1
   fi

.evergreen/scripts/run-getdata.sh

@@ -2,6 +2,13 @@
 # Get the debug data for an evergreen task.
 set -eu
 
+HERE=$(dirname ${BASH_SOURCE:-$0})
+HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )"
+ROOT=$(dirname "$(dirname $HERE)")
+
+# Default to the drivers-evergreen-tools submodule when unset.
+: "${DRIVERS_TOOLS:=$ROOT/drivers-evergreen-tools}"
+
 . ${DRIVERS_TOOLS}/.evergreen/get-distro.sh || true
 get_distro || true
 echo $DISTRO

.evergreen/scripts/run_server.py

@@ -6,19 +6,17 @@
 from pathlib import Path
 from typing import Any
 
-from utils import DRIVERS_TOOLS, ROOT, get_test_options, run_command
+from utils import DRIVERS_TOOLS, ROOT, check_drivers_tools, get_test_options, run_command
 
 
 def set_env(name: str, value: Any = "1") -> None:
     os.environ[name] = str(value)
 
 
 def start_server():
-    run_mongodb_script = (
-        Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh" if DRIVERS_TOOLS else None
-    )
+    run_mongodb_script = Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh"
     want_help = bool({"-h", "--help"} & set(sys.argv[1:]))
-    if want_help and run_mongodb_script and run_mongodb_script.is_file():
+    if want_help and run_mongodb_script.is_file():
         # Forward straight to run-mongodb.sh's own help, without run_command's
         # "Running command..." logging noise.
         subprocess.run(  # noqa: S603
@@ -28,13 +26,11 @@ def start_server():
         )
         return
 
-    # DRIVERS_TOOLS is only needed to actually start a server. When it's unset and
-    # -h/--help was requested, fall through to get_test_options' own argparse help below.
-    if not want_help and not DRIVERS_TOOLS:
-        raise ValueError(
-            "DRIVERS_TOOLS is not set; run `just run-server` from an Evergreen task "
-            "or set DRIVERS_TOOLS to a drivers-evergreen-tools checkout."
-        )
+    # DRIVERS_TOOLS is only needed to actually start a server. When the
+    # submodule is missing and -h/--help was requested, fall through to
+    # get_test_options' own argparse help below.
+    if not want_help:
+        check_drivers_tools()
 
     opts, extra_opts = get_test_options(
         "Run a MongoDB server.  All given flags will be passed to run-mongodb.sh in DRIVERS_TOOLS.",

.evergreen/scripts/setup-dev-env.sh

@@ -53,6 +53,16 @@ if [ "${CI:-}" != "true" ] && [ "${GITHUB_ACTIONS:-}" != "true" ]; then
     printf 'export PATH="%s:$PATH"\n' "$PYMONGO_BIN_DIR_POSIX" >> "$_rc"
 fi
 
+# Initialize the drivers-evergreen-tools submodule before
+# install-dependencies.sh (which sources ensure-uv.sh from the tools
+# checkout). Evergreen's git.get_project does not init submodules, so this
+# must happen in our scripts. Tolerate non-git contexts (containers) with a
+# warning rather than a hard failure.
+if ! git -C "$ROOT" submodule update --init --recursive; then
+  echo "WARNING: could not initialize the drivers-evergreen-tools submodule;" \
+    "set DRIVERS_TOOLS to a drivers-evergreen-tools checkout instead."
+fi
+
 # Ensure dependencies are installed.
 bash $HERE/install-dependencies.sh
 

.evergreen/scripts/setup_tests.py

@@ -17,6 +17,7 @@
     ROOT,
     TEST_SUITE_MAP,
     Distro,
+    check_drivers_tools,
     get_test_options,
     read_env,
     run_command,
@@ -327,8 +328,7 @@ def handle_test_env() -> None:
             MULTI_MONGOS_LB_URI += "&tls=true"
         write_env("SINGLE_MONGOS_LB_URI", SINGLE_MONGOS_LB_URI)
         write_env("MULTI_MONGOS_LB_URI", MULTI_MONGOS_LB_URI)
-        if not DRIVERS_TOOLS:
-            raise RuntimeError("Missing DRIVERS_TOOLS")
+        check_drivers_tools()
         cmd = f'bash "{DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh" start'
         run_command(cmd)
 
@@ -374,8 +374,7 @@ def handle_test_env() -> None:
         run_command(cmd, cwd=DRIVERS_TOOLS)
 
     if SSL != "nossl":
-        if not DRIVERS_TOOLS:
-            raise RuntimeError("Missing DRIVERS_TOOLS")
+        check_drivers_tools()
         write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem")
         write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem")
 
@@ -427,8 +426,7 @@ def handle_test_env() -> None:
         # PATH is updated by configure-env.sh for access to mongocryptd.
 
     if test_name == "encryption":
-        if not DRIVERS_TOOLS:
-            raise RuntimeError("Missing DRIVERS_TOOLS")
+        check_drivers_tools()
         csfle_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/csfle")
         # Opt in to corporate Azure credentials (DRIVERS-3392)
         os.environ["FLE_AZURE_USE_CORPORATE"] = "YES"

.evergreen/scripts/stop-server.sh

@@ -4,11 +4,15 @@ set -eu
 
 HERE=$(dirname ${BASH_SOURCE:-$0})
 HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )"
+ROOT=$(dirname "$(dirname $HERE)")
 
 # Try to source the env file.
 if [ -f $HERE/env.sh ]; then
   echo "Sourcing env file"
   source $HERE/env.sh
 fi
 
+# Default to the drivers-evergreen-tools submodule when unset.
+: "${DRIVERS_TOOLS:=$ROOT/drivers-evergreen-tools}"
+
 bash ${DRIVERS_TOOLS}/.evergreen/run-mongodb.sh stop

.evergreen/scripts/utils.py

@@ -12,7 +12,11 @@
 
 HERE = Path(__file__).absolute().parent
 ROOT = HERE.parent.parent
-DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/")
+# DRIVERS_TOOLS defaults to the drivers-evergreen-tools submodule; an env var
+# override wins.
+DRIVERS_TOOLS = (os.environ.get("DRIVERS_TOOLS") or str(ROOT / "drivers-evergreen-tools")).replace(
+    os.sep, "/"
+)
 TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz"  # noqa: S108
 
 LOGGER = logging.getLogger("test")
@@ -309,6 +313,16 @@ def run_command(cmd: str | list[str], **kwargs: Any) -> None:
     LOGGER.info("Running command '%s'... done.", cmd)
 
 
+def check_drivers_tools() -> None:
+    """Raise a clear error when the drivers-evergreen-tools checkout is missing."""
+    if not Path(DRIVERS_TOOLS).is_dir():
+        raise RuntimeError(
+            "The drivers-evergreen-tools checkout is missing; run `just install` to "
+            "initialize the submodule, or set DRIVERS_TOOLS to a drivers-evergreen-tools "
+            "checkout."
+        )
+
+
 def create_archive() -> str:
     run_command("git add .", cwd=ROOT)
     run_command('git commit --no-verify -m "add files"', check=False, cwd=ROOT)

.github/dependabot.yml

@@ -19,3 +19,12 @@ updates:
     schedule:
       interval: "weekly"
     open-pull-requests-limit: 0
+  # drivers-evergreen-tools submodule
+  - package-ecosystem: "gitsubmodule"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    cooldown:
+      default-days: 7
+    allow:
+      - dependency-name: "drivers-evergreen-tools"

.github/workflows/test-python.yml

@@ -85,7 +85,7 @@ jobs:
           # The beta Python here relies on the action's prerelease default.
           python-version: ${{ matrix.python-version }}
       - id: setup-mongodb
-        uses: mongodb-labs/drivers-evergreen-tools@master
+        uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0
         with:
           version: "${{ matrix.mongodb-version }}"
       - name: Run tests
@@ -101,12 +101,13 @@ jobs:
       - uses: actions/checkout@v7.0.1
         with:
           persist-credentials: false
+          submodules: true
       - name: Install Python tooling
         uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218  # v3.0.5
         with:
           python-version: "3.10"
       - id: setup-mongodb
-        uses: mongodb-labs/drivers-evergreen-tools@master
+        uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0
         with:
           version: "8.0"
       - name: Setup tests
@@ -126,14 +127,15 @@ jobs:
       - uses: actions/checkout@v7.0.1
         with:
           persist-credentials: false
+          submodules: true
       - name: Install Python tooling
         uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218  # v3.0.5
         with:
           python-version: "3.10"
       - name: Install dependencies
         run: just install
       - id: setup-mongodb
-        uses: mongodb-labs/drivers-evergreen-tools@master
+        uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0
         with:
           version: "8.0"
       - name: Run tests
@@ -184,19 +186,20 @@ jobs:
       - uses: actions/checkout@v7.0.1
         with:
           persist-credentials: false
+          submodules: true
       - name: Install Python tooling
         uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218  # v3.0.5
         with:
           python-version: "3.10"
       - name: Install dependencies
         run: just install
       - id: setup-mongodb
-        uses: mongodb-labs/drivers-evergreen-tools@master
+        uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0
       - name: Run tests
         run:  |
           just integration-tests
       - id: setup-mongodb-ssl
-        uses: mongodb-labs/drivers-evergreen-tools@master
+        uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0
         with:
           ssl: true
       - name: Run tests
@@ -251,7 +254,7 @@ jobs:
             # Test sdist on lowest supported Python
             python-version: "3.9"
         - id: setup-mongodb
-          uses: mongodb-labs/drivers-evergreen-tools@master
+          uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0
         - name: Run connect test from sdist
           shell: bash
           run: |
@@ -275,7 +278,7 @@ jobs:
           with:
             python-version: "3.9"
         - id: setup-mongodb
-          uses: mongodb-labs/drivers-evergreen-tools@master
+          uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0
           with:
             version: "8.0"
         - name: Run tests

.github/zizmor.yml

@@ -4,4 +4,4 @@ rules:
       policies:
         actions/*: ref-pin
         mongodb-labs/drivers-github-tools/*: hash-pin
-        mongodb-labs/drivers-evergreen-tools: ref-pin
+        mongodb-labs/drivers-evergreen-tools: hash-pin

.gitignore

@@ -9,7 +9,6 @@ build/
 doc/_build/
 dist/
 tools/settings.py
-drivers-evergreen-tools
 pymongo.egg-info/
 *.so
 *.egg*

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "drivers-evergreen-tools"]
+	path = drivers-evergreen-tools
+	url = https://github.com/mongodb-labs/drivers-evergreen-tools.git