diff --git a/.evergreen/scripts/cleanup.sh b/.evergreen/scripts/cleanup.sh index f04a936fd2..ba2cfe2b9a 100755 --- a/.evergreen/scripts/cleanup.sh +++ b/.evergreen/scripts/cleanup.sh @@ -10,5 +10,7 @@ if [ -f $HERE/env.sh ]; then source $HERE/env.sh fi -rm -rf "${DRIVERS_TOOLS}" || true +# DRIVERS_TOOLS now points inside the checkout (the drivers-evergreen-tools +# submodule); deleting it would corrupt the workdir for later tasks on the +# same host, so it is intentionally not removed here. rm -f $HERE/../../secrets-export.sh || true diff --git a/.evergreen/scripts/configure-env.sh b/.evergreen/scripts/configure-env.sh index fec3cdadae..911cbeb83c 100755 --- a/.evergreen/scripts/configure-env.sh +++ b/.evergreen/scripts/configure-env.sh @@ -12,7 +12,7 @@ else fi PROJECT_DIRECTORY="$(pwd)" -DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools" +DRIVERS_TOOLS="$PROJECT_DIRECTORY/drivers-evergreen-tools" CARGO_HOME=${CARGO_HOME:-${DRIVERS_TOOLS}/.cargo} DRIVERS_TOOLS_BINARIES="$DRIVERS_TOOLS/.bin" MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" @@ -93,12 +93,12 @@ export PROJECT="${project:-mongo-python-driver}" export PIP_QUIET=1 EOT -# Write the .env file for drivers-tools. -rm -rf $DRIVERS_TOOLS -BRANCH=master -ORG=mongodb-labs -git clone --branch $BRANCH https://github.com/$ORG/drivers-evergreen-tools.git $DRIVERS_TOOLS +# Initialize the drivers-evergreen-tools submodule (Evergreen's +# git.get_project does not init submodules). Checks out the gitlink recorded +# in this checkout. +git -C "$PROJECT_DIRECTORY" submodule update --init --recursive +# Write the .env file for drivers-tools. cat < ${DRIVERS_TOOLS}/.env SKIP_LEGACY_SHELL=1 DRIVERS_TOOLS="$DRIVERS_TOOLS" diff --git a/.evergreen/scripts/create-spec-pr.sh b/.evergreen/scripts/create-spec-pr.sh index 1de1a7cae1..95013b591e 100755 --- a/.evergreen/scripts/create-spec-pr.sh +++ b/.evergreen/scripts/create-spec-pr.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -tools="$(realpath -s "../drivers-tools")" +tools="$(realpath -s "./drivers-evergreen-tools")" pushd $tools/.evergreen/github_app || exit owner="mongodb" @@ -46,5 +46,3 @@ resp=$(curl -L \ --url https://api.github.com/repos/$owner/$repo/pulls) echo $resp | jq '.html_url' echo "Creating the PR... done." - -rm -rf $tools diff --git a/.evergreen/scripts/install-dependencies.sh b/.evergreen/scripts/install-dependencies.sh index 36e7866735..a0f42b375b 100755 --- a/.evergreen/scripts/install-dependencies.sh +++ b/.evergreen/scripts/install-dependencies.sh @@ -60,8 +60,11 @@ fi # Set up uv if needed. if [ "$_need_setup" = "1" ]; then - # ensure-uv.sh (drivers-evergreen-tools) finds or installs uv and scopes its env. - if [ -n "${DRIVERS_TOOLS:-}" ] && [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then + # ensure-uv.sh (drivers-evergreen-tools) finds or installs uv and scopes its + # env. Default DRIVERS_TOOLS to the drivers-evergreen-tools submodule so it + # is used whenever present; an env var override wins. + : "${DRIVERS_TOOLS:=$(dirname "$(dirname "$HERE")")/drivers-evergreen-tools}" + if [ -f "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ]; then . "$DRIVERS_TOOLS/.evergreen/ensure-uv.sh" ensure_uv || exit 1 fi diff --git a/.evergreen/scripts/run-getdata.sh b/.evergreen/scripts/run-getdata.sh index 9435a5fcc3..416744ac03 100755 --- a/.evergreen/scripts/run-getdata.sh +++ b/.evergreen/scripts/run-getdata.sh @@ -2,6 +2,13 @@ # Get the debug data for an evergreen task. set -eu +HERE=$(dirname ${BASH_SOURCE:-$0}) +HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" +ROOT=$(dirname "$(dirname $HERE)") + +# Default to the drivers-evergreen-tools submodule when unset. +: "${DRIVERS_TOOLS:=$ROOT/drivers-evergreen-tools}" + . ${DRIVERS_TOOLS}/.evergreen/get-distro.sh || true get_distro || true echo $DISTRO diff --git a/.evergreen/scripts/run_server.py b/.evergreen/scripts/run_server.py index f81964f367..ef452f1060 100644 --- a/.evergreen/scripts/run_server.py +++ b/.evergreen/scripts/run_server.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -from utils import DRIVERS_TOOLS, ROOT, get_test_options, run_command +from utils import DRIVERS_TOOLS, ROOT, check_drivers_tools, get_test_options, run_command def set_env(name: str, value: Any = "1") -> None: @@ -14,11 +14,9 @@ def set_env(name: str, value: Any = "1") -> None: def start_server(): - run_mongodb_script = ( - Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh" if DRIVERS_TOOLS else None - ) + run_mongodb_script = Path(DRIVERS_TOOLS) / ".evergreen" / "run-mongodb.sh" want_help = bool({"-h", "--help"} & set(sys.argv[1:])) - if want_help and run_mongodb_script and run_mongodb_script.is_file(): + if want_help and run_mongodb_script.is_file(): # Forward straight to run-mongodb.sh's own help, without run_command's # "Running command..." logging noise. subprocess.run( # noqa: S603 @@ -28,13 +26,11 @@ def start_server(): ) return - # DRIVERS_TOOLS is only needed to actually start a server. When it's unset and - # -h/--help was requested, fall through to get_test_options' own argparse help below. - if not want_help and not DRIVERS_TOOLS: - raise ValueError( - "DRIVERS_TOOLS is not set; run `just run-server` from an Evergreen task " - "or set DRIVERS_TOOLS to a drivers-evergreen-tools checkout." - ) + # DRIVERS_TOOLS is only needed to actually start a server. When the + # submodule is missing and -h/--help was requested, fall through to + # get_test_options' own argparse help below. + if not want_help: + check_drivers_tools() opts, extra_opts = get_test_options( "Run a MongoDB server. All given flags will be passed to run-mongodb.sh in DRIVERS_TOOLS.", diff --git a/.evergreen/scripts/setup-dev-env.sh b/.evergreen/scripts/setup-dev-env.sh index 508eb099bc..12534f50a7 100755 --- a/.evergreen/scripts/setup-dev-env.sh +++ b/.evergreen/scripts/setup-dev-env.sh @@ -53,6 +53,16 @@ if [ "${CI:-}" != "true" ] && [ "${GITHUB_ACTIONS:-}" != "true" ]; then printf 'export PATH="%s:$PATH"\n' "$PYMONGO_BIN_DIR_POSIX" >> "$_rc" fi +# Initialize the drivers-evergreen-tools submodule before +# install-dependencies.sh (which sources ensure-uv.sh from the tools +# checkout). Evergreen's git.get_project does not init submodules, so this +# must happen in our scripts. Tolerate non-git contexts (containers) with a +# warning rather than a hard failure. +if ! git -C "$ROOT" submodule update --init --recursive; then + echo "WARNING: could not initialize the drivers-evergreen-tools submodule;" \ + "set DRIVERS_TOOLS to a drivers-evergreen-tools checkout instead." +fi + # Ensure dependencies are installed. bash $HERE/install-dependencies.sh diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 5a3f441f58..4ec219795d 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -17,6 +17,7 @@ ROOT, TEST_SUITE_MAP, Distro, + check_drivers_tools, get_test_options, read_env, run_command, @@ -327,8 +328,7 @@ def handle_test_env() -> None: MULTI_MONGOS_LB_URI += "&tls=true" write_env("SINGLE_MONGOS_LB_URI", SINGLE_MONGOS_LB_URI) write_env("MULTI_MONGOS_LB_URI", MULTI_MONGOS_LB_URI) - if not DRIVERS_TOOLS: - raise RuntimeError("Missing DRIVERS_TOOLS") + check_drivers_tools() cmd = f'bash "{DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh" start' run_command(cmd) @@ -374,8 +374,7 @@ def handle_test_env() -> None: run_command(cmd, cwd=DRIVERS_TOOLS) if SSL != "nossl": - if not DRIVERS_TOOLS: - raise RuntimeError("Missing DRIVERS_TOOLS") + check_drivers_tools() write_env("CLIENT_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/client.pem") write_env("CA_PEM", f"{DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem") @@ -427,8 +426,7 @@ def handle_test_env() -> None: # PATH is updated by configure-env.sh for access to mongocryptd. if test_name == "encryption": - if not DRIVERS_TOOLS: - raise RuntimeError("Missing DRIVERS_TOOLS") + check_drivers_tools() csfle_dir = Path(f"{DRIVERS_TOOLS}/.evergreen/csfle") # Opt in to corporate Azure credentials (DRIVERS-3392) os.environ["FLE_AZURE_USE_CORPORATE"] = "YES" diff --git a/.evergreen/scripts/stop-server.sh b/.evergreen/scripts/stop-server.sh index 045a655cbd..a254a2715a 100755 --- a/.evergreen/scripts/stop-server.sh +++ b/.evergreen/scripts/stop-server.sh @@ -4,6 +4,7 @@ set -eu HERE=$(dirname ${BASH_SOURCE:-$0}) HERE="$( cd -- "$HERE" > /dev/null 2>&1 && pwd )" +ROOT=$(dirname "$(dirname $HERE)") # Try to source the env file. if [ -f $HERE/env.sh ]; then @@ -11,4 +12,7 @@ if [ -f $HERE/env.sh ]; then source $HERE/env.sh fi +# Default to the drivers-evergreen-tools submodule when unset. +: "${DRIVERS_TOOLS:=$ROOT/drivers-evergreen-tools}" + bash ${DRIVERS_TOOLS}/.evergreen/run-mongodb.sh stop diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index a7decdfaa8..b5b51a9376 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -12,7 +12,11 @@ HERE = Path(__file__).absolute().parent ROOT = HERE.parent.parent -DRIVERS_TOOLS = os.environ.get("DRIVERS_TOOLS", "").replace(os.sep, "/") +# DRIVERS_TOOLS defaults to the drivers-evergreen-tools submodule; an env var +# override wins. +DRIVERS_TOOLS = (os.environ.get("DRIVERS_TOOLS") or str(ROOT / "drivers-evergreen-tools")).replace( + os.sep, "/" +) TMP_DRIVER_FILE = "/tmp/mongo-python-driver.tgz" # noqa: S108 LOGGER = logging.getLogger("test") @@ -309,6 +313,16 @@ def run_command(cmd: str | list[str], **kwargs: Any) -> None: LOGGER.info("Running command '%s'... done.", cmd) +def check_drivers_tools() -> None: + """Raise a clear error when the drivers-evergreen-tools checkout is missing.""" + if not Path(DRIVERS_TOOLS).is_dir(): + raise RuntimeError( + "The drivers-evergreen-tools checkout is missing; run `just install` to " + "initialize the submodule, or set DRIVERS_TOOLS to a drivers-evergreen-tools " + "checkout." + ) + + def create_archive() -> str: run_command("git add .", cwd=ROOT) run_command('git commit --no-verify -m "add files"', check=False, cwd=ROOT) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 72d1e1e084..de7dd1125f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,3 +19,12 @@ updates: schedule: interval: "weekly" open-pull-requests-limit: 0 + # drivers-evergreen-tools submodule + - package-ecosystem: "gitsubmodule" + directory: "/" + schedule: + interval: "weekly" + cooldown: + default-days: 7 + allow: + - dependency-name: "drivers-evergreen-tools" diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 5e0d89f7a4..f2e40b12aa 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -85,7 +85,7 @@ jobs: # The beta Python here relies on the action's prerelease default. python-version: ${{ matrix.python-version }} - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "${{ matrix.mongodb-version }}" - name: Run tests @@ -101,12 +101,13 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false + submodules: true - name: Install Python tooling uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 with: python-version: "3.10" - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "8.0" - name: Setup tests @@ -126,6 +127,7 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false + submodules: true - name: Install Python tooling uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 with: @@ -133,7 +135,7 @@ jobs: - name: Install dependencies run: just install - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "8.0" - name: Run tests @@ -184,6 +186,7 @@ jobs: - uses: actions/checkout@v7.0.1 with: persist-credentials: false + submodules: true - name: Install Python tooling uses: mongodb-labs/drivers-github-tools/python/setup@f137fdd28483af14ebf466ebc5aa789fbf867218 # v3.0.5 with: @@ -191,12 +194,12 @@ jobs: - name: Install dependencies run: just install - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 - name: Run tests run: | just integration-tests - id: setup-mongodb-ssl - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: ssl: true - name: Run tests @@ -251,7 +254,7 @@ jobs: # Test sdist on lowest supported Python python-version: "3.9" - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 - name: Run connect test from sdist shell: bash run: | @@ -275,7 +278,7 @@ jobs: with: python-version: "3.9" - id: setup-mongodb - uses: mongodb-labs/drivers-evergreen-tools@master + uses: mongodb-labs/drivers-evergreen-tools@ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 # v1.1.0 with: version: "8.0" - name: Run tests diff --git a/.github/zizmor.yml b/.github/zizmor.yml index af9f715dd4..2a69f0dd1a 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -4,4 +4,4 @@ rules: policies: actions/*: ref-pin mongodb-labs/drivers-github-tools/*: hash-pin - mongodb-labs/drivers-evergreen-tools: ref-pin + mongodb-labs/drivers-evergreen-tools: hash-pin diff --git a/.gitignore b/.gitignore index 8546aeabb7..8fc21c5301 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ build/ doc/_build/ dist/ tools/settings.py -drivers-evergreen-tools pymongo.egg-info/ *.so *.egg* diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..d89986aa20 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "drivers-evergreen-tools"] + path = drivers-evergreen-tools + url = https://github.com/mongodb-labs/drivers-evergreen-tools.git diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 20b7210736..0fb1def488 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -205,7 +205,7 @@ the pages will re-render and the browser will automatically refresh. project `uv` command (e.g. `just test`) runs with a different `uv` version, `uv` fails fast and tells you how to update. - Ensure you have started the appropriate Mongo Server(s). You can run `just run-server` with optional args to set up the server. All given options will be passed to - [`run-mongodb.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-mongodb.sh). Run `$DRIVERS_TOOLS/.evergreen/run-mongodb.sh start -h` + [`run-mongodb.sh`](https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-mongodb.sh). Run `drivers-evergreen-tools/.evergreen/run-mongodb.sh start -h` for a full list of options. - Run `just test` or `pytest` to run all of the tests. - Append `test/.py::::` to run @@ -220,10 +220,10 @@ the pages will re-render and the browser will automatically refresh. ### Prerequisites -- Clone `drivers-evergreen-tools`: - `git clone git@github.com:mongodb-labs/drivers-evergreen-tools.git`. -- Run `export DRIVERS_TOOLS=$PWD/drivers-evergreen-tools`. This can be put into a `.bashrc` file - for convenience. +- The `drivers-evergreen-tools` submodule (see + [The drivers-evergreen-tools submodule](#the-drivers-evergreen-tools-submodule)). + `just install` initializes it; no manual clone or `export DRIVERS_TOOLS` is needed. + `DRIVERS_TOOLS` remains an optional override for an existing local checkout. - Some tests require access to [Drivers test secrets](https://github.com/mongodb-labs/drivers-evergreen-tools/tree/master/.evergreen/secrets_handling#secrets-handling). ### Usage @@ -356,7 +356,7 @@ You will need to set up access to the `drivers-test-secrets-role`, see the [Wiki ### OCSP tests - Export the orchestration file, e.g. `export ORCHESTRATION_FILE=rsa-basic-tls-ocsp-disableStapling.json`. -This corresponds to a config file in `$DRIVERS_TOOLS/.evergreen/orchestration/configs/servers`. +This corresponds to a config file in `drivers-evergreen-tools/.evergreen/orchestration/configs/servers`. MongoDB servers on MacOS and Windows do not staple OCSP responses and only support RSA. NOTE: because the mock ocsp responder MUST be started prior to the server starting, the ocsp tests start the server as part of `setup-tests`. @@ -372,6 +372,44 @@ If you are running one of the `no-responder` tests, omit the `run-server` step. - Set up the tests with `sync` or `async`: `just setup-tests perf sync`. - Run the tests: `just run-tests`. +## The drivers-evergreen-tools submodule + +The `drivers-evergreen-tools` repository is consumed as a git submodule at the repo root, +pinned to a specific commit. Dependabot bumps the pin weekly. + +### Daily flow + +Nothing to do: `just install` initializes the submodule, and Dependabot keeps it fresh. + +### Manually bumping the submodule + +```bash +git -C drivers-evergreen-tools fetch --tags +git -C drivers-evergreen-tools checkout vX.Y.Z +git add drivers-evergreen-tools +``` + +### Evergreen patches that need tools changes + +Point the submodule at the needed commit and commit the new gitlink in the patch branch; +`configure-env.sh` checks out the recorded SHA on Evergreen hosts (`setup-dev-env.sh` does +the same for local checkouts with `just install`). + +### Using a local checkout instead + +Set `DRIVERS_TOOLS` to the path of a local clone — the environment variable wins over the +submodule default: + +```bash +export DRIVERS_TOOLS=/path/to/drivers-evergreen-tools +``` + +Alternatively, keep a local pin from being reset by `git submodule update`: + +```bash +git config submodule.drivers-evergreen-tools.update none +``` + ## Enable Debug Logs - Use `-o log_cli_level="DEBUG" -o log_cli=1` with `just test` or `pytest` to output all debug logs to the terminal. **Warning**: This will output a huge amount of logs. diff --git a/drivers-evergreen-tools b/drivers-evergreen-tools new file mode 160000 index 0000000000..ec0b1497b3 --- /dev/null +++ b/drivers-evergreen-tools @@ -0,0 +1 @@ +Subproject commit ec0b1497b3d351acbfc4fd7579bf3de4de0a1046 diff --git a/pyproject.toml b/pyproject.toml index 2f7a68c837..cf824a72f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,11 @@ validate-bump = false [tool.hatch.build.targets.wheel] packages = ["bson","gridfs", "pymongo"] +[tool.hatch.build.targets.sdist] +# The drivers-evergreen-tools submodule must not ship in the sdist; hatchling's +# walk would otherwise pick it up when the submodule is populated locally. +exclude = ["drivers-evergreen-tools"] + [tool.hatch.metadata.hooks.requirements_txt] files = ["requirements.txt"]