diff --git a/.github/workflows/integration-cassandra.yml b/.github/workflows/integration-cassandra.yml index ad0c68068..10daa86fc 100644 --- a/.github/workflows/integration-cassandra.yml +++ b/.github/workflows/integration-cassandra.yml @@ -123,6 +123,31 @@ jobs: env: EXTENDDB_TEST_CASSANDRA: required + # One release build shared by the jobs that test it; see integration.yml. + build-cassandra: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-on-failure: true + # Only pushes to main save, so other runs restore main's cache. + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Build release (cassandra backend) + run: cargo build --release --no-default-features --features cassandra + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: extenddb-cassandra + path: target/release/extenddb + if-no-files-found: error + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + cassandra-pytest: runs-on: extenddb_ubuntu-2404_4-core # With the PartiQL wire tests the job took up to 42 min 10 s on #411's CI, @@ -130,6 +155,16 @@ jobs: # cold build cache should take about 44 min; 60 leaves room for a slow # runner. timeout-minutes: 60 + needs: build-cassandra + # Five shards by test file, each with its own Cassandra node; + # cassandra-integration checks that together they collected every test + # of the whole suite. + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5] + env: + EXTENDDB_TEST_SHARD: ${{ matrix.shard }}/5 services: cassandra: image: cassandra:4.1 @@ -147,14 +182,6 @@ jobs: --health-start-period 60s steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master - with: - toolchain: stable - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - cache-on-failure: true - # Only pushes to main save, so other runs restore main's cache. - save-if: ${{ github.ref == 'refs/heads/main' }} - name: Configure Cassandra authentication run: | docker exec ${{ job.services.cassandra.id }} sed -i \ @@ -177,8 +204,12 @@ jobs: python-version: "3.11" - name: Install Python dependencies run: pip install -r requirements.txt - - name: Build release (cassandra backend) - run: cargo build --release --no-default-features --features cassandra + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-cassandra + path: target/release + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Install Terraform (pinned version, checksum asserted) run: echo "TERRAFORM=$(tests/terraform/install-terraform.sh "$RUNNER_TEMP/terraform")" >> "$GITHUB_ENV" - name: Run Cassandra pytest suite @@ -188,9 +219,21 @@ jobs: EXTENDDB_TEST_BACKUP_STORE: none run: devtools/run-cassandra-tests -- --pytest --comprehensive --terraform --parallel + # cassandra-integration checks these lists. + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: shards-cassandra-pytest-${{ matrix.shard }} + path: discussions/shards/ + if-no-files-found: warn + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + cassandra-rust: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 45 + needs: build-cassandra services: cassandra: image: cassandra:4.1 @@ -238,8 +281,13 @@ jobs: python-version: "3.11" - name: Install Python dependencies run: pip install -r requirements.txt - - name: Build release (cassandra backend) - run: cargo build --release --no-default-features --features cassandra + # After the cache restore, so a cached target/ cannot replace it. + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-cassandra + path: target/release + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Run Cassandra rust integration suite env: # run-cassandra-tests initializes the server without [backup.store]: @@ -268,6 +316,11 @@ jobs: cassandra-catalog-checks: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 60 + needs: build-cassandra + env: + # The cargo features of the v0.1.14 build, read by its cache key + # and its build command. + V0114_FEATURES: cassandra services: cassandra: image: cassandra:4.1 @@ -285,10 +338,26 @@ jobs: --health-start-period 60s steps: - uses: actions/checkout@v6 + # A tag does not change, so the v0.1.14 binary is cached by the tag, + # the cargo features and the runner's OS and architecture. Every run + # restores it, and only a push to main saves it, as for the Rust + # caches, so no pull request run writes the entry. On a hit the job + # skips that build and the Rust toolchain and cache steps, which only + # that build uses. + - name: Restore the v0.1.14 release (cassandra backend) + id: cache-v0-1-14 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ runner.temp }}/target-v0.1.14/release/extenddb + key: extenddb-v0.1.14-no-default-features-${{ env.V0114_FEATURES }}-${{ runner.os }}-${{ runner.arch }} - uses: dtolnay/rust-toolchain@stable + if: steps.cache-v0-1-14.outputs.cache-hit != 'true' - uses: Swatinem/rust-cache@v2 + if: steps.cache-v0-1-14.outputs.cache-hit != 'true' with: cache-on-failure: true + # Only pushes to main save, so other runs restore main's cache. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Configure Cassandra authentication run: | docker exec ${{ job.services.cassandra.id }} sed -i \ @@ -311,8 +380,13 @@ jobs: python-version: "3.11" - name: Install Python dependencies run: pip install -r requirements.txt - - name: Build release (cassandra backend) - run: cargo build --release --no-default-features --features cassandra + # After the cache restore, so a cached target/ cannot replace it. + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-cassandra + path: target/release + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb # A fresh install's schema against the committed file. On a difference # the job fails with the diff and uploads the regenerated file to commit. @@ -345,15 +419,25 @@ jobs: ${{ job.services.cassandra.id }} 127.0.0.1:9042 18454 # No released image carries the Cassandra backend, so the job builds - # the last 0.1.x release from its tag. + # the last 0.1.x release from its tag, on a cache miss only. - name: Build the v0.1.14 release (cassandra backend) + if: steps.cache-v0-1-14.outputs.cache-hit != 'true' run: | git fetch --depth 1 origin tag v0.1.14 git worktree add "$RUNNER_TEMP/extenddb-v0.1.14" v0.1.14 - cargo build --release --no-default-features --features cassandra \ + cargo build --release --no-default-features --features "$V0114_FEATURES" \ --manifest-path "$RUNNER_TEMP/extenddb-v0.1.14/Cargo.toml" \ --target-dir "$RUNNER_TEMP/target-v0.1.14" + - name: Save the v0.1.14 release (cassandra backend) + if: >- + github.event_name == 'push' && github.ref == 'refs/heads/main' && + steps.cache-v0-1-14.outputs.cache-hit != 'true' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ runner.temp }}/target-v0.1.14/release/extenddb + key: extenddb-v0.1.14-no-default-features-${{ env.V0114_FEATURES }}-${{ runner.os }}-${{ runner.arch }} + - name: Check the last 0.1.x release against migrated catalogs run: >- devtools/check-cassandra-older-binary ./target/release/extenddb @@ -372,6 +456,8 @@ jobs: - uses: Swatinem/rust-cache@v2 with: cache-on-failure: true + # Only pushes to main save, so other runs restore main's cache. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Start a two-node cluster run: | docker network create cassandra-two-nodes @@ -421,6 +507,7 @@ jobs: cassandra-authorizer: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 45 + needs: build-cassandra services: cassandra: image: cassandra:4.1 @@ -438,10 +525,6 @@ jobs: --health-start-period 60s steps: - uses: actions/checkout@v6 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - name: Configure Cassandra authentication and CassandraAuthorizer run: | docker exec ${{ job.services.cassandra.id }} sed -i \ @@ -465,8 +548,12 @@ jobs: python-version: "3.11" - name: Install Python dependencies run: pip install -r requirements.txt - - name: Build release (cassandra backend) - run: cargo build --release --no-default-features --features cassandra + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-cassandra + path: target/release + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Check init, migrate, serve and CreateAccount under CassandraAuthorizer run: >- devtools/check-cassandra-authorizer ./target/release/extenddb @@ -495,3 +582,12 @@ jobs: [ "${{ needs.cassandra-authorizer.result }}" != "success" ]; then exit 1 fi + # Fails unless the shards together collected every test of the whole + # suite exactly once, so a bad split cannot pass. + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: shards-cassandra-pytest-* + path: shards/cassandra-pytest + - name: Check the shards ran every test + run: python3 devtools/test-shards check shards/cassandra-pytest diff --git a/.github/workflows/integration-mongodb.yml b/.github/workflows/integration-mongodb.yml index 15f87245b..83708ef5f 100644 --- a/.github/workflows/integration-mongodb.yml +++ b/.github/workflows/integration-mongodb.yml @@ -29,10 +29,8 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - mongodb-pytest: - # Dedicated ExtendDB runner group (Ubuntu 24.04, 4 vCPU / 16 GB, capped at - # 20 concurrent). ubuntu-latest is an Amazon-wide shared pool where our jobs - # waited 10-90 minutes to start (2026-09-29/30); see integration.yml. + # One release build shared by the jobs that test it; see integration.yml. + build-mongodb: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 steps: @@ -45,13 +43,45 @@ jobs: cache-on-failure: true # Only pushes to main save, so other runs restore main's cache. save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Build release (mongodb backend) + run: cargo build --release --no-default-features --features mongodb-test-hooks + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: extenddb-mongodb + path: target/release/extenddb + if-no-files-found: error + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + + mongodb-pytest: + # Dedicated ExtendDB runner group (Ubuntu 24.04, 4 vCPU / 16 GB, capped at + # 20 concurrent). ubuntu-latest is an Amazon-wide shared pool where our jobs + # waited 10-90 minutes to start (2026-09-29/30); see integration.yml. + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + needs: build-mongodb + # Three shards by test file; mongodb-integration checks that together + # they collected every test of the whole suite. + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + env: + EXTENDDB_TEST_SHARD: ${{ matrix.shard }}/3 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" - name: Install Python dependencies run: pip install -r requirements.txt - - name: Build release (mongodb backend) - run: cargo build --release --no-default-features --features mongodb-test-hooks + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-mongodb + path: target/release + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Install Terraform (pinned version, checksum asserted) run: echo "TERRAFORM=$(tests/terraform/install-terraform.sh "$RUNNER_TEMP/terraform")" >> "$GITHUB_ENV" - name: Run MongoDB pytest suite @@ -63,6 +93,17 @@ jobs: EXTENDDB_TEST_BACKUP_STORE: filesystem run: devtools/run-mongodb-tests -- --pytest --comprehensive --terraform --parallel + # mongodb-integration checks these lists. + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: shards-mongodb-pytest-${{ matrix.shard }} + path: discussions/shards/ + if-no-files-found: warn + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + # SnapshotEngine consistency check against a disposable replica set. The # tests build their own per-test scratch databases (snap_cat and # _data) and drop them; they run in this job because it is the one that @@ -121,6 +162,7 @@ jobs: # The job took up to 38 min 42 s on main's CI in November 2026, near the # old 40-minute bound; 55 leaves room for a slow runner. timeout-minutes: 55 + needs: build-mongodb steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master @@ -136,8 +178,13 @@ jobs: python-version: "3.11" - name: Install Python dependencies run: pip install -r requirements.txt - - name: Build release (mongodb backend) - run: cargo build --release --no-default-features --features mongodb-test-hooks + # After the cache restore, so a cached target/ cannot replace it. + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-mongodb + path: target/release + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Run MongoDB rust integration suite env: # run-mongodb-tests gives the server a filesystem [backup.store]: @@ -155,23 +202,20 @@ jobs: mongodb-backup-store: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 + needs: build-mongodb steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master - with: - toolchain: stable - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - cache-on-failure: true - # Only pushes to main save, so other runs restore main's cache. - save-if: ${{ github.ref == 'refs/heads/main' }} - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" - name: Install Python dependencies run: pip install -r requirements.txt - - name: Build release (mongodb backend) - run: cargo build --release --no-default-features --features mongodb-test-hooks + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-mongodb + path: target/release + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Run the backup selection against a filesystem store env: EXTENDDB_TEST_BACKUP_STORE: filesystem @@ -201,19 +245,20 @@ jobs: mongodb-catalog: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 + needs: build-mongodb steps: - uses: actions/checkout@v6 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - uses: actions/setup-python@v5 with: python-version: "3.11" - name: Install Python dependencies run: pip install -r requirements.txt - - name: Build release (mongodb backend) - run: cargo build --release --no-default-features --features mongodb-test-hooks + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-mongodb + path: target/release + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Check the current schema run: devtools/check-mongodb-schema ./target/release/extenddb schema-mongodb.txt 27071 - name: Upload the regenerated schema @@ -266,3 +311,12 @@ jobs: [ "${{ needs.mongodb-snapshot.result }}" != "success" ]; then exit 1 fi + # Fails unless the shards together collected every test of the whole + # suite exactly once, so a bad split cannot pass. + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: shards-mongodb-pytest-* + path: shards/mongodb-pytest + - name: Check the shards ran every test + run: python3 devtools/test-shards check shards/mongodb-pytest diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bf1fa9789..071500c00 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,6 +20,65 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + # One release build per feature set. The jobs below download the binary + # instead of each compiling the same one, so the cargo flags and features of + # a test job live in the build job it needs. + build-postgres: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-on-failure: true + # Only pushes to main save, so other runs restore main's cache. + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Build release + run: cargo build --release + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: extenddb-postgres + path: target/release/extenddb + if-no-files-found: error + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + + build-sqlite: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-on-failure: true + # Only pushes to main save, so other runs restore main's cache. + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Build release (SQLite backend) + run: cargo build --release -p extenddb --no-default-features --features sqlite + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: extenddb-sqlite + path: target/release/extenddb + if-no-files-found: error + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + run-integration: # Dedicated ExtendDB runner group (4 vCPU / 16 GB, capped at 20 concurrent), # pinned to Ubuntu 24.04. ubuntu-latest is an Amazon-wide pool of 1,000 @@ -33,6 +92,15 @@ jobs: # PartiQL wire tests it took 48 min 4 s on #411's CI, before #740. A run # with both should take about 50 min; 70 leaves room for a slow runner. timeout-minutes: 70 + needs: build-postgres + # Three shards by test file (devtools/test-shards); the integration job + # checks that together they collected every test of the whole suite. + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + env: + EXTENDDB_TEST_SHARD: ${{ matrix.shard }}/3 services: postgres: # pgvector's own image, which is postgres:16 plus the extension. The @@ -53,18 +121,13 @@ jobs: steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - toolchain: stable + name: extenddb-postgres + path: target/release - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - cache-on-failure: true - # Only pushes to main save, so other runs restore main's cache. - save-if: ${{ github.ref == 'refs/heads/main' }} - - - name: Build release - run: cargo build --release + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Initialize ExtendDB id: init @@ -131,6 +194,9 @@ jobs: name: schema-postgres path: schema-postgres.txt if-no-files-found: ignore + # Every leg runs the schema check, so a difference fails each + # leg and each one uploads this name. + overwrite: true # PgBouncer 1.26.1 in transaction mode in front of the job's server, for # the lock script's pass through a connection pooler. It logs every role @@ -215,6 +281,17 @@ jobs: if: failure() run: sudo journalctl -t extenddb --no-pager -n 500 || true + # The integration job checks these lists. + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: shards-run-integration-${{ matrix.shard }} + path: discussions/shards/ + if-no-files-found: warn + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + run-integration-sqlite: runs-on: extenddb_ubuntu-2404_4-core # With the backup store and job suites it ends with (4 min 16 s of it), the @@ -222,21 +299,23 @@ jobs: # PartiQL wire tests it took 43 min 27 s on #411's CI, before #740. 60 # leaves room for both and a slow runner. timeout-minutes: 60 + needs: build-sqlite + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + env: + EXTENDDB_TEST_SHARD: ${{ matrix.shard }}/3 steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - toolchain: stable + name: extenddb-sqlite + path: target/release - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - cache-on-failure: true - # Only pushes to main save, so other runs restore main's cache. - save-if: ${{ github.ref == 'refs/heads/main' }} - - - name: Build release (SQLite backend) - run: cargo build --release -p extenddb --no-default-features --features sqlite + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb # The migration lock between serve and migrate, on a catalog of its own # (port 18450), before the suite's server starts. @@ -278,6 +357,9 @@ jobs: name: schema-sqlite path: schema-sqlite.txt if-no-files-found: ignore + # Every leg runs the schema check, so a difference fails each + # leg and each one uploads this name. + overwrite: true - name: Start ExtendDB run: | @@ -355,6 +437,17 @@ jobs: if: failure() run: cat moto.log || true + # The integration job checks these lists. + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: shards-run-integration-sqlite-${{ matrix.shard }} + path: discussions/shards/ + if-no-files-found: warn + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + # The S3 leg: the SQLite build against an S3 backup store on moto. The # backup tests of the main suite (the tests named "backup") run against it # with EXTENDDB_TEST_BACKUP_STORE=s3, and the fidelity suite's health check @@ -368,21 +461,17 @@ jobs: run-integration-sqlite-s3: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 + needs: build-sqlite steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - toolchain: stable + name: extenddb-sqlite + path: target/release - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - cache-on-failure: true - # Only pushes to main save, so other runs restore main's cache. - save-if: ${{ github.ref == 'refs/heads/main' }} - - - name: Build release (SQLite backend) - run: cargo build --release -p extenddb --no-default-features --features sqlite + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: @@ -538,6 +627,7 @@ jobs: # #411's CI, before #740. A run with both should take about 36 min; 50 # leaves room for a slow runner. timeout-minutes: 50 + needs: build-postgres services: postgres: image: pgvector/pgvector:pg16 @@ -553,9 +643,6 @@ jobs: steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - with: - # migrations_tag_blobs reads the release tags. - fetch-depth: 0 - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: @@ -567,8 +654,14 @@ jobs: # Only pushes to main save, so other runs restore main's cache. save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Build release - run: cargo build --release + # After the cache restore, so a cached target/ cannot replace it. + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-postgres + path: target/release + + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Initialize ExtendDB id: init @@ -624,6 +717,47 @@ jobs: EXTENDDB_TEST_BACKUP_STORE: none run: devtools/run-tests --extenddb --rust-integration --release + # The daemonized server logs to syslog; dump it so server-side failures + # are diagnosable from the job log. + - name: Dump server syslog on failure + if: failure() + run: sudo journalctl -t extenddb --no-pager -n 500 || true + + # The PostgreSQL storage-level tests drive the storage layer directly and + # need only the database, not a running server, so they run beside + # run-rust-integration instead of after it. + run-postgres-storage-tests: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_PASSWORD: devpass + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 5 + ports: + - 127.0.0.1:5432:5432 + + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + # migrations_tag_blobs reads the release tags. + fetch-depth: 0 + + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-on-failure: true + # Only pushes to main save, so other runs restore main's cache. + save-if: ${{ github.ref == 'refs/heads/main' }} + # The control plane for vector indexes is not reachable over the wire while # this backend declares no vector search capability, so its tests drive the # storage layer directly against this job's PostgreSQL. The PutItem create @@ -647,12 +781,6 @@ jobs: EXTENDDB_TEST_PG_REQUIRED: "1" run: cargo test --release -p extenddb-storage-postgres --tests - # The daemonized server logs to syslog; dump it so server-side failures - # are diagnosable from the job log. - - name: Dump server syslog on failure - if: failure() - run: sudo journalctl -t extenddb --no-pager -n 500 || true - # The runtime-detection proof: the same binary, against a PostgreSQL with no # pgvector, must refuse vector indexes over the wire. Today the job above runs # on a plain image too, so this looks like a duplicate of it; it is not, because @@ -662,6 +790,7 @@ jobs: run-rust-integration-postgres-novector: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 + needs: build-postgres services: postgres: image: postgres:16 @@ -688,8 +817,14 @@ jobs: # Only pushes to main save, so other runs restore main's cache. save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Build release - run: cargo build --release + # After the cache restore, so a cached target/ cannot replace it. + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-postgres + path: target/release + + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Initialize ExtendDB id: init @@ -750,6 +885,7 @@ jobs: run-rust-integration-sqlite: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 + needs: build-sqlite steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 @@ -764,8 +900,14 @@ jobs: # Only pushes to main save, so other runs restore main's cache. save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Build release (SQLite backend) - run: cargo build --release -p extenddb --no-default-features --features sqlite + # After the cache restore, so a cached target/ cannot replace it. + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-sqlite + path: target/release + + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Initialize ExtendDB id: init @@ -834,6 +976,7 @@ jobs: run-integration-sqlite-s3, run-integration-dev-mode, run-rust-integration, + run-postgres-storage-tests, run-rust-integration-postgres-novector, run-rust-integration-sqlite, ] @@ -845,7 +988,22 @@ jobs: [ "${{ needs.run-integration-sqlite-s3.result }}" != "success" ] || \ [ "${{ needs.run-integration-dev-mode.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration.result }}" != "success" ] || \ + [ "${{ needs.run-postgres-storage-tests.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration-postgres-novector.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ]; then exit 1 fi + + # Fails unless the shards of each sharded job together collected every + # test of the whole suite exactly once, so a bad split cannot pass. + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: shards-run-integration-[0-9]* + path: shards/run-integration + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: shards-run-integration-sqlite-* + path: shards/run-integration-sqlite + - name: Check the shards ran every test + run: python3 devtools/test-shards check shards/run-integration shards/run-integration-sqlite diff --git a/devtools/pytest_shard.py b/devtools/pytest_shard.py new file mode 100644 index 000000000..c11080514 --- /dev/null +++ b/devtools/pytest_shard.py @@ -0,0 +1,35 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""pytest plugin for `devtools/run-tests --shard K/N`. + +Loaded with `-p pytest_shard` (run-tests puts devtools/ on PYTHONPATH). It +keeps the collected tests whose file is listed in $EXTENDDB_SHARD_FILES, one +path per line relative to the working directory, as `devtools/test-shards +plan` writes them, and deselects the rest. The suite's own arguments are left +as they are, so filters, ignores and xdist options apply exactly as in an +unsharded run. Under xdist every worker reads the same list and keeps the +same tests. +""" + +import os + +import pytest + + +def _shard_files(): + path = os.environ.get("EXTENDDB_SHARD_FILES") + if not path: + raise pytest.UsageError( + "pytest_shard: EXTENDDB_SHARD_FILES is not set") + with open(path, encoding="utf-8") as f: + return {line.strip() for line in f if line.strip()} + + +def pytest_collection_modifyitems(config, items): + files = _shard_files() + keep, drop = [], [] + for item in items: + (keep if os.path.relpath(str(item.path)) in files else drop).append(item) + if drop: + config.hook.pytest_deselected(items=drop) + items[:] = keep diff --git a/devtools/run-tests b/devtools/run-tests index 436a2e618..ed44b9fe8 100755 --- a/devtools/run-tests +++ b/devtools/run-tests @@ -37,6 +37,8 @@ # --release use release build for Rust tests # --filter PATTERN pass to cargo test / pytest -k # --parallel[=N] run pytest in parallel (default: 1/3 of CPU cores) +# --shard K/N run only shard K of N of each pytest suite (CI); only +# shard 1 runs --terraform # --config PATH config file path (default: extenddb.toml in repo root) # # Environment (--pytest against extenddb): @@ -81,6 +83,8 @@ RELEASE=false FILTER="" CONFIG_PATH="" PARALLEL="" +# --shard K/N; CI sets EXTENDDB_TEST_SHARD instead. +SHARD="${EXTENDDB_TEST_SHARD:-}" BACKEND="postgres" usage() { @@ -109,6 +113,8 @@ Options: --release release build for Rust tests --filter PATTERN filter for cargo test / pytest -k --parallel[=N] run pytest in parallel (default: 1/3 of CPU cores) + --shard K/N run only shard K of N of each pytest suite; only shard 1 + runs --terraform --config PATH config file path Environment (--extenddb with an integration suite): @@ -150,6 +156,7 @@ while [[ $# -gt 0 ]]; do --parallel=*) PARALLEL="${1#--parallel=}"; shift ;; + --shard) SHARD="$2"; shift 2 ;; -h|--help) usage ;; *) echo "error: unknown option: $1"; usage ;; esac @@ -518,6 +525,36 @@ if $RUN_RUST_INTEGRATION; then echo "" fi +# --- Shards (--shard K/N) --- +# CI splits the long pytest suites across jobs. Before each suite runs, +# devtools/test-shards prepare collects the whole suite with the suite's own +# arguments, picks this shard's files, and records both lists under +# $SHARD_DIR for the summary job's `devtools/test-shards check`. pytest then +# loads devtools/pytest_shard.py, which keeps only those files. A shard with +# no files of a suite runs nothing for it. shard_suite SUITE CMD... leaves +# the command to run in SHARD_CMD. +SHARD_DIR="${EXTENDDB_SHARD_DIR:-discussions/shards}" +shard_suite() { + local suite="$1" rc=0 + shift + # "${@:4}" is the suite's arguments after "python3 -m pytest". + "$SCRIPT_DIR/test-shards" prepare --shard "$SHARD" --out "$SHARD_DIR" \ + --backend "$BACKEND" --config "${CONFIG_PATH:-extenddb.toml}" \ + "$suite" -- "${@:4}" || rc=$? + SHARD_CMD=("$@") + if (( rc == 0 )); then + export PYTHONPATH="$SCRIPT_DIR${PYTHONPATH:+:$PYTHONPATH}" + export EXTENDDB_SHARD_FILES="$SHARD_DIR/files-$suite.txt" + SHARD_CMD+=(-p pytest_shard) + elif (( rc == 10 )); then + echo " shard $SHARD has no $suite files; nothing to run" + SHARD_CMD=(true) + else + echo " ✗ devtools/test-shards prepare failed for the $suite suite" + SHARD_CMD=(false) + fi +} + # --- Pytest --- if $RUN_PYTEST; then OUTFILE="discussions/test-pytest-${HASH}.txt" @@ -550,6 +587,7 @@ if $RUN_PYTEST; then if [[ -n "$FILTER" ]]; then PYTEST_ARGS+=(-k "$FILTER") fi + if [[ -n "$SHARD" ]]; then shard_suite main "${PYTEST_ARGS[@]}"; PYTEST_ARGS=("${SHARD_CMD[@]}"); fi if "${PYTEST_ARGS[@]}" 2>&1 | tee "$OUTFILE"; then echo " ✓ Pytest passed" else @@ -612,6 +650,7 @@ if $RUN_COMPREHENSIVE; then if [[ -n "$FILTER" ]]; then COMP_ARGS+=(-k "$FILTER") fi + if [[ -n "$SHARD" ]]; then shard_suite comprehensive "${COMP_ARGS[@]}"; COMP_ARGS=("${SHARD_CMD[@]}"); fi if "${COMP_ARGS[@]}" 2>&1 | tee "$OUTFILE"; then echo " ✓ Comprehensive tests passed" else @@ -627,6 +666,10 @@ if $RUN_TERRAFORM; then echo "=== Terraform apply → ${OUTFILE} ===" if [[ "$TARGET" == "real-dynamodb" ]]; then echo " skipped: the configuration points at an ExtendDB endpoint" + elif [[ -n "$SHARD" && "${SHARD%%/*}" != "1" ]]; then + # The apply is one run, not a suite of test files, so a sharded job + # runs it on shard 1 only. + echo " skipped: shard $SHARD (shard 1 runs the apply)" elif EXTENDDB_TEST_ENDPOINT="$EXTENDDB_TEST_ENDPOINT" \ tests/terraform/warm-throughput/run.sh 2>&1 | tee "$OUTFILE"; then echo " ✓ Terraform apply passed" @@ -648,6 +691,7 @@ if $RUN_PYTEST \ if [[ -n "$FILTER" ]]; then CLI_ARGS+=(-k "$FILTER") fi + if [[ -n "$SHARD" ]]; then shard_suite cli "${CLI_ARGS[@]}"; CLI_ARGS=("${SHARD_CMD[@]}"); fi if "${CLI_ARGS[@]}" 2>&1 | tee "$CLI_OUTFILE"; then echo " ✓ CLI lifecycle tests passed" else diff --git a/devtools/test-shard-weights.tsv b/devtools/test-shard-weights.tsv new file mode 100644 index 000000000..a3243bde2 --- /dev/null +++ b/devtools/test-shard-weights.tsv @@ -0,0 +1,233 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Seconds each test file (pytest suites) or tests/rust module (Rust +# integration suite) took per backend. The main and comprehensive rows are +# read from the job logs of the push run for 0cfb0d92f on 2026-11-08 +# (run-integration, run-integration-sqlite, mongodb-pytest and +# cassandra-pytest). A row of a file that run did not run keeps its earlier +# value. The cli and rust-integration rows are read from the job logs of +# the merge group run for c5fe7818 on 2026-10-30. A "-" is a value that +# was not measured. devtools/test-shards uses them only to balance +# shards: a file missing here is weighted by its test count, and a stale +# value can make one shard slower but cannot drop a test, because CI +# checks that the shards together collected every test of the whole suite +# (devtools/test-shards check). +# +# suite unit postgres sqlite mongodb cassandra +main tests/test_abac.py 290 280 249 246 +main tests/test_attribute_name_limit_on_update.py 1 1 1 2 +main tests/test_attribute_value_validation.py 4 5 4 5 +main tests/test_auth_error_fidelity.py 1 1 0 1 +main tests/test_auth_integration.py 110 123 108 94 +main tests/test_auth_permissions.py 91 103 99 81 +main tests/test_authz_arn_conditions.py 31 29 26 27 +main tests/test_authz_cross_account.py 23 25 22 22 +main tests/test_authz_resource_arn.py 96 113 93 79 +main tests/test_backup_restore_fidelity.py 64 83 176 100 +main tests/test_batch_operations.py 9 12 3 12 +main tests/test_cache_coherence.py 16 17 14 15 +main tests/test_cli_settings_redaction.py 1 0 0 0 +main tests/test_concurrency.py 139 171 143 148 +main tests/test_condition_nested_paths.py 1 1 1 1 +main tests/test_conditional_atomicity.py 6 5 3 9 +main tests/test_conditional_operator.py 4 3 1 3 +main tests/test_conditional_writes.py 9 17 6 18 +main tests/test_config_fields.py 7 6 2 6 +main tests/test_console_account_scope.py 82 93 84 69 +main tests/test_console_cache_coherence.py 7 7 7 6 +main tests/test_console_integration.py 5 5 5 24 +main tests/test_console_metrics_admin_only.py 127 110 112 71 +main tests/test_console_session_revocation.py 7 7 7 8 +main tests/test_console_settings_redaction.py 0 10 0 0 +main tests/test_consumed_capacity_shape.py 10 10 2 13 +main tests/test_continuous_backups.py 12 11 3 11 +main tests/test_cross_account_isolation.py 32 32 27 33 +main tests/test_dev_mode_authorization.py 0 0 0 0 +main tests/test_expr_operand_type_and_size.py 5 5 1 4 +main tests/test_expression_attribute_map_validation.py 4 2 1 2 +main tests/test_expression_error_prefix.py 1 0 0 1 +main tests/test_expression_param_usage.py 1 1 0 1 +main tests/test_expression_size_limit.py 1 1 0 1 +main tests/test_fgac_attributes.py 10 10 9 10 +main tests/test_fgac_leading_keys.py 112 120 100 107 +main tests/test_gsi_async.py 11 7 13 10 +main tests/test_gsi_async_queue_sqlite.py 0 275 0 0 +main tests/test_gsi_attribute_definitions.py 2 1 1 1 +main tests/test_gsi_multi_range_query.py 0 1 0 1 +main tests/test_gsi_online_backfill.py 12 5 22 0 +main tests/test_idempotency_account_scope.py 15 14 10 26 +main tests/test_import_export.py - - 2 8 +main tests/test_index_key_validation_on_update.py 2 1 1 2 +main tests/test_index_rows_track_base_item.py 4 3 4 6 +main tests/test_item_operations.py 19 28 8 28 +main tests/test_item_size_at_limit.py 17 18 1 2 +main tests/test_list_map_item_size.py 12 11 4 11 +main tests/test_management_settings_redaction.py 1 1 1 1 +main tests/test_metrics.py 5 5 4 5 +main tests/test_misc_operations.py 6 7 2 7 +main tests/test_mongodb_backup_restore.py 0 0 0 0 +main tests/test_mongodb_commit_outcome.py 0 0 1 0 +main tests/test_mongodb_physical_cleanup.py 0 0 17 0 +main tests/test_mongodb_ttl_cleanup.py 0 0 5 0 +main tests/test_multipart_gsi.py 8 7 2 8 +main tests/test_nested_projection.py 1 1 0 1 +main tests/test_partiql_authorization.py 199 227 128 218 +main tests/test_partiql_authorization_keys.py 112 122 80 114 +main tests/test_partiql_authorization_order.py 122 124 68 135 +main tests/test_partiql_batch.py 3 3 4 5 +main tests/test_partiql_boolean_keys.py 2 2 1 2 +main tests/test_partiql_check_order.py 1 1 0 1 +main tests/test_partiql_decomposed_reads.py 5 4 2 5 +main tests/test_partiql_exists.py 1 1 0 1 +main tests/test_partiql_forms.py 2 2 1 2 +main tests/test_partiql_grammar.py 6 8 4 7 +main tests/test_partiql_index_disjuncts.py 1 1 1 2 +main tests/test_partiql_index_keys.py 14 16 5 13 +main tests/test_partiql_index_reads.py 1 1 1 2 +main tests/test_partiql_item_depth.py 1 2 1 2 +main tests/test_partiql_item_size.py 1 1 1 1 +main tests/test_partiql_key_conditions.py 4 6 3 7 +main tests/test_partiql_key_pages.py 9 5 1 5 +main tests/test_partiql_key_types.py 2 1 1 6 +main tests/test_partiql_limits.py 2 3 1 3 +main tests/test_partiql_not_forms.py 6 6 2 6 +main tests/test_partiql_numbers.py 0 2 1 3 +main tests/test_partiql_order_by.py 5 5 1 5 +main tests/test_partiql_page_bytes.py 4 3 1 3 +main tests/test_partiql_page_reads.py 2 6 3 11 +main tests/test_partiql_phase3.py 10 9 3 8 +main tests/test_partiql_read_capacity.py 1 1 0 1 +main tests/test_partiql_repeated_names.py 1 1 0 1 +main tests/test_partiql_requests.py 1 1 0 1 +main tests/test_partiql_select.py 1 1 2 1 +main tests/test_partiql_set_members.py 1 1 0 1 +main tests/test_partiql_transaction.py 3 3 2 4 +main tests/test_partiql_update_paths.py 1 2 0 1 +main tests/test_partiql_visit_order.py 2 4 3 6 +main tests/test_partiql_words.py 1 8 1 0 +main tests/test_partiql_write.py 4 4 4 5 +main tests/test_projection_overlapping_paths.py 1 1 0 1 +main tests/test_put_item_create_race.py 8 7 5 17 +main tests/test_query_scan.py 17 21 6 22 +main tests/test_rate_limiting.py 24 28 24 35 +main tests/test_read_paging.py 9 10 6 7 +main tests/test_resource_policy_variables.py 34 40 34 30 +main tests/test_resource_tag_migration.py 0 14 0 0 +main tests/test_search_vectors_condition_limit.py 2 3 5 0 +main tests/test_set_evaluation.py 2 2 1 3 +main tests/test_stream_arn_scope.py 40 45 39 35 +main tests/test_stream_table_reuse.py 11 9 6 13 +main tests/test_streams.py 11 16 4 16 +main tests/test_table_operations.py 21 23 6 23 +main tests/test_table_throughput_mode.py 9 9 3 10 +main tests/test_tag_request_allow_conditions.py 106 101 90 101 +main tests/test_tag_request_array_conditions.py 65 63 52 60 +main tests/test_tag_request_conditions.py 103 100 85 98 +main tests/test_tag_request_repeated_keys.py 16 7 6 7 +main tests/test_tag_request_variable_conditions.py 29 26 21 26 +main tests/test_transact_absent_item_isolation.py 5 4 4 19 +main tests/test_transact_expression_validation.py 1 4 1 4 +main tests/test_transact_get_snapshot.py 5 6 5 5 +main tests/test_transact_update_remove_only.py 3 4 1 5 +main tests/test_transact_write_conflict.py 2 1 2 13 +main tests/test_transaction_key_size_validation.py 7 7 2 6 +main tests/test_transaction_operations.py 1 0 1 2 +main tests/test_ttl.py 127 120 114 95 +main tests/test_unused_expression_attribute_names.py 1 1 0 1 +main tests/test_update_item_creates_key.py 7 5 3 5 +main tests/test_update_item_dollar_values.py 39 41 10 41 +main tests/test_update_item_number_validation.py 5 5 10 5 +main tests/test_update_remove_only_missing_item.py 4 4 1 5 +main tests/test_update_table_billing_validation.py 4 13 2 1 +main tests/test_validation_precedence.py 0 0 0 0 +main tests/test_vector_index_query_scan.py 1 1 0 0 +main tests/test_wording_parity_validation.py 7 7 2 7 +comprehensive tests/python/test_batch.py 23 17 5 17 +comprehensive tests/python/test_begins_with_type_check.py 2 2 0 1 +comprehensive tests/python/test_composite_keys.py 16 10 3 11 +comprehensive tests/python/test_control_plane.py 26 18 5 19 +comprehensive tests/python/test_data_types.py 46 32 9 33 +comprehensive tests/python/test_error_handling.py 6 4 1 4 +comprehensive tests/python/test_expressions.py 27 19 5 19 +comprehensive tests/python/test_gsi.py 8 6 3 7 +comprehensive tests/python/test_item_edge_cases.py 60 40 10 40 +comprehensive tests/python/test_items.py 60 40 12 42 +comprehensive tests/python/test_query_scan.py 3 2 1 2 +comprehensive tests/python/test_query_start_key_index.py 5 5 2 6 +comprehensive tests/python/test_query_start_key_partition.py 31 20 5 20 +comprehensive tests/python/test_query_start_key_range.py 56 36 10 37 +comprehensive tests/python/test_rfc0003_concurrency.py 10 6 3 7 +comprehensive tests/python/test_scan_edge_cases.py 35 24 8 25 +comprehensive tests/python/test_stream_shard_ownership.py 7 5 2 6 +comprehensive tests/python/test_streams.py 37 31 21 31 +comprehensive tests/python/test_streams_transactions.py 16 13 10 14 +comprehensive tests/python/test_tables.py 22 13 4 15 +comprehensive tests/python/test_tagging.py 11 7 2 7 +comprehensive tests/python/test_transactions.py 22 14 4 14 +comprehensive tests/python/test_ttl.py 7 4 1 4 +cli tests/test_cli_container_readiness.py 22 - - - +cli tests/test_cli_lifecycle.py 23 - - - +cli tests/test_cli_migrate_concurrency.py 2 - - - +cli tests/test_cli_vector_catalog_migration.py 8 - - - +rust-integration authorization 2 2 1 - +rust-integration backup_arn_scoping 5 4 4 - +rust-integration backup_restore 10 10 11 - +rust-integration batch_get_item 8 6 5 - +rust-integration batch_transact_authz 31 26 26 - +rust-integration batch_write_item 0 0 0 - +rust-integration binary_sort_key 5 5 5 - +rust-integration capacity_throttling 97 104 98 - +rust-integration composite_keys 0 0 0 - +rust-integration concurrency 6 5 5 - +rust-integration condition_expressions 0 0 0 - +rust-integration condition_expressions_more 0 0 0 - +rust-integration consumed_capacity_shape 0 0 0 - +rust-integration create_table_validation 0 0 0 - +rust-integration data_types 16 13 13 - +rust-integration delete_item 0 0 0 - +rust-integration delete_item_more 0 0 0 - +rust-integration empty_values 11 11 11 - +rust-integration error_handling 3 3 3 - +rust-integration get_item 0 0 0 - +rust-integration gsi 14 14 14 - +rust-integration gsi_more 10 11 11 - +rust-integration gsi_on_hash_only_table 3 1 11 - +rust-integration gsi_update_table_attr_defs 8 4 20 - +rust-integration index_key_validation 0 0 0 - +rust-integration lsi 8 7 7 - +rust-integration misc_control_plane 0 0 0 - +rust-integration number_validation 0 0 0 - +rust-integration put_item 1 1 1 - +rust-integration put_item_edge 0 0 0 - +rust-integration query 0 0 0 - +rust-integration query_more 3 3 3 - +rust-integration raw_http 0 0 0 - +rust-integration restore_active_completeness 45 18 63 - +rust-integration scan 13 14 12 - +rust-integration select_projection_validation 0 0 0 - +rust-integration table_operations 6 5 5 - +rust-integration table_operations_more 5 4 4 - +rust-integration tagging 10 11 11 - +rust-integration transact_get_items 0 0 0 - +rust-integration transact_key_validation 0 0 0 - +rust-integration transact_write_concurrency 8 5 7 - +rust-integration transact_write_items 0 0 0 - +rust-integration transact_write_items_more 0 0 0 - +rust-integration transaction_key_size_validation 0 0 0 - +rust-integration transaction_validation 0 0 0 - +rust-integration ttl 23 41 51 - +rust-integration unicode 10 8 8 - +rust-integration update_expressions 1 0 0 - +rust-integration update_expressions_more 0 0 0 - +rust-integration update_item 0 0 0 - +rust-integration update_item_edge 0 0 0 - +rust-integration update_item_more 0 0 0 - +rust-integration update_item_number_validation 0 0 0 - +rust-integration update_table_attribute_definitions 10 4 4 - +rust-integration update_table_billing_validation 0 0 0 - +rust-integration vector_index_search 67 59 0 - +rust-integration vector_index_unsupported 0 0 0 - +rust-integration vector_index_update_rules 6 6 0 - +rust-integration vector_index_write_capacity 6 6 0 - +rust-integration wording_parity_validation 1 1 1 - diff --git a/devtools/test-shards b/devtools/test-shards new file mode 100755 index 000000000..c04b25552 --- /dev/null +++ b/devtools/test-shards @@ -0,0 +1,359 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Split the pytest suites into shards by test file, and check that the +shards of one CI run together collected every test of the whole suite. + + test-shards collect OUT -- PYTEST_ARGS... + Collect PYTEST_ARGS as `pytest --collect-only -q` does and write OUT + with one line per collected test: its file relative to the working + directory, a tab, and its id (that file, then the rest of the pytest + node id). The ids do not depend on which rootdir pytest picked, so a + shard's list and the whole suite's list compare line for line. + + test-shards plan --shard K/N --out DIR [--backend NAME] [--config PATH] + [--weights TSV] SUITE=COLLECTED... + Assign every file of the collected suites to one of N shards and write + DIR/files-SUITE.txt with the files of shard K, one per line, sorted. + Every shard computes the same assignment from the same collection, so + each file lands in exactly one shard. Files are spread by the seconds in + the weights file; a file missing from it counts its tests at the + suite's mean seconds per test. Weights only balance the shards: a wrong + weight makes one shard slower, it cannot drop or repeat a file. + + test-shards prepare --shard K/N --out DIR [--backend NAME] [--config PATH] + SUITE -- PYTEST_ARGS... + What `devtools/run-tests --shard K/N` runs before each pytest suite: + collect the whole SUITE with PYTEST_ARGS into DIR/whole-SUITE.tsv, plan + shard K's files into DIR/files-SUITE.txt, collect what the shard will + run (PYTEST_ARGS plus `-p pytest_shard`) into DIR/ran-SUITE.tsv, and + write K/N to DIR/shard.txt. Exits 0 when the shard has files of SUITE, + 10 when it has none (the caller runs nothing), 1 on any failure. + + test-shards check DIR... + Each DIR holds the lists written by the legs of one sharded job, one + subdirectory per leg (shard.txt, whole-SUITE.tsv, ran-SUITE.tsv), as + actions/download-artifact leaves them. Fails unless the legs are shards + 1 to N of the same N, agree on the whole suite, and their ran lists + together hold every test of it exactly once. +""" + +import argparse +import contextlib +import os +import subprocess +import sys + +# The main suite runs under pytest-xdist; on the CI runners (4 vCPU) that is +# two workers, so a main-suite file costs a shard about half its seconds. The +# other suites run one file after another. +PARALLEL_SUITES = {"main": 2} +DEFAULT_WEIGHTS = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "test-shard-weights.tsv") + + +def parse_shard(text): + try: + k, n = (int(part) for part in text.split("/")) + except ValueError: + raise SystemExit(f"test-shards: --shard must be K/N, got {text!r}") + if not 1 <= k <= n: + raise SystemExit(f"test-shards: --shard {text}: need 1 <= K <= N") + return k, n + + +def read_tsv(path): + with open(path, encoding="utf-8") as f: + return [line.rstrip("\n").split("\t") for line in f if line.strip()] + + +# --- collect --------------------------------------------------------------- + +def cmd_collect(out, pytest_args): + import pytest # noqa: PLC0415 (only collect needs pytest) + + # `python3 -m pytest` puts the working directory first on sys.path; do the + # same so the suites import exactly as they do when they run. + sys.path[0] = os.getcwd() + rows = [] + + class Recorder: + def pytest_collection_finish(self, session): + for item in session.items: + path = os.path.relpath(str(item.path)) + rest = item.nodeid.partition("::")[2] + rows.append((path, f"{path}::{rest}" if rest else path)) + + # Tabs and newlines would split a row; escape them in every list alike. + def field(text): + return (text.replace("\\", "\\\\").replace("\t", "\\t") + .replace("\n", "\\n")) + + # pytest prints the collected ids; keep stdout for callers that read it. + with contextlib.redirect_stdout(sys.stderr): + rc = pytest.main(["--collect-only", "-q", *pytest_args], + plugins=[Recorder()]) + # 5 is pytest's "no tests collected", which is a valid, empty selection. + if rc not in (0, 5): + print(f"test-shards: collection failed (pytest exit {int(rc)})", + file=sys.stderr) + return int(rc) + with open(out, "w", encoding="utf-8") as f: + for path, test_id in rows: + f.write(f"{field(path)}\t{field(test_id)}\n") + print(f"test-shards: {len(rows)} tests collected into {out}", + file=sys.stderr) + return 0 + + +# --- plan ------------------------------------------------------------------ + +def load_weights(path): + """{(suite, file): {backend: seconds}} from the weights file.""" + columns, weights = [], {} + with open(path, encoding="utf-8") as f: + for line in f: + line = line.rstrip("\n") + if line.startswith("# suite\t"): + columns = line[2:].split("\t")[2:] + if not line or line.startswith("#"): + continue + suite, unit, *secs = line.split("\t") + weights[(suite, unit)] = { + b: float(s) for b, s in zip(columns, secs) if s != "-"} + return columns, weights + + +def config_backend(path): + """The [storage] backend a config file names, or None.""" + try: + import tomllib # noqa: PLC0415 (Python 3.11+) + with open(path, "rb") as f: + return tomllib.load(f).get("storage", {}).get("backend") + except (ImportError, OSError, ValueError): + return None + + +def assign(units, n): + """Longest weight first, each to the least loaded shard; ties go to the + lower shard number, so every caller gets the same answer.""" + loads = [0.0] * n + shards = [[] for _ in range(n)] + for unit in sorted(units, key=lambda u: (-u[2], u[0], u[1])): + i = loads.index(min(loads)) + shards[i].append(unit) + loads[i] += unit[2] + return shards, loads + + +def cmd_plan(args): + k, n = parse_shard(args.shard) + backend = (config_backend(args.config) if args.config else None) \ + or args.backend + columns, weights = load_weights(args.weights) + column = backend if backend in columns else (columns[0] if columns else None) + + suites = [] + for spec in args.collected: + suite, sep, path = spec.partition("=") + if not sep: + raise SystemExit(f"test-shards: expected SUITE=FILE, got {spec!r}") + counts = {} + for row in read_tsv(path): + counts[row[0]] = counts.get(row[0], 0) + 1 + suites.append((suite, counts)) + + units = [] + for suite, counts in suites: + known = {f: weights[(suite, f)][column] for f in counts + if column in weights.get((suite, f), {})} + known_tests = sum(counts[f] for f in known) + rate = sum(known.values()) / known_tests if known_tests else 1.0 + divisor = PARALLEL_SUITES.get(suite, 1) + for f, count in counts.items(): + secs = known.get(f, count * rate) + units.append((suite, f, secs / divisor)) + + shards, loads = assign(units, n) + os.makedirs(args.out, exist_ok=True) + for suite, _ in suites: + files = sorted(u[1] for u in shards[k - 1] if u[0] == suite) + with open(os.path.join(args.out, f"files-{suite}.txt"), "w", + encoding="utf-8") as f: + f.writelines(f"{name}\n" for name in files) + print(f"test-shards: {len(units)} files, weights column {column}") + for i, (shard, load) in enumerate(zip(shards, loads), start=1): + per_suite = ", ".join( + f"{suite} {sum(1 for u in shard if u[0] == suite)}" + for suite, _ in suites) + mark = " <- this shard" if i == k else "" + print(f" shard {i}/{n}: {load / 60:5.1f} min modelled ({per_suite})" + f"{mark}") + return 0 + + +# --- check ----------------------------------------------------------------- + +def read_ids(path): + return [row[1] if len(row) > 1 else row[0] for row in read_tsv(path)] + + +def check_group(group): + """Problems found in one sharded job's lists; prints its summary.""" + legs = sorted(root for root, _, files in os.walk(group) + if "shard.txt" in files) + if not legs: + return [f"{group}: no shard lists found (did every leg upload?)"] + problems, shards = [], {} + for leg in legs: + with open(os.path.join(leg, "shard.txt"), encoding="utf-8") as f: + k, n = parse_shard(f.read().strip()) + if k in shards: + problems.append(f"{group}: shard {k} appears twice " + f"({shards[k][0]} and {leg})") + shards[k] = (leg, n) + totals = {n for _, n in shards.values()} + if len(totals) != 1: + return problems + [f"{group}: legs disagree on N: {sorted(totals)}"] + n = totals.pop() + missing = sorted(set(range(1, n + 1)) - set(shards)) + if missing: + problems.append(f"{group}: no lists from shard(s) {missing} of {n}") + + suites = sorted({name[len("whole-"):-len(".tsv")] + for leg, _ in shards.values() for name in os.listdir(leg) + if name.startswith("whole-") and name.endswith(".tsv")}) + if not suites: + problems.append(f"{group}: no whole-suite lists found") + for suite in suites: + whole, ran = None, {} + for k, (leg, _) in sorted(shards.items()): + whole_path = os.path.join(leg, f"whole-{suite}.tsv") + ran_path = os.path.join(leg, f"ran-{suite}.tsv") + if not os.path.exists(whole_path) or not os.path.exists(ran_path): + problems.append(f"{group}: shard {k} has no {suite} lists") + continue + ids = sorted(read_ids(whole_path)) + if whole is None: + whole = ids + elif ids != whole: + problems.append(f"{group}: shard {k} collected a different " + f"whole {suite} suite ({len(ids)} tests, " + f"others {len(whole)})") + ran[k] = read_ids(ran_path) + if whole is None: + continue + seen = {} + for k, ids in ran.items(): + for test_id in ids: + seen.setdefault(test_id, []).append(k) + repeated = sorted(t for t, ks in seen.items() if len(ks) > 1) + absent = sorted(set(whole) - set(seen)) + extra = sorted(set(seen) - set(whole)) + counts = " + ".join(str(len(ran.get(k, []))) for k in range(1, n + 1)) + total = sum(len(ids) for ids in ran.values()) + ok = bool(whole) and not (repeated or absent or extra) + print(f"{group} {suite}: whole {len(whole)}, shards {counts} = " + f"{total} {'OK' if ok else 'MISMATCH'}") + if not whole: + problems.append(f"{group}: the whole {suite} suite is empty") + for label, ids in (("not run by any shard", absent), + ("run by more than one shard", repeated), + ("not in the whole suite", extra)): + if ids: + problems.append(f"{group} {suite}: {len(ids)} test(s) {label}" + f", first: {ids[0]}") + return problems + + +def cmd_check(groups): + problems = [p for group in groups for p in check_group(group)] + for problem in problems: + print(f"error: {problem}", file=sys.stderr) + return 1 if problems else 0 + + +# --- prepare --------------------------------------------------------------- + +EMPTY_SHARD = 10 + + +def cmd_prepare(argv): + if "--" not in argv: + raise SystemExit("usage: test-shards prepare --shard K/N --out DIR " + "[--backend NAME] [--config PATH] SUITE -- " + "PYTEST_ARGS...") + split = argv.index("--") + parser = argparse.ArgumentParser(prog="test-shards prepare") + parser.add_argument("--shard", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--backend", default="postgres") + parser.add_argument("--config") + parser.add_argument("--weights", default=DEFAULT_WEIGHTS) + parser.add_argument("suite") + args = parser.parse_args(argv[:split]) + pytest_args = argv[split + 1:] + k, n = parse_shard(args.shard) + os.makedirs(args.out, exist_ok=True) + whole, ran, files = (os.path.join(args.out, f"{kind}-{args.suite}{ext}") + for kind, ext in (("whole", ".tsv"), ("ran", ".tsv"), + ("files", ".txt"))) + + # Each collection runs in its own interpreter: pytest does not support + # two sessions importing the same test modules in one process. Its + # output is the whole collection tree, so it is shown only on failure. + def collect(out, extra, env=None): + proc = subprocess.run([sys.executable, os.path.abspath(__file__), + "collect", out, "--", *extra, *pytest_args], + env=env, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True) + if proc.returncode: + sys.stderr.write(proc.stdout) + else: + sys.stderr.writelines(line for line in proc.stdout.splitlines(True) + if line.startswith("test-shards:")) + return proc.returncode + + if collect(whole, []): + return 1 + args.collected = [f"{args.suite}={whole}"] + if cmd_plan(args): + return 1 + devtools = os.path.dirname(os.path.abspath(__file__)) + path = os.environ.get("PYTHONPATH") + env = dict(os.environ, EXTENDDB_SHARD_FILES=files, + PYTHONPATH=f"{devtools}{os.pathsep}{path}" if path else devtools) + if collect(ran, ["-p", "pytest_shard"], env): + return 1 + with open(os.path.join(args.out, "shard.txt"), "w", encoding="utf-8") as f: + f.write(f"{k}/{n}\n") + with open(files, encoding="utf-8") as f: + return 0 if f.read().strip() else EMPTY_SHARD + + +def main(argv): + if len(argv) > 1 and argv[1] == "collect": + if len(argv) < 4 or argv[3] != "--": + raise SystemExit("usage: test-shards collect OUT -- PYTEST_ARGS...") + return cmd_collect(argv[2], argv[4:]) + if len(argv) > 1 and argv[1] == "prepare": + return cmd_prepare(argv[2:]) + parser = argparse.ArgumentParser(prog="test-shards") + sub = parser.add_subparsers(dest="command", required=True) + plan = sub.add_parser("plan") + plan.add_argument("--shard", required=True) + plan.add_argument("--out", required=True) + plan.add_argument("--backend", default="postgres") + plan.add_argument("--config") + plan.add_argument("--weights", default=DEFAULT_WEIGHTS) + plan.add_argument("collected", nargs="+", metavar="SUITE=COLLECTED") + check = sub.add_parser("check") + check.add_argument("groups", nargs="+", metavar="DIR") + args = parser.parse_args(argv[1:]) + if args.command == "plan": + return cmd_plan(args) + return cmd_check(args.groups) + + +if __name__ == "__main__": + sys.exit(main(sys.argv))