Shard smoketests using nextest

.github/workflows/ci.yml

@@ -70,25 +70,20 @@ jobs:
             echo "Merge queue commit ${GITHUB_SHA} differs from PR #${pr_number} head ${pr_head_sha}; running CI normally."
           fi
 
-  smoketests:
+  upload-build-artifacts-linux:
     needs: [merge_queue_noop, lints]
     if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }}
-    name: Smoketests (${{ matrix.name }})
-    strategy:
-      matrix:
-        include:
-          - name: Linux
-            runner: spacetimedb-new-runner-2
-          - name: Windows
-            runner: spacetimedb-windows-runner
-    runs-on: ${{ matrix.runner }}
-    timeout-minutes: 120
+    name: Upload build artifacts (Linux)
+    runs-on: spacetimedb-new-runner-2
+    timeout-minutes: 75
     env:
       CARGO_TARGET_DIR: ${{ github.workspace }}/target
       RUST_BACKTRACE: full
-      SPACETIMEDB_CPP_DIR: ${{ github.workspace }}/crates/bindings-cpp
-    steps:
-      - name: Find Git ref
+      ARTIFACT_SUFFIX: linux
+      EXE_SUFFIX: ""
+    steps: &upload-build-artifact-steps
+      - &find-git-ref
+        name: Find Git ref
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         shell: bash
@@ -101,16 +96,19 @@ jobs:
           fi
           echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
 
-      - name: Checkout sources
+      - &checkout-sources
+        name: Checkout sources
         uses: actions/checkout@v4
         with:
           ref: ${{ env.GIT_REF }}
 
       - uses: dsherret/rust-toolchain-file@v1
-      - name: Set default rust toolchain
+      - &set-default-rust-toolchain
+        name: Set default rust toolchain
         run: rustup default $(rustup show active-toolchain | cut -d' ' -f1)
 
-      - name: Cache Rust dependencies
+      - &cache-rust-dependencies
+        name: Cache Rust dependencies
         uses: Swatinem/rust-cache@v2
         with:
           workspaces: ${{ github.workspace }}
@@ -120,6 +118,159 @@ jobs:
           cache-workspace-crates: true
           prefix-key: v1
 
+      # Some runner caches contain V8 metadata without the corresponding static library.
+      - name: Check v8 outputs
+        shell: bash
+        run: |
+          find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true
+          if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then
+            echo "Could not find v8 output file librusty_v8.a; rebuilding manually."
+            cargo clean --release -p v8 || true
+            cargo build --timings --release -p v8
+          fi
+
+      - name: Build smoketest binaries
+        shell: bash
+        run: |
+          cargo build --timings --release \
+            -p spacetimedb-cli \
+            -p spacetimedb-standalone \
+            --features spacetimedb-standalone/allow_loopback_http_for_tests
+
+      - name: Package build artifacts
+        shell: bash
+        run: |
+          tar -czf build-support.tar.gz \
+            "target/release/spacetimedb-cli${EXE_SUFFIX}" \
+            "target/release/spacetimedb-standalone${EXE_SUFFIX}"
+
+      - name: Upload Cargo timing reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-artifacts-cargo-timings-${{ env.ARTIFACT_SUFFIX }}
+          path: ${{ github.workspace }}/target/cargo-timings/
+          if-no-files-found: warn
+          overwrite: true
+          retention-days: 14
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-artifacts-${{ env.ARTIFACT_SUFFIX }}
+          path: build-support.tar.gz
+          if-no-files-found: error
+          overwrite: true
+          retention-days: 14
+
+  upload-build-artifacts-windows:
+    needs: [merge_queue_noop, lints]
+    if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }}
+    name: Upload build artifacts (Windows)
+    runs-on: spacetimedb-windows-runner
+    timeout-minutes: 75
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
+      RUST_BACKTRACE: full
+      ARTIFACT_SUFFIX: windows
+      EXE_SUFFIX: .exe
+    steps: *upload-build-artifact-steps
+
+  smoketest_build_linux:
+    needs: [upload-build-artifacts-linux]
+    name: Build smoketests (Linux)
+    runs-on: spacetimedb-new-runner-2
+    timeout-minutes: 75
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
+      RUST_BACKTRACE: full
+      ARTIFACT_SUFFIX: linux
+      EXE_SUFFIX: ""
+    steps: &smoketest-build-steps
+      - *find-git-ref
+
+      - *checkout-sources
+
+      - uses: dsherret/rust-toolchain-file@v1
+      - *set-default-rust-toolchain
+
+      - *cache-rust-dependencies
+
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@nextest
+
+      - name: Build smoketest dependencies and archive test binaries
+        shell: bash
+        run: |
+          cargo ci smoketests archive --archive-file smoketest-nextest.tar.zst
+
+          shopt -s nullglob
+          precompiled_modules=(target/wasm32-unknown-unknown/release/smoketest_module_*.wasm)
+          if (( ${#precompiled_modules[@]} == 0 )); then
+            echo "No precompiled smoketest modules were produced."
+            exit 1
+          fi
+
+          tar -czf smoketest-support.tar.gz \
+            "target/debug/ci${EXE_SUFFIX}" \
+            "${precompiled_modules[@]}"
+
+      - name: Upload Cargo timing reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: smoketest-build-cargo-timings-${{ env.ARTIFACT_SUFFIX }}
+          path: ${{ github.workspace }}/target/cargo-timings/
+          if-no-files-found: warn
+          overwrite: true
+          retention-days: 14
+
+      - name: Upload smoketest build
+        uses: actions/upload-artifact@v4
+        with:
+          name: smoketest-build-${{ env.ARTIFACT_SUFFIX }}
+          path: |
+            smoketest-nextest.tar.zst
+            smoketest-support.tar.gz
+          if-no-files-found: error
+          overwrite: true
+          retention-days: 14
+
+  smoketest_build_windows:
+    needs: [upload-build-artifacts-windows]
+    name: Build smoketests (Windows)
+    runs-on: spacetimedb-windows-runner
+    timeout-minutes: 75
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
+      RUST_BACKTRACE: full
+      ARTIFACT_SUFFIX: windows
+      EXE_SUFFIX: .exe
+    steps: *smoketest-build-steps
+
+  smoketest_partitions_linux:
+    needs: [smoketest_build_linux]
+    name: Smoketests (Linux ${{ matrix.partition }}/2)
+    strategy:
+      fail-fast: false
+      matrix:
+        partition: [1, 2]
+    runs-on: spacetimedb-new-runner-2
+    timeout-minutes: 120
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
+      RUST_BACKTRACE: full
+      SPACETIMEDB_CPP_DIR: ${{ github.workspace }}/crates/bindings-cpp
+      ARTIFACT_SUFFIX: linux
+      PARTITION_COUNT: 2
+    steps: &smoketest-partition-steps
+      - *find-git-ref
+
+      - *checkout-sources
+
+      - uses: dsherret/rust-toolchain-file@v1
+      - *set-default-rust-toolchain
+
       - uses: actions/setup-dotnet@v4
         with:
           global-json-file: global.json
@@ -153,7 +304,7 @@ jobs:
           .\emsdk install 4.0.21
           .\emsdk activate 4.0.21
 
-      - name: Install psql (Windows)
+      - name: Install psql
         if: runner.os == 'Windows'
         shell: pwsh
         run: |
@@ -167,6 +318,7 @@ jobs:
 
       - name: Update dotnet workloads
         if: runner.os == 'Windows'
+        shell: pwsh
         run: |
           # Fail properly if any individual command fails
           $ErrorActionPreference = 'Stop'
@@ -191,53 +343,77 @@ jobs:
           cd sdks/csharp
           ./tools~/write-nuget-config.sh ../..
 
-      # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a.
-      # ChatGPT suspects that this could be due to different build invocations using the same target dir,
-      # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with
-      # `cargo build --manifest-path` (which apparently build different dependency trees).
-      # However, we've been unable to fix it so... /shrug
-      - name: Check v8 outputs
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@nextest
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: build-artifacts-${{ env.ARTIFACT_SUFFIX }}
+          path: build-artifacts
+
+      - name: Extract build artifacts
         shell: bash
         run: |
-          find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true
-          if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then
-            echo "Could not find v8 output file librusty_v8.a; rebuilding manually."
-            cargo clean --release -p v8 || true
-            cargo build --release -p v8
+          tar -xzf build-artifacts/build-support.tar.gz
+          exe_suffix=""
+          if [[ "${RUNNER_OS}" == "Windows" ]]; then
+            exe_suffix=".exe"
           fi
+          test -f "${CARGO_TARGET_DIR}/release/spacetimedb-cli${exe_suffix}"
+          test -f "${CARGO_TARGET_DIR}/release/spacetimedb-standalone${exe_suffix}"
 
-      - name: Install cargo-nextest
-        uses: taiki-e/install-action@nextest
+      - name: Download smoketest build
+        uses: actions/download-artifact@v4
+        with:
+          name: smoketest-build-${{ env.ARTIFACT_SUFFIX }}
 
-      # --test-threads=1 eliminates contention in the C# tests where they fight over bindings
-      # build artifacts.
-      # It also seemed to improve performance a fair amount (11m -> 6m)
-      - name: Run smoketests (Linux)
+      - name: Extract smoketest support files
+        shell: bash
+        run: tar -xzf smoketest-support.tar.gz
+
+      # Serial execution avoids contention between the C# tests over generated bindings.
+      - name: Run smoketest partition (Linux)
         if: runner.os == 'Linux'
         shell: bash
         run: |
           if [ -f ~/emsdk/emsdk_env.sh ]; then
             source ~/emsdk/emsdk_env.sh
           fi
-          cargo ci smoketests -- --test-threads=1
+          ./target/debug/ci smoketests run-archive \
+            --archive-file smoketest-nextest.tar.zst \
+            -- \
+            --partition hash:${{ matrix.partition }}/${{ env.PARTITION_COUNT }}
 
-      # Due to Emscripten PATH issues this was separated to make sure OpenSSL still builds correctly
-      - name: Run smoketests (Windows)
+      # Due to Emscripten PATH issues this remains separate from the Linux command.
+      - name: Run smoketest partition (Windows)
         if: runner.os == 'Windows'
         shell: pwsh
         run: |
           if (Test-Path "$env:USERPROFILE\emsdk\emsdk_env.ps1") {
             & "$env:USERPROFILE\emsdk\emsdk_env.ps1" | Out-Null
           }
-          cargo ci smoketests -- --test-threads=1          
-
-      - name: Upload Cargo timing reports
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: cargo-timings-smoketests-${{ matrix.name }}
-          path: ${{ github.workspace }}/target/cargo-timings/
-          retention-days: 30
+          .\target\debug\ci.exe smoketests run-archive `
+            --archive-file smoketest-nextest.tar.zst `
+            -- `
+            --partition hash:${{ matrix.partition }}/${{ env.PARTITION_COUNT }}
+
+  smoketest_partitions_windows:
+    needs: [smoketest_build_windows]
+    name: Smoketests (Windows ${{ matrix.partition }}/4)
+    strategy:
+      fail-fast: false
+      matrix:
+        partition: [1, 2, 3, 4]
+    runs-on: spacetimedb-windows-runner
+    timeout-minutes: 120
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
+      RUST_BACKTRACE: full
+      SPACETIMEDB_CPP_DIR: ${{ github.workspace }}/crates/bindings-cpp
+      ARTIFACT_SUFFIX: windows
+      PARTITION_COUNT: 4
+    steps: *smoketest-partition-steps
 
   # this is a no-op version of the above check with a trivially-passing body.
   # we can't just let the check be entirely skipped because each matrix target is a required check,
@@ -249,8 +425,12 @@ jobs:
     strategy:
       matrix:
         include:
-          - name: Linux
-          - name: Windows
+          - name: Linux 1/2
+          - name: Linux 2/2
+          - name: Windows 1/4
+          - name: Windows 2/4
+          - name: Windows 3/4
+          - name: Windows 4/4
     runs-on: ubuntu-latest
     steps:
       - name: Skip duplicate merge queue smoketest
@@ -451,7 +631,7 @@ jobs:
         with:
           name: cargo-timings-public-lints
           path: ${{ github.workspace }}/target/cargo-timings/
-          retention-days: 30
+          retention-days: 14
 
   codeowners_check:
     if: ${{ github.event_name == 'pull_request' }}

crates/smoketests/src/lib.rs

@@ -56,7 +56,6 @@ pub mod modules;
 use anyhow::{bail, Context, Result};
 use regex::Regex;
 use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard};
-use std::env;
 use std::fs;
 use std::io::{BufRead, BufReader};
 use std::path::{Path, PathBuf};

crates/smoketests/tests/smoketests/default_module_clippy.rs

@@ -1,17 +1,8 @@
 //! These tests verify that the Rust module templates have no clippy warnings.
 
-use std::path::PathBuf;
+use spacetimedb_smoketests::workspace_root;
 use std::process::Command;
 
-fn workspace_root() -> PathBuf {
-    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .parent()
-        .unwrap()
-        .parent()
-        .unwrap()
-        .to_path_buf()
-}
-
 /// Run clippy on a template's spacetimedb module directory.
 /// Both templates use workspace dependencies, so they can be checked in place.
 fn check_template_clippy(template_name: &str) {