diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c39b4a2458e..ff9ec6242d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,25 +70,20 @@ jobs: echo "Merge queue commit ${GITHUB_SHA} differs from PR #${pr_number} head ${pr_head_sha}; running CI normally." fi - smoketests: + upload-build-artifacts-linux: needs: [merge_queue_noop, lints] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} - name: Smoketests (${{ matrix.name }}) - strategy: - matrix: - include: - - name: Linux - runner: spacetimedb-new-runner-2 - - name: Windows - runner: spacetimedb-windows-runner - runs-on: ${{ matrix.runner }} - timeout-minutes: 120 + name: Upload build artifacts (Linux) + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 75 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full - SPACETIMEDB_CPP_DIR: ${{ github.workspace }}/crates/bindings-cpp - steps: - - name: Find Git ref + ARTIFACT_SUFFIX: linux + EXE_SUFFIX: "" + steps: &upload-build-artifact-steps + - &find-git-ref + name: Find Git ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash @@ -101,16 +96,19 @@ jobs: fi echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - - name: Checkout sources + - &checkout-sources + name: Checkout sources uses: actions/checkout@v4 with: ref: ${{ env.GIT_REF }} - uses: dsherret/rust-toolchain-file@v1 - - name: Set default rust toolchain + - &set-default-rust-toolchain + name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - name: Cache Rust dependencies + - &cache-rust-dependencies + name: Cache Rust dependencies uses: Swatinem/rust-cache@v2 with: workspaces: ${{ github.workspace }} @@ -120,6 +118,159 @@ jobs: cache-workspace-crates: true prefix-key: v1 + # Some runner caches contain V8 metadata without the corresponding static library. + - name: Check v8 outputs + shell: bash + run: | + find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true + if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then + echo "Could not find v8 output file librusty_v8.a; rebuilding manually." + cargo clean --release -p v8 || true + cargo build --timings --release -p v8 + fi + + - name: Build smoketest binaries + shell: bash + run: | + cargo build --timings --release \ + -p spacetimedb-cli \ + -p spacetimedb-standalone \ + --features spacetimedb-standalone/allow_loopback_http_for_tests + + - name: Package build artifacts + shell: bash + run: | + tar -czf build-support.tar.gz \ + "target/release/spacetimedb-cli${EXE_SUFFIX}" \ + "target/release/spacetimedb-standalone${EXE_SUFFIX}" + + - name: Upload Cargo timing reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: build-artifacts-cargo-timings-${{ env.ARTIFACT_SUFFIX }} + path: ${{ github.workspace }}/target/cargo-timings/ + if-no-files-found: warn + overwrite: true + retention-days: 14 + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: build-artifacts-${{ env.ARTIFACT_SUFFIX }} + path: build-support.tar.gz + if-no-files-found: error + overwrite: true + retention-days: 14 + + upload-build-artifacts-windows: + needs: [merge_queue_noop, lints] + if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} + name: Upload build artifacts (Windows) + runs-on: spacetimedb-windows-runner + timeout-minutes: 75 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + ARTIFACT_SUFFIX: windows + EXE_SUFFIX: .exe + steps: *upload-build-artifact-steps + + smoketest_build_linux: + needs: [upload-build-artifacts-linux] + name: Build smoketests (Linux) + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 75 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + ARTIFACT_SUFFIX: linux + EXE_SUFFIX: "" + steps: &smoketest-build-steps + - *find-git-ref + + - *checkout-sources + + - uses: dsherret/rust-toolchain-file@v1 + - *set-default-rust-toolchain + + - *cache-rust-dependencies + + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - name: Build smoketest dependencies and archive test binaries + shell: bash + run: | + cargo ci smoketests archive --archive-file smoketest-nextest.tar.zst + + shopt -s nullglob + precompiled_modules=(target/wasm32-unknown-unknown/release/smoketest_module_*.wasm) + if (( ${#precompiled_modules[@]} == 0 )); then + echo "No precompiled smoketest modules were produced." + exit 1 + fi + + tar -czf smoketest-support.tar.gz \ + "target/debug/ci${EXE_SUFFIX}" \ + "${precompiled_modules[@]}" + + - name: Upload Cargo timing reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: smoketest-build-cargo-timings-${{ env.ARTIFACT_SUFFIX }} + path: ${{ github.workspace }}/target/cargo-timings/ + if-no-files-found: warn + overwrite: true + retention-days: 14 + + - name: Upload smoketest build + uses: actions/upload-artifact@v4 + with: + name: smoketest-build-${{ env.ARTIFACT_SUFFIX }} + path: | + smoketest-nextest.tar.zst + smoketest-support.tar.gz + if-no-files-found: error + overwrite: true + retention-days: 14 + + smoketest_build_windows: + needs: [upload-build-artifacts-windows] + name: Build smoketests (Windows) + runs-on: spacetimedb-windows-runner + timeout-minutes: 75 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + ARTIFACT_SUFFIX: windows + EXE_SUFFIX: .exe + steps: *smoketest-build-steps + + smoketest_partitions_linux: + needs: [smoketest_build_linux] + name: Smoketests (Linux ${{ matrix.partition }}/2) + strategy: + fail-fast: false + matrix: + partition: [1, 2] + runs-on: spacetimedb-new-runner-2 + timeout-minutes: 120 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + SPACETIMEDB_CPP_DIR: ${{ github.workspace }}/crates/bindings-cpp + ARTIFACT_SUFFIX: linux + PARTITION_COUNT: 2 + steps: &smoketest-partition-steps + - *find-git-ref + + - *checkout-sources + + - uses: dsherret/rust-toolchain-file@v1 + - *set-default-rust-toolchain + - uses: actions/setup-dotnet@v4 with: global-json-file: global.json @@ -153,7 +304,7 @@ jobs: .\emsdk install 4.0.21 .\emsdk activate 4.0.21 - - name: Install psql (Windows) + - name: Install psql if: runner.os == 'Windows' shell: pwsh run: | @@ -167,6 +318,7 @@ jobs: - name: Update dotnet workloads if: runner.os == 'Windows' + shell: pwsh run: | # Fail properly if any individual command fails $ErrorActionPreference = 'Stop' @@ -191,53 +343,77 @@ jobs: cd sdks/csharp ./tools~/write-nuget-config.sh ../.. - # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a. - # ChatGPT suspects that this could be due to different build invocations using the same target dir, - # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with - # `cargo build --manifest-path` (which apparently build different dependency trees). - # However, we've been unable to fix it so... /shrug - - name: Check v8 outputs + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: build-artifacts-${{ env.ARTIFACT_SUFFIX }} + path: build-artifacts + + - name: Extract build artifacts shell: bash run: | - find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true - if ! [ -f "${CARGO_TARGET_DIR}"/release/gn_out/obj/librusty_v8.a ]; then - echo "Could not find v8 output file librusty_v8.a; rebuilding manually." - cargo clean --release -p v8 || true - cargo build --release -p v8 + tar -xzf build-artifacts/build-support.tar.gz + exe_suffix="" + if [[ "${RUNNER_OS}" == "Windows" ]]; then + exe_suffix=".exe" fi + test -f "${CARGO_TARGET_DIR}/release/spacetimedb-cli${exe_suffix}" + test -f "${CARGO_TARGET_DIR}/release/spacetimedb-standalone${exe_suffix}" - - name: Install cargo-nextest - uses: taiki-e/install-action@nextest + - name: Download smoketest build + uses: actions/download-artifact@v4 + with: + name: smoketest-build-${{ env.ARTIFACT_SUFFIX }} - # --test-threads=1 eliminates contention in the C# tests where they fight over bindings - # build artifacts. - # It also seemed to improve performance a fair amount (11m -> 6m) - - name: Run smoketests (Linux) + - name: Extract smoketest support files + shell: bash + run: tar -xzf smoketest-support.tar.gz + + # Serial execution avoids contention between the C# tests over generated bindings. + - name: Run smoketest partition (Linux) if: runner.os == 'Linux' shell: bash run: | if [ -f ~/emsdk/emsdk_env.sh ]; then source ~/emsdk/emsdk_env.sh fi - cargo ci smoketests -- --test-threads=1 + ./target/debug/ci smoketests run-archive \ + --archive-file smoketest-nextest.tar.zst \ + -- \ + --partition hash:${{ matrix.partition }}/${{ env.PARTITION_COUNT }} - # Due to Emscripten PATH issues this was separated to make sure OpenSSL still builds correctly - - name: Run smoketests (Windows) + # Due to Emscripten PATH issues this remains separate from the Linux command. + - name: Run smoketest partition (Windows) if: runner.os == 'Windows' shell: pwsh run: | if (Test-Path "$env:USERPROFILE\emsdk\emsdk_env.ps1") { & "$env:USERPROFILE\emsdk\emsdk_env.ps1" | Out-Null } - cargo ci smoketests -- --test-threads=1 - - - name: Upload Cargo timing reports - if: always() - uses: actions/upload-artifact@v4 - with: - name: cargo-timings-smoketests-${{ matrix.name }} - path: ${{ github.workspace }}/target/cargo-timings/ - retention-days: 30 + .\target\debug\ci.exe smoketests run-archive ` + --archive-file smoketest-nextest.tar.zst ` + -- ` + --partition hash:${{ matrix.partition }}/${{ env.PARTITION_COUNT }} + + smoketest_partitions_windows: + needs: [smoketest_build_windows] + name: Smoketests (Windows ${{ matrix.partition }}/4) + strategy: + fail-fast: false + matrix: + partition: [1, 2, 3, 4] + runs-on: spacetimedb-windows-runner + timeout-minutes: 120 + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + RUST_BACKTRACE: full + SPACETIMEDB_CPP_DIR: ${{ github.workspace }}/crates/bindings-cpp + ARTIFACT_SUFFIX: windows + PARTITION_COUNT: 4 + steps: *smoketest-partition-steps # this is a no-op version of the above check with a trivially-passing body. # we can't just let the check be entirely skipped because each matrix target is a required check, @@ -249,8 +425,12 @@ jobs: strategy: matrix: include: - - name: Linux - - name: Windows + - name: Linux 1/2 + - name: Linux 2/2 + - name: Windows 1/4 + - name: Windows 2/4 + - name: Windows 3/4 + - name: Windows 4/4 runs-on: ubuntu-latest steps: - name: Skip duplicate merge queue smoketest @@ -451,7 +631,7 @@ jobs: with: name: cargo-timings-public-lints path: ${{ github.workspace }}/target/cargo-timings/ - retention-days: 30 + retention-days: 14 codeowners_check: if: ${{ github.event_name == 'pull_request' }} diff --git a/crates/smoketests/src/lib.rs b/crates/smoketests/src/lib.rs index 5b90de38a03..5c90ae988d2 100644 --- a/crates/smoketests/src/lib.rs +++ b/crates/smoketests/src/lib.rs @@ -56,7 +56,6 @@ pub mod modules; use anyhow::{bail, Context, Result}; use regex::Regex; use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; -use std::env; use std::fs; use std::io::{BufRead, BufReader}; use std::path::{Path, PathBuf}; diff --git a/crates/smoketests/tests/smoketests/default_module_clippy.rs b/crates/smoketests/tests/smoketests/default_module_clippy.rs index f32b0fd94e2..73e76f40357 100644 --- a/crates/smoketests/tests/smoketests/default_module_clippy.rs +++ b/crates/smoketests/tests/smoketests/default_module_clippy.rs @@ -1,17 +1,8 @@ //! These tests verify that the Rust module templates have no clippy warnings. -use std::path::PathBuf; +use spacetimedb_smoketests::workspace_root; use std::process::Command; -fn workspace_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .parent() - .unwrap() - .to_path_buf() -} - /// Run clippy on a template's spacetimedb module directory. /// Both templates use workspace dependencies, so they can be checked in place. fn check_template_clippy(template_name: &str) { diff --git a/crates/smoketests/tests/smoketests/namespaces.rs b/crates/smoketests/tests/smoketests/namespaces.rs index 3d632edd7e2..971de3ae027 100644 --- a/crates/smoketests/tests/smoketests/namespaces.rs +++ b/crates/smoketests/tests/smoketests/namespaces.rs @@ -1,15 +1,6 @@ -use spacetimedb_smoketests::Smoketest; +use spacetimedb_smoketests::{workspace_root, Smoketest}; use std::fs; -use std::path::{Path, PathBuf}; - -fn workspace_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .parent() - .unwrap() - .to_path_buf() -} +use std::path::Path; /// Count occurrences of a needle string in all .cs files under a directory fn count_matches(dir: &Path, needle: &str) -> usize { diff --git a/crates/smoketests/tests/smoketests/permissions.rs b/crates/smoketests/tests/smoketests/permissions.rs index 35a9f8a7c37..36e0b0dcd1e 100644 --- a/crates/smoketests/tests/smoketests/permissions.rs +++ b/crates/smoketests/tests/smoketests/permissions.rs @@ -1,14 +1,4 @@ -use spacetimedb_smoketests::Smoketest; -use std::path::PathBuf; - -fn workspace_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .parent() - .unwrap() - .to_path_buf() -} +use spacetimedb_smoketests::{workspace_root, Smoketest}; /// Ensure that anyone has the permission to call any standard reducer #[test] diff --git a/tools/ci/README.md b/tools/ci/README.md index f0563733ee2..c20d264469d 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -108,13 +108,13 @@ This is required for servers that reject direct server-issued logins for privile Optionally accepts an auth host to pass through to `spacetime login`, for example `--auth-host=https://spacetimedb.com`. -- `--dotnet `: +- `--dotnet `: Run .NET smoketests - `args `: Additional arguments to pass to the test runner - `--help`: Print help (see a summary with '-h') #### `prepare` -Only build binaries without running tests +Local helper: only build binaries without running tests. Use this before running `cargo test --all` to ensure binaries are built. @@ -138,6 +138,31 @@ Usage: check-mod-list - `--help`: Print help +#### `archive` + +**Usage:** +```bash +Usage: archive --archive-file +``` + +**Options:** + +- `--archive-file `: Path to the nextest archive to create +- `--help`: Print help + +#### `run-archive` + +**Usage:** +```bash +Usage: run-archive --archive-file [ARGS]... +``` + +**Options:** + +- `--archive-file `: Path to the nextest archive to run +- `args `: Additional arguments to pass to nextest +- `--help`: Print help + #### `help` **Usage:** diff --git a/tools/ci/src/smoketest.rs b/tools/ci/src/smoketest.rs index 2a35b91ca59..90648b69eb0 100644 --- a/tools/ci/src/smoketest.rs +++ b/tools/ci/src/smoketest.rs @@ -4,7 +4,7 @@ use clap::{Args, Subcommand}; use duct::cmd; use spacetimedb_guard::ensure_binaries_built; use std::ffi::OsStr; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::{env, fs}; use tempfile::TempDir; @@ -36,6 +36,7 @@ pub struct SmoketestsArgs { #[arg(long, num_args = 0..=1, require_equals = true, default_missing_value = "")] auth_host: Option, + /// Run .NET smoketests. #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] dotnet: bool, @@ -46,11 +47,29 @@ pub struct SmoketestsArgs { #[derive(Subcommand)] enum SmoketestCmd { - /// Only build binaries without running tests + /// Local helper: only build binaries without running tests. /// /// Use this before running `cargo test --all` to ensure binaries are built. Prepare, CheckModList, + + /// CI build job: build dependencies and archive the smoketest binaries. + Archive { + /// Path to the nextest archive to create. + #[arg(long)] + archive_file: PathBuf, + }, + + /// CI partition job: run smoketests from an existing nextest archive. + RunArchive { + /// Path to the nextest archive to run. + #[arg(long)] + archive_file: PathBuf, + + /// Additional arguments to pass to nextest. + #[arg(trailing_var_arg = true)] + args: Vec, + }, } pub fn run(args: SmoketestsArgs) -> Result<()> { @@ -66,6 +85,8 @@ pub fn run(args: SmoketestsArgs) -> Result<()> { eprintln!("smoketests/mod.rs is up to date."); Ok(()) } + Some(SmoketestCmd::Archive { archive_file }) => archive_smoketests(&archive_file), + Some(SmoketestCmd::RunArchive { archive_file, args }) => run_smoketest_archive(&archive_file, args), None => run_smoketest(args.server, args.dotnet, args.auth_host.as_deref(), args.args), } } @@ -142,6 +163,56 @@ fn build_precompiled_modules() -> Result<()> { Ok(()) } +fn archive_smoketests(archive_file: &Path) -> Result<()> { + build_precompiled_modules()?; + + let status = Command::new("cargo") + .args([ + "nextest", + "archive", + "--release", + "--timings", + "-p", + "spacetimedb-smoketests", + "--archive-file", + ]) + .arg(archive_file) + .status()?; + ensure!(status.success(), "Failed to archive smoketests"); + Ok(()) +} + +fn run_smoketest_archive(archive_file: &Path, args: Vec) -> Result<()> { + let workspace_root = env::current_dir()?; + let archive_file = if archive_file.is_absolute() { + archive_file.to_path_buf() + } else { + workspace_root.join(archive_file) + }; + + // CI supplies the release CLI and standalone through the shared build artifact. + let cli_path = ensure_binaries_built(); + let base_config_dir = prepare_base_config(&cli_path, None, None)?; + let base_config_path = base_config_dir.path().join("config.toml"); + + let mut cmd = Command::new("cargo"); + set_env(&mut cmd, None, true, false, &base_config_path); + cmd.args(["nextest", "run", "--archive-file"]) + .arg(archive_file) + .args(["--workspace-remap"]) + .arg(&workspace_root) + .args(["--no-fail-fast", "--no-tests", "pass", "-j", "1"]) + .args(args); + + ensure!(cmd.status()?.success(), "Tests failed"); + let diff_status = cmd!("bash", "tools/check-diff.sh", "crates/smoketests").run()?; + ensure!( + diff_status.status.success(), + "There is a diff in the smoketests directory." + ); + Ok(()) +} + /// Default parallelism for smoketests. /// 16 was found to be optimal - higher values cause OS scheduler overhead. const DEFAULT_PARALLELISM: &str = "16";