From 973339e706d73996247a802cb69457f3fec89eed Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Fri, 7 Aug 2026 12:01:42 -0400 Subject: [PATCH 01/34] Split cargo ci subcommands into packages --- .github/CODEOWNERS | 4 +- Cargo.lock | 154 ++- Cargo.toml | 16 + tools/ci/Cargo.toml | 11 - tools/ci/README.md | 315 ++---- tools/ci/commands/cli-docs/Cargo.toml | 11 + tools/ci/commands/cli-docs/src/main.rs | 53 + tools/ci/commands/dlls/Cargo.toml | 9 + tools/ci/commands/dlls/src/main.rs | 10 + tools/ci/commands/docs/Cargo.toml | 9 + tools/ci/commands/docs/src/main.rs | 28 + .../ci/commands/global-json-policy/Cargo.toml | 14 + .../commands/global-json-policy/src/main.rs | 337 ++++++ tools/ci/commands/keynote-bench/Cargo.toml | 10 + .../keynote-bench/src/main.rs} | 7 +- tools/ci/commands/lint/Cargo.toml | 14 + tools/ci/commands/lint/src/main.rs | 407 ++++++++ tools/ci/commands/other-workflows/Cargo.toml | 14 + .../other-workflows}/src/cla_assistant.rs | 18 +- .../other-workflows}/src/codeowners_check.rs | 3 +- tools/ci/commands/other-workflows/src/main.rs | 42 + tools/ci/commands/publish-checks/Cargo.toml | 9 + tools/ci/commands/publish-checks/src/main.rs | 37 + tools/ci/commands/self-docs/Cargo.toml | 9 + tools/ci/commands/self-docs/src/main.rs | 137 +++ tools/ci/commands/smoketests/Cargo.toml | 12 + .../smoketests/src/main.rs} | 24 +- tools/ci/commands/test/Cargo.toml | 9 + tools/ci/commands/test/src/main.rs | 131 +++ tools/ci/commands/typescript-test/Cargo.toml | 9 + tools/ci/commands/typescript-test/src/main.rs | 45 + tools/ci/commands/update-flow/Cargo.toml | 12 + tools/ci/commands/update-flow/src/main.rs | 69 ++ .../commands/version-upgrade-check/Cargo.toml | 9 + .../version-upgrade-check/src/main.rs | 22 + tools/ci/commands/wasm-bindings/Cargo.toml | 9 + tools/ci/commands/wasm-bindings/src/main.rs | 50 + tools/ci/common/Cargo.toml | 6 + tools/ci/common/src/lib.rs | 10 + tools/ci/src/ci_docs.rs | 85 -- tools/ci/src/main.rs | 985 ++---------------- tools/ci/src/util.rs | 11 - 42 files changed, 1921 insertions(+), 1255 deletions(-) create mode 100644 tools/ci/commands/cli-docs/Cargo.toml create mode 100644 tools/ci/commands/cli-docs/src/main.rs create mode 100644 tools/ci/commands/dlls/Cargo.toml create mode 100644 tools/ci/commands/dlls/src/main.rs create mode 100644 tools/ci/commands/docs/Cargo.toml create mode 100644 tools/ci/commands/docs/src/main.rs create mode 100644 tools/ci/commands/global-json-policy/Cargo.toml create mode 100644 tools/ci/commands/global-json-policy/src/main.rs create mode 100644 tools/ci/commands/keynote-bench/Cargo.toml rename tools/ci/{src/keynote_bench.rs => commands/keynote-bench/src/main.rs} (81%) create mode 100644 tools/ci/commands/lint/Cargo.toml create mode 100644 tools/ci/commands/lint/src/main.rs create mode 100644 tools/ci/commands/other-workflows/Cargo.toml rename tools/ci/{ => commands/other-workflows}/src/cla_assistant.rs (95%) rename tools/ci/{ => commands/other-workflows}/src/codeowners_check.rs (99%) create mode 100644 tools/ci/commands/other-workflows/src/main.rs create mode 100644 tools/ci/commands/publish-checks/Cargo.toml create mode 100644 tools/ci/commands/publish-checks/src/main.rs create mode 100644 tools/ci/commands/self-docs/Cargo.toml create mode 100644 tools/ci/commands/self-docs/src/main.rs create mode 100644 tools/ci/commands/smoketests/Cargo.toml rename tools/ci/{src/smoketest.rs => commands/smoketests/src/main.rs} (96%) create mode 100644 tools/ci/commands/test/Cargo.toml create mode 100644 tools/ci/commands/test/src/main.rs create mode 100644 tools/ci/commands/typescript-test/Cargo.toml create mode 100644 tools/ci/commands/typescript-test/src/main.rs create mode 100644 tools/ci/commands/update-flow/Cargo.toml create mode 100644 tools/ci/commands/update-flow/src/main.rs create mode 100644 tools/ci/commands/version-upgrade-check/Cargo.toml create mode 100644 tools/ci/commands/version-upgrade-check/src/main.rs create mode 100644 tools/ci/commands/wasm-bindings/Cargo.toml create mode 100644 tools/ci/commands/wasm-bindings/src/main.rs create mode 100644 tools/ci/common/Cargo.toml create mode 100644 tools/ci/common/src/lib.rs delete mode 100644 tools/ci/src/ci_docs.rs delete mode 100644 tools/ci/src/util.rs diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7ed75409a55..4d6f7594981 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,8 +8,8 @@ LICENSE.txt @cloutiertyler /crates/cli/src/ @bfops @cloutiertyler @jdetter /tools/ci/ @bfops @cloutiertyler @jdetter -/tools/ci/src/keynote_bench.rs @joshua-spacetime @cloutiertyler @jdetter -/tools/ci/src/codeowners_check.rs @cloutiertyler +/tools/ci/commands/keynote-bench/ @joshua-spacetime @cloutiertyler @jdetter +/tools/ci/commands/other-workflows/src/codeowners_check.rs @cloutiertyler /tools/upgrade-version/ @bfops @jdetter @cloutiertyler @rekhoff /tools/release/ @bfops @jdetter @cloutiertyler @rekhoff /tools/license-check/ @bfops @jdetter @cloutiertyler diff --git a/Cargo.lock b/Cargo.lock index bb5abb64fdd..f63db3e9e7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -899,20 +899,172 @@ name = "ci" version = "0.1.0" dependencies = [ "anyhow", - "chrono", + "duct", +] + +[[package]] +name = "ci-cli-docs" +version = "0.1.0" +dependencies = [ + "anyhow", "clap 4.5.50", "duct", "env_logger 0.10.2", + "log", +] + +[[package]] +name = "ci-common" +version = "0.1.0" + +[[package]] +name = "ci-dlls" +version = "0.1.0" +dependencies = [ + "anyhow", + "duct", + "env_logger 0.10.2", +] + +[[package]] +name = "ci-docs-build" +version = "0.1.0" +dependencies = [ + "anyhow", + "duct", + "env_logger 0.10.2", +] + +[[package]] +name = "ci-global-json-policy" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.50", + "duct", + "env_logger 0.10.2", + "log", + "regex", + "serde_json", + "tempfile", +] + +[[package]] +name = "ci-keynote-bench" +version = "0.1.0" +dependencies = [ + "anyhow", + "env_logger 0.10.2", "keynote-bench-harness", + "spacetimedb-guard", +] + +[[package]] +name = "ci-lint" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.50", + "duct", + "env_logger 0.10.2", "log", "regex", + "serde_json", + "tempfile", +] + +[[package]] +name = "ci-other-workflows" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.50", + "duct", + "env_logger 0.10.2", + "regex", "reqwest 0.12.24", "serde", "serde_json", +] + +[[package]] +name = "ci-publish-checks" +version = "0.1.0" +dependencies = [ + "anyhow", + "duct", + "env_logger 0.10.2", +] + +[[package]] +name = "ci-self-docs" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.50", + "duct", +] + +[[package]] +name = "ci-smoketests" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.50", + "duct", + "env_logger 0.10.2", "spacetimedb-guard", "tempfile", ] +[[package]] +name = "ci-test" +version = "0.1.0" +dependencies = [ + "anyhow", + "duct", + "env_logger 0.10.2", +] + +[[package]] +name = "ci-typescript-test" +version = "0.1.0" +dependencies = [ + "anyhow", + "duct", + "env_logger 0.10.2", +] + +[[package]] +name = "ci-update-flow" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.50", + "duct", + "env_logger 0.10.2", + "log", + "tempfile", +] + +[[package]] +name = "ci-version-upgrade-check" +version = "0.1.0" +dependencies = [ + "anyhow", + "duct", + "env_logger 0.10.2", +] + +[[package]] +name = "ci-wasm-bindings" +version = "0.1.0" +dependencies = [ + "anyhow", + "duct", + "env_logger 0.10.2", +] + [[package]] name = "ciborium" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index c31354942a5..2f4930191b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,22 @@ members = [ "sdks/rust/tests/procedural-view-pk-client", "sdks/rust/tests/event-table-client", "tools/ci", + "tools/ci/commands/test", + "tools/ci/commands/lint", + "tools/ci/commands/wasm-bindings", + "tools/ci/commands/dlls", + "tools/ci/commands/smoketests", + "tools/ci/commands/keynote-bench", + "tools/ci/commands/update-flow", + "tools/ci/commands/cli-docs", + "tools/ci/commands/self-docs", + "tools/ci/commands/global-json-policy", + "tools/ci/commands/publish-checks", + "tools/ci/commands/typescript-test", + "tools/ci/commands/version-upgrade-check", + "tools/ci/commands/docs", + "tools/ci/commands/other-workflows", + "tools/ci/common", "tools/keynote-bench-harness", "tools/license-check", "tools/replace-spacetimedb", diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml index 3374b66b34d..61bd0a0e422 100644 --- a/tools/ci/Cargo.toml +++ b/tools/ci/Cargo.toml @@ -4,16 +4,5 @@ version = "0.1.0" edition.workspace = true [dependencies] -log.workspace = true anyhow.workspace = true -chrono = { workspace = true, features=["clock"] } -clap.workspace = true -regex.workspace = true -reqwest = { workspace = true, features = ["blocking", "json"] } -serde.workspace = true -serde_json.workspace = true duct.workspace = true -tempfile.workspace = true -env_logger.workspace = true -keynote-bench-harness = { path = "../keynote-bench-harness" } -spacetimedb-guard.workspace = true diff --git a/tools/ci/README.md b/tools/ci/README.md index 46eefb9cadc..ebc5a4fbbcd 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -6,360 +6,191 @@ This document provides an overview of the `cargo ci` command-line tool, and docu ## `cargo ci` -SpacetimeDB CI tasks - -This tool provides several subcommands for automating CI workflows in SpacetimeDB. - -It may be invoked via `cargo ci `, or simply `cargo ci` to run all subcommands in sequence. It is mostly designed to be run in CI environments via the github workflows, but can also be run locally - **Usage:** ```bash -Usage: cargo ci [OPTIONS] [COMMAND] +cargo ci [--skip ] [COMMAND] [ARGS]... ``` -**Options:** - -- `--skip `: Skip specified subcommands when running all - -When no subcommand is specified, all subcommands are run in sequence. This option allows specifying subcommands to skip when running all. For example, to skip the `unreal-tests` subcommand, use `--skip unreal-tests`. - -- `--help`: Print help (see a summary with '-h') - ### `test` -Runs tests - -Runs rust tests, codegens csharp sdk and runs csharp tests. This does not include Unreal tests. This expects to run in a clean git state. - **Usage:** ```bash -Usage: test +cargo ci test ``` -**Options:** - -- `--help`: Print help (see a summary with '-h') - ### `lint` -Lints the codebase - -Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there are no warnings. - **Usage:** ```bash -Usage: lint +cargo ci lint ``` -**Options:** - -- `--help`: Print help (see a summary with '-h') - ### `wasm-bindings` -Tests Wasm bindings - -Runs tests for the codegen crate and builds a test module with the wasm bindings. - **Usage:** ```bash -Usage: wasm-bindings +cargo ci wasm-bindings ``` -**Options:** - -- `--help`: Print help (see a summary with '-h') - ### `dlls` -Deprecated; use `cargo regen csharp dlls`. - -Builds and packs C# DLLs and NuGet packages for local Unity workflows. - **Usage:** ```bash -Usage: dlls +cargo ci dlls ``` -**Options:** - -- `--help`: Print help (see a summary with '-h') - ### `smoketests` -Runs smoketests - -Executes the smoketests suite with some default exclusions. - **Usage:** ```bash -Usage: smoketests [OPTIONS] [ARGS]... [COMMAND] -``` - -**Options:** - -- `--server `: Run tests against a remote server instead of spawning local servers. - -When specified, tests will connect to the given URL instead of starting local server instances. Tests that require local server control (like restart tests) will be skipped. - -- `--auth-host `: Use a SpacetimeAuth-issued login for remote-server tests. +This command builds the binaries needed by the smoketests, then runs them. This prevents race +conditions when running tests in parallel with nextest, where multiple test processes might try to +build the same binaries simultaneously -This is required for servers that reject direct server-issued logins for privileged operations. +Usage: ci-smoketests [OPTIONS] [ARGS]... [COMMAND] -Optionally accepts an auth host to pass through to `spacetime login`, for example `--auth-host=https://spacetimedb.com`. +Commands: + prepare Only build binaries without running tests + check-mod-list + help Print this message or the help of the given subcommand(s) -- `--dotnet `: -- `args `: Additional arguments to pass to the test runner -- `--help`: Print help (see a summary with '-h') +Arguments: + [ARGS]... + Additional arguments to pass to the test runner -#### `prepare` +Options: + --server + Run tests against a remote server instead of spawning local servers. -Only build binaries without running tests + When specified, tests will connect to the given URL instead of starting local server + instances. Tests that require local server control (like restart tests) will be skipped. -Use this before running `cargo test --all` to ensure binaries are built. + --auth-host[=] + Use a SpacetimeAuth-issued login for remote-server tests. -**Usage:** -```bash -Usage: prepare -``` - -**Options:** + This is required for servers that reject direct server-issued logins for privileged + operations. -- `--help`: Print help (see a summary with '-h') + Optionally accepts an auth host to pass through to `spacetime login`, for example + `--auth-host=https://spacetimedb.com`. -#### `check-mod-list` + --dotnet + [default: true] + [possible values: true, false] -**Usage:** -```bash -Usage: check-mod-list + -h, --help + Print help (see a summary with '-h') ``` -**Options:** - -- `--help`: Print help - -#### `help` - -**Usage:** -```bash -Usage: help [COMMAND]... -``` - -**Options:** - -- `subcommand `: Print help for the subcommand(s) - ### `keynote-bench` -Runs the keynote benchmark as a CI performance regression gate. - -Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. - **Usage:** ```bash -Usage: keynote-bench +cargo ci keynote-bench ``` -**Options:** - -- `--help`: Print help (see a summary with '-h') - ### `update-flow` -Tests the update flow - -Tests the self-update flow by building the spacetimedb-update binary for the specified target, by default the current target, and performing a self-install into a temporary directory. - **Usage:** ```bash -Usage: update-flow [OPTIONS] -``` +Usage: ci-update-flow [OPTIONS] -**Options:** +Options: + --target + Target triple to build for, by default the current target. Used by github workflows to + check the update flow on multiple platforms. -- `--target `: Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms. -- `--github-token-auth`: Whether to enable github token authentication feature when building the update binary. By default this is disabled. -- `--help`: Print help (see a summary with '-h') + --github-token-auth + Whether to enable github token authentication feature when building the update binary. By + default this is disabled. + + -h, --help + Print help (see a summary with '-h') +``` ### `cli-docs` **Usage:** ```bash -Usage: cli-docs [OPTIONS] -``` +Usage: ci-cli-docs [OPTIONS] -**Options:** +Options: + --spacetime-path + specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is + located) -- `--spacetime-path `: specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located) -- `--help`: Print help (see a summary with '-h') + -h, --help + Print help (see a summary with '-h') +``` ### `self-docs` **Usage:** ```bash -Usage: self-docs [OPTIONS] -``` +Usage: ci-self-docs [OPTIONS] -**Options:** +Options: + --check + Only check for changes, do not generate the docs -- `--check`: Only check for changes, do not generate the docs -- `--help`: Print help (see a summary with '-h') + -h, --help + Print help (see a summary with '-h') +``` ### `global-json-policy` **Usage:** ```bash -Usage: global-json-policy +cargo ci global-json-policy ``` -**Options:** - -- `--help`: Print help - ### `publish-checks` **Usage:** ```bash -Usage: publish-checks +cargo ci publish-checks ``` -**Options:** - -- `--help`: Print help - ### `typescript-test` **Usage:** ```bash -Usage: typescript-test +cargo ci typescript-test ``` -**Options:** - -- `--help`: Print help - ### `version-upgrade-check` **Usage:** ```bash -Usage: version-upgrade-check +cargo ci version-upgrade-check ``` -**Options:** - -- `--help`: Print help - ### `docs` **Usage:** ```bash -Usage: docs +cargo ci docs ``` -**Options:** - -- `--help`: Print help - ### `other-workflows` **Usage:** ```bash -Usage: other-workflows -``` - -**Options:** - -- `--help`: Print help - -#### `codeowners-check` - -**Usage:** -```bash -Usage: codeowners-check --base-ref --pr-number -``` - -**Options:** +Usage: ci-other-workflows -- `--base-ref `: Git ref to compare against, usually origin/ -- `--pr-number `: Pull request number to inspect for approval state -- `--help`: Print help +Commands: + codeowners-check Checks that sensitive CODEOWNERS-controlled files have the required approvals + cla-assistant Interacts with CLA Assistant + help Print this message or the help of the given subcommand(s) -#### `cla-assistant` - -**Usage:** -```bash -Usage: cla-assistant +Options: + -h, --help Print help ``` -**Options:** - -- `--help`: Print help - -##### `retry` - -**Usage:** -```bash -Usage: retry [OPTIONS] --pr-number -``` - -**Options:** - -- `--pr-number `: Pull request number to check -- `--repo `: Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY -- `--help`: Print help - -##### `status` - -**Usage:** -```bash -Usage: status [OPTIONS] <--pr |--sha > -``` - -**Options:** - -- `--pr `: Pull request number whose head commit should be checked -- `--sha `: Commit SHA to check -- `--repo `: Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY -- `--help`: Print help - -##### `help` - -**Usage:** -```bash -Usage: help [COMMAND]... -``` - -**Options:** - -- `subcommand `: Print help for the subcommand(s) - -#### `help` - -**Usage:** -```bash -Usage: help [COMMAND]... -``` - -**Options:** - -- `subcommand `: Print help for the subcommand(s) - -### `help` - -**Usage:** -```bash -Usage: help [COMMAND]... -``` - -**Options:** - -- `subcommand `: Print help for the subcommand(s) - - --- This document is auto-generated by running: ```bash cargo ci self-docs -``` \ No newline at end of file +``` diff --git a/tools/ci/commands/cli-docs/Cargo.toml b/tools/ci/commands/cli-docs/Cargo.toml new file mode 100644 index 00000000000..5e8dcee6dcd --- /dev/null +++ b/tools/ci/commands/cli-docs/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ci-cli-docs" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +clap.workspace = true +log.workspace = true +env_logger.workspace = true diff --git a/tools/ci/commands/cli-docs/src/main.rs b/tools/ci/commands/cli-docs/src/main.rs new file mode 100644 index 00000000000..14cae753286 --- /dev/null +++ b/tools/ci/commands/cli-docs/src/main.rs @@ -0,0 +1,53 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::Result; +use clap::Parser; +use duct::{cmd, Expression}; +use std::ffi::OsStr; +fn pnpm(args: I) -> Expression +where + I: IntoIterator, + S: AsRef, +{ + let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); + if cfg!(windows) { + let mut full: Vec = vec!["/c".into(), "pnpm".into()]; + full.extend(args); + cmd("cmd", full) + } else { + cmd("pnpm", args) + } +} +use std::env; + +#[derive(Parser)] +struct Args { + #[arg( + long, + long_help = "specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located)" + )] + spacetime_path: Option, +} + +fn main() -> Result<()> { + env_logger::init(); + let Args { spacetime_path } = Args::parse(); + if let Some(path) = spacetime_path { + env::set_current_dir(path).ok(); + } + let current_dir = env::current_dir().expect("No current directory!"); + let dir_name = current_dir.file_name().expect("No current directory!"); + if dir_name != "SpacetimeDB" && dir_name != "public" { + anyhow::bail!("You must execute this binary from inside of the SpacetimeDB directory, or use --spacetime-path"); + } + + pnpm(["install", "--recursive"]).run()?; + pnpm(["generate-cli-docs"]).dir("docs").run()?; + let out = cmd!("git", "status", "--porcelain", "--", "docs").read()?; + if out.is_empty() { + log::info!("No docs changes detected"); + } else { + anyhow::bail!("CLI docs are out of date:\n{out}"); + } + + Ok(()) +} diff --git a/tools/ci/commands/dlls/Cargo.toml b/tools/ci/commands/dlls/Cargo.toml new file mode 100644 index 00000000000..7ade3c64682 --- /dev/null +++ b/tools/ci/commands/dlls/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "ci-dlls" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +env_logger.workspace = true diff --git a/tools/ci/commands/dlls/src/main.rs b/tools/ci/commands/dlls/src/main.rs new file mode 100644 index 00000000000..fb31e24d1b0 --- /dev/null +++ b/tools/ci/commands/dlls/src/main.rs @@ -0,0 +1,10 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::Result; +use duct::cmd; +fn main() -> Result<()> { + env_logger::init(); + eprintln!("warning: `cargo ci dlls` is deprecated; use `cargo regen csharp dlls` instead"); + cmd!("cargo", "regen", "csharp", "dlls").run()?; + + Ok(()) +} diff --git a/tools/ci/commands/docs/Cargo.toml b/tools/ci/commands/docs/Cargo.toml new file mode 100644 index 00000000000..4bf39b2f08f --- /dev/null +++ b/tools/ci/commands/docs/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "ci-docs-build" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +env_logger.workspace = true diff --git a/tools/ci/commands/docs/src/main.rs b/tools/ci/commands/docs/src/main.rs new file mode 100644 index 00000000000..76de2c90747 --- /dev/null +++ b/tools/ci/commands/docs/src/main.rs @@ -0,0 +1,28 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::Result; +use duct::{cmd, Expression}; +use std::ffi::OsStr; +fn pnpm(args: I) -> Expression +where + I: IntoIterator, + S: AsRef, +{ + let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); + if cfg!(windows) { + let mut full: Vec = vec!["/c".into(), "pnpm".into()]; + full.extend(args); + cmd("cmd", full) + } else { + cmd("pnpm", args) + } +} +fn run_docs_build() -> Result<()> { + pnpm(["install"]).dir("docs").run()?; + pnpm(["build"]).dir("docs").run()?; + Ok(()) +} + +fn main() -> Result<()> { + env_logger::init(); + run_docs_build() +} diff --git a/tools/ci/commands/global-json-policy/Cargo.toml b/tools/ci/commands/global-json-policy/Cargo.toml new file mode 100644 index 00000000000..af85fbba7b6 --- /dev/null +++ b/tools/ci/commands/global-json-policy/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "ci-global-json-policy" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +clap.workspace = true +regex.workspace = true +serde_json.workspace = true +tempfile.workspace = true +env_logger.workspace = true +log.workspace = true diff --git a/tools/ci/commands/global-json-policy/src/main.rs b/tools/ci/commands/global-json-policy/src/main.rs new file mode 100644 index 00000000000..0aa152ebda0 --- /dev/null +++ b/tools/ci/commands/global-json-policy/src/main.rs @@ -0,0 +1,337 @@ +#![allow(clippy::disallowed_macros)] +#![allow(dead_code)] + +use anyhow::{bail, Context, Result}; +use duct::{cmd, Expression}; +use serde_json::Value; +use std::collections::BTreeSet; +use std::fs; +use std::path::Path; +use std::path::PathBuf; + +/// On Windows, `pnpm` is installed as a `.cmd` shim which `CreateProcess` cannot +/// find without going through the shell. Wrapping with `cmd /c` fixes this. +/// On Unix, we invoke `pnpm` directly. +fn pnpm(args: I) -> Expression +where + I: IntoIterator, + S: AsRef, +{ + let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); + if cfg!(windows) { + let mut full: Vec = vec!["/c".into(), "pnpm".into()]; + full.extend(args); + cmd("cmd", full) + } else { + cmd("pnpm", args) + } +} + +fn check_global_json_policy() -> Result<()> { + ensure_repo_root()?; + + let root_json = Path::new("global.json"); + let root_contents = fs::read_to_string(root_json)?; + + let globals = git_tracked_files(":(glob)**/global.json")?; + + let mut ok = true; + for p in globals { + let meta = fs::symlink_metadata(&p)?; + let is_symlink = meta.file_type().is_symlink(); + let is_template_global_json = p.strip_prefix(".").unwrap_or(&p).starts_with(Path::new("templates")); + if is_template_global_json && is_symlink { + eprintln!( + "Error: {} is a symlink. Template files must not be symlinks; they are copied literally and this will break if the CLI is built under Windows where symlinks are not supported.", + p.display() + ); + ok = false; + } + + let contents = fs::read_to_string(&p)?; + if contents != root_contents { + eprintln!("Error: {} does not match the root global.json contents", p.display()); + ok = false; + } else if !is_template_global_json || !is_symlink { + println!("OK: {}", p.display()); + } + } + + if !ok { + bail!("global.json policy check failed"); + } + + Ok(()) +} + +fn package_json_pnpm_version(package_manager: &str) -> Option<&str> { + package_manager.strip_prefix("pnpm@") +} + +fn git_tracked_files(pathspec: &str) -> Result> { + let output = cmd!("git", "ls-files", pathspec).read()?; + Ok(output.lines().map(PathBuf::from).collect()) +} + +fn package_json_string_value(package_json: &Value, key: &str) -> Option { + package_json.get(key)?.as_str().map(str::to_owned) +} + +fn package_json_engines_pnpm(package_json: &Value) -> Option { + package_json.get("engines")?.get("pnpm")?.as_str().map(str::to_owned) +} + +fn read_package_json(path: &Path) -> Result { + let contents = fs::read_to_string(path)?; + Ok(serde_json::from_str(&contents)?) +} + +fn is_npm_package_json(package_json: &Value) -> bool { + [ + "bin", + "dependencies", + "devDependencies", + "exports", + "main", + "optionalDependencies", + "packageManager", + "peerDependencies", + "scripts", + "type", + ] + .iter() + .any(|key| package_json.get(key).is_some()) +} + +fn is_template_path(path: &Path) -> bool { + path.starts_with("templates") +} + +fn minimum_release_age(path: &Path) -> Result { + let workspace = fs::read_to_string(path)?; + workspace + .lines() + .find_map(|line| { + let line = line.trim(); + let value = line.strip_prefix("minimumReleaseAge:")?.trim(); + value.parse::().ok() + }) + .ok_or_else(|| anyhow::anyhow!("{} is missing minimumReleaseAge", path.display())) +} + +fn npmrc_minimum_release_age(path: &Path, expected_minimum_release_age: u64) -> Result { + let contents = fs::read_to_string(path).map_err(|err| { + if err.kind() == std::io::ErrorKind::NotFound { + anyhow::anyhow!( + "{} is tracked but missing from the working tree. Restore it with:\nminimum-release-age={}", + path.display(), + expected_minimum_release_age + ) + } else { + anyhow::anyhow!( + "failed to read {} while checking pnpm minimum package age: {err}", + path.display() + ) + } + })?; + contents + .lines() + .find_map(|line| { + let line = line.trim(); + let value = line.strip_prefix("minimum-release-age=")?.trim(); + value.parse::().ok() + }) + .ok_or_else(|| { + anyhow::anyhow!( + "{} must contain `minimum-release-age={}` to match root pnpm-workspace.yaml", + path.display(), + expected_minimum_release_age + ) + }) +} + +fn check_pnpm_release_age_policy() -> Result<()> { + ensure_repo_root()?; + + let root_package_json_path = Path::new("package.json"); + let root_package_json = read_package_json(root_package_json_path)?; + let package_manager = package_json_string_value(&root_package_json, "packageManager") + .ok_or_else(|| anyhow::anyhow!("package.json is missing packageManager"))?; + let package_manager_version = package_json_pnpm_version(&package_manager) + .ok_or_else(|| anyhow::anyhow!("packageManager must be pnpm@, found {package_manager:?}"))?; + + let expected_engine_pnpm = format!(">={package_manager_version}"); + let engine_pnpm = package_json_engines_pnpm(&root_package_json) + .ok_or_else(|| anyhow::anyhow!("package.json engines is missing pnpm"))?; + if engine_pnpm != expected_engine_pnpm { + bail!("package.json engines.pnpm must be {expected_engine_pnpm:?}, found {engine_pnpm:?}"); + } + + for package_json_path in git_tracked_files(":(glob)**/package.json")? { + let package_json = read_package_json(&package_json_path)?; + let Some(found_package_manager) = package_json_string_value(&package_json, "packageManager") else { + continue; + }; + if found_package_manager != package_manager { + bail!( + "{} packageManager must match root package.json: expected {:?}, found {:?}", + package_json_path.display(), + package_manager, + found_package_manager + ); + } + } + + let root_workspace_path = Path::new("pnpm-workspace.yaml"); + let root_minimum_release_age = minimum_release_age(root_workspace_path)?; + for workspace_path in git_tracked_files(":(glob)**/pnpm-workspace.yaml")? { + let found_minimum_release_age = minimum_release_age(&workspace_path)?; + if found_minimum_release_age != root_minimum_release_age { + bail!( + "{} minimumReleaseAge must match root pnpm-workspace.yaml: expected {}, found {}", + workspace_path.display(), + root_minimum_release_age, + found_minimum_release_age + ); + } + } + + for npmrc_path in git_tracked_files(":(glob)**/.npmrc")? { + // Template package roots are copied into projects created by `spacetime init`. + // They must not embed this repo's package-age policy; smoketests enforce it + // at the pnpm process boundary instead. + if is_template_path(&npmrc_path) { + continue; + } + let found_minimum_release_age = npmrc_minimum_release_age(&npmrc_path, root_minimum_release_age)?; + if found_minimum_release_age != root_minimum_release_age { + bail!( + "{} minimum-release-age must match root pnpm-workspace.yaml: expected {}, found {}", + npmrc_path.display(), + root_minimum_release_age, + found_minimum_release_age + ); + } + } + + for package_json_path in git_tracked_files(":(glob)**/package.json")? { + // Template package roots are copied into projects created by `spacetime init`. + // They must not require adjacent .npmrc files for this repo's package-age + // policy; smoketests enforce it at the pnpm process boundary instead. + if is_template_path(&package_json_path) { + continue; + } + let package_json = read_package_json(&package_json_path)?; + if !is_npm_package_json(&package_json) { + continue; + } + let package_dir = package_json_path + .parent() + .expect("git-tracked package.json path should have a parent"); + let npmrc_path = package_dir.join(".npmrc"); + if !npmrc_path.is_file() { + bail!( + "{} is required because {} is an npm/pnpm package manifest.\nAdd {} containing:\nminimum-release-age={}", + npmrc_path.display(), + package_json_path.display(), + npmrc_path.display(), + root_minimum_release_age + ); + } + let found_minimum_release_age = npmrc_minimum_release_age(&npmrc_path, root_minimum_release_age)?; + if found_minimum_release_age != root_minimum_release_age { + bail!( + "{} minimum-release-age must match root pnpm-workspace.yaml: expected {}, found {}", + npmrc_path.display(), + root_minimum_release_age, + found_minimum_release_age + ); + } + } + + for workflow_path in git_tracked_files(".github/workflows/*")? { + let contents = fs::read_to_string(&workflow_path)?; + if contents.contains("pnpm/action-setup@v4") { + bail!( + "{} must use ./.github/actions/setup-pnpm instead of pnpm/action-setup@v4", + workflow_path.display() + ); + } + } + + Ok(()) +} + +/// Codex plugin ships a copy of `skills/`, because plugin installers do not follow symlinks, +/// this checks if the copy is in sync +fn check_codex_plugin_skills_sync() -> Result<()> { + let source = Path::new("skills"); + let copy = Path::new("codex-plugin/plugins/spacetimedb/skills"); + + if fs::symlink_metadata(copy) + .with_context(|| format!("reading {}", copy.display()))? + .file_type() + .is_symlink() + { + bail!( + "{} must be a real directory, not a symlink, plugin installers do not follow symlinks", + copy.display() + ); + } + + fn walk(root: &Path, dir: &Path, out: &mut BTreeSet) -> Result<()> { + for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { + let entry = entry?; + if entry.file_type()?.is_dir() { + walk(root, &entry.path(), out)?; + } else { + let rel = entry + .path() + .strip_prefix(root) + .expect("walked path should be under its root") + .to_path_buf(); + out.insert(rel); + } + } + Ok(()) + } + + let mut source_files = BTreeSet::new(); + walk(source, source, &mut source_files)?; + let mut copy_files = BTreeSet::new(); + walk(copy, copy, &mut copy_files)?; + + let mut drift = Vec::new(); + for rel in &source_files { + if !copy_files.contains(rel) { + drift.push(format!("missing from copy: {}", rel.display())); + } else if fs::read(source.join(rel))? != fs::read(copy.join(rel))? { + drift.push(format!("differs: {}", rel.display())); + } + } + for rel in ©_files { + if !source_files.contains(rel) { + drift.push(format!("extraneous in copy: {}", rel.display())); + } + } + + if !drift.is_empty() { + bail!( + "codex-plugin skills copy is out of sync with skills/:\n {}\nRun: node codex-plugin/scripts/check-skills-sync.ts --fix", + drift.join("\n ") + ); + } + Ok(()) +} + +fn ensure_repo_root() -> Result<()> { + if !Path::new("Cargo.toml").exists() { + bail!("You must execute this command from the SpacetimeDB repository root (where Cargo.toml is located)"); + } + Ok(()) +} + +fn main() -> Result<()> { + env_logger::init(); + check_global_json_policy() +} diff --git a/tools/ci/commands/keynote-bench/Cargo.toml b/tools/ci/commands/keynote-bench/Cargo.toml new file mode 100644 index 00000000000..c14587165e5 --- /dev/null +++ b/tools/ci/commands/keynote-bench/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "ci-keynote-bench" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +keynote-bench-harness = { path = "../../../keynote-bench-harness" } +spacetimedb-guard.workspace = true +env_logger.workspace = true diff --git a/tools/ci/src/keynote_bench.rs b/tools/ci/commands/keynote-bench/src/main.rs similarity index 81% rename from tools/ci/src/keynote_bench.rs rename to tools/ci/commands/keynote-bench/src/main.rs index ed49bdd8185..8fcff0dd769 100644 --- a/tools/ci/src/keynote_bench.rs +++ b/tools/ci/commands/keynote-bench/src/main.rs @@ -2,10 +2,15 @@ use anyhow::Result; use keynote_bench_harness::KeynoteBenchConfig; use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; -pub fn run() -> Result<()> { +fn run() -> Result<()> { let cli_path = ensure_binaries_built(); let server = SpacetimeDbGuard::spawn_in_temp_data_dir(); let server_url = server.host_url.clone(); keynote_bench_harness::run(KeynoteBenchConfig::standalone(".", cli_path, server_url)) } + +fn main() -> Result<()> { + env_logger::init(); + run() +} diff --git a/tools/ci/commands/lint/Cargo.toml b/tools/ci/commands/lint/Cargo.toml new file mode 100644 index 00000000000..c1e456a84ca --- /dev/null +++ b/tools/ci/commands/lint/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "ci-lint" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +clap.workspace = true +regex.workspace = true +serde_json.workspace = true +tempfile.workspace = true +env_logger.workspace = true +log.workspace = true diff --git a/tools/ci/commands/lint/src/main.rs b/tools/ci/commands/lint/src/main.rs new file mode 100644 index 00000000000..190c014cd66 --- /dev/null +++ b/tools/ci/commands/lint/src/main.rs @@ -0,0 +1,407 @@ +#![allow(clippy::disallowed_macros)] +#![allow(dead_code)] + +use anyhow::{bail, Context, Result}; +use duct::{cmd, Expression}; +use serde_json::Value; +use std::collections::BTreeSet; +use std::ffi::OsString; +use std::fs; +use std::path::Path; +use std::path::PathBuf; + +/// On Windows, `pnpm` is installed as a `.cmd` shim which `CreateProcess` cannot +/// find without going through the shell. Wrapping with `cmd /c` fixes this. +/// On Unix, we invoke `pnpm` directly. +fn pnpm(args: I) -> Expression +where + I: IntoIterator, + S: AsRef, +{ + let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); + if cfg!(windows) { + let mut full: Vec = vec!["/c".into(), "pnpm".into()]; + full.extend(args); + cmd("cmd", full) + } else { + cmd("pnpm", args) + } +} + +fn check_global_json_policy() -> Result<()> { + ensure_repo_root()?; + + let root_json = Path::new("global.json"); + let root_contents = fs::read_to_string(root_json)?; + + let globals = git_tracked_files(":(glob)**/global.json")?; + + let mut ok = true; + for p in globals { + let meta = fs::symlink_metadata(&p)?; + let is_symlink = meta.file_type().is_symlink(); + let is_template_global_json = p.strip_prefix(".").unwrap_or(&p).starts_with(Path::new("templates")); + if is_template_global_json && is_symlink { + eprintln!( + "Error: {} is a symlink. Template files must not be symlinks; they are copied literally and this will break if the CLI is built under Windows where symlinks are not supported.", + p.display() + ); + ok = false; + } + + let contents = fs::read_to_string(&p)?; + if contents != root_contents { + eprintln!("Error: {} does not match the root global.json contents", p.display()); + ok = false; + } else if !is_template_global_json || !is_symlink { + println!("OK: {}", p.display()); + } + } + + if !ok { + bail!("global.json policy check failed"); + } + + Ok(()) +} + +fn package_json_pnpm_version(package_manager: &str) -> Option<&str> { + package_manager.strip_prefix("pnpm@") +} + +fn git_tracked_files(pathspec: &str) -> Result> { + let output = cmd!("git", "ls-files", pathspec).read()?; + Ok(output.lines().map(PathBuf::from).collect()) +} + +fn package_json_string_value(package_json: &Value, key: &str) -> Option { + package_json.get(key)?.as_str().map(str::to_owned) +} + +fn package_json_engines_pnpm(package_json: &Value) -> Option { + package_json.get("engines")?.get("pnpm")?.as_str().map(str::to_owned) +} + +fn read_package_json(path: &Path) -> Result { + let contents = fs::read_to_string(path)?; + Ok(serde_json::from_str(&contents)?) +} + +fn is_npm_package_json(package_json: &Value) -> bool { + [ + "bin", + "dependencies", + "devDependencies", + "exports", + "main", + "optionalDependencies", + "packageManager", + "peerDependencies", + "scripts", + "type", + ] + .iter() + .any(|key| package_json.get(key).is_some()) +} + +fn is_template_path(path: &Path) -> bool { + path.starts_with("templates") +} + +fn minimum_release_age(path: &Path) -> Result { + let workspace = fs::read_to_string(path)?; + workspace + .lines() + .find_map(|line| { + let line = line.trim(); + let value = line.strip_prefix("minimumReleaseAge:")?.trim(); + value.parse::().ok() + }) + .ok_or_else(|| anyhow::anyhow!("{} is missing minimumReleaseAge", path.display())) +} + +fn npmrc_minimum_release_age(path: &Path, expected_minimum_release_age: u64) -> Result { + let contents = fs::read_to_string(path).map_err(|err| { + if err.kind() == std::io::ErrorKind::NotFound { + anyhow::anyhow!( + "{} is tracked but missing from the working tree. Restore it with:\nminimum-release-age={}", + path.display(), + expected_minimum_release_age + ) + } else { + anyhow::anyhow!( + "failed to read {} while checking pnpm minimum package age: {err}", + path.display() + ) + } + })?; + contents + .lines() + .find_map(|line| { + let line = line.trim(); + let value = line.strip_prefix("minimum-release-age=")?.trim(); + value.parse::().ok() + }) + .ok_or_else(|| { + anyhow::anyhow!( + "{} must contain `minimum-release-age={}` to match root pnpm-workspace.yaml", + path.display(), + expected_minimum_release_age + ) + }) +} + +fn check_pnpm_release_age_policy() -> Result<()> { + ensure_repo_root()?; + + let root_package_json_path = Path::new("package.json"); + let root_package_json = read_package_json(root_package_json_path)?; + let package_manager = package_json_string_value(&root_package_json, "packageManager") + .ok_or_else(|| anyhow::anyhow!("package.json is missing packageManager"))?; + let package_manager_version = package_json_pnpm_version(&package_manager) + .ok_or_else(|| anyhow::anyhow!("packageManager must be pnpm@, found {package_manager:?}"))?; + + let expected_engine_pnpm = format!(">={package_manager_version}"); + let engine_pnpm = package_json_engines_pnpm(&root_package_json) + .ok_or_else(|| anyhow::anyhow!("package.json engines is missing pnpm"))?; + if engine_pnpm != expected_engine_pnpm { + bail!("package.json engines.pnpm must be {expected_engine_pnpm:?}, found {engine_pnpm:?}"); + } + + for package_json_path in git_tracked_files(":(glob)**/package.json")? { + let package_json = read_package_json(&package_json_path)?; + let Some(found_package_manager) = package_json_string_value(&package_json, "packageManager") else { + continue; + }; + if found_package_manager != package_manager { + bail!( + "{} packageManager must match root package.json: expected {:?}, found {:?}", + package_json_path.display(), + package_manager, + found_package_manager + ); + } + } + + let root_workspace_path = Path::new("pnpm-workspace.yaml"); + let root_minimum_release_age = minimum_release_age(root_workspace_path)?; + for workspace_path in git_tracked_files(":(glob)**/pnpm-workspace.yaml")? { + let found_minimum_release_age = minimum_release_age(&workspace_path)?; + if found_minimum_release_age != root_minimum_release_age { + bail!( + "{} minimumReleaseAge must match root pnpm-workspace.yaml: expected {}, found {}", + workspace_path.display(), + root_minimum_release_age, + found_minimum_release_age + ); + } + } + + for npmrc_path in git_tracked_files(":(glob)**/.npmrc")? { + // Template package roots are copied into projects created by `spacetime init`. + // They must not embed this repo's package-age policy; smoketests enforce it + // at the pnpm process boundary instead. + if is_template_path(&npmrc_path) { + continue; + } + let found_minimum_release_age = npmrc_minimum_release_age(&npmrc_path, root_minimum_release_age)?; + if found_minimum_release_age != root_minimum_release_age { + bail!( + "{} minimum-release-age must match root pnpm-workspace.yaml: expected {}, found {}", + npmrc_path.display(), + root_minimum_release_age, + found_minimum_release_age + ); + } + } + + for package_json_path in git_tracked_files(":(glob)**/package.json")? { + // Template package roots are copied into projects created by `spacetime init`. + // They must not require adjacent .npmrc files for this repo's package-age + // policy; smoketests enforce it at the pnpm process boundary instead. + if is_template_path(&package_json_path) { + continue; + } + let package_json = read_package_json(&package_json_path)?; + if !is_npm_package_json(&package_json) { + continue; + } + let package_dir = package_json_path + .parent() + .expect("git-tracked package.json path should have a parent"); + let npmrc_path = package_dir.join(".npmrc"); + if !npmrc_path.is_file() { + bail!( + "{} is required because {} is an npm/pnpm package manifest.\nAdd {} containing:\nminimum-release-age={}", + npmrc_path.display(), + package_json_path.display(), + npmrc_path.display(), + root_minimum_release_age + ); + } + let found_minimum_release_age = npmrc_minimum_release_age(&npmrc_path, root_minimum_release_age)?; + if found_minimum_release_age != root_minimum_release_age { + bail!( + "{} minimum-release-age must match root pnpm-workspace.yaml: expected {}, found {}", + npmrc_path.display(), + root_minimum_release_age, + found_minimum_release_age + ); + } + } + + for workflow_path in git_tracked_files(".github/workflows/*")? { + let contents = fs::read_to_string(&workflow_path)?; + if contents.contains("pnpm/action-setup@v4") { + bail!( + "{} must use ./.github/actions/setup-pnpm instead of pnpm/action-setup@v4", + workflow_path.display() + ); + } + } + + Ok(()) +} + +/// Codex plugin ships a copy of `skills/`, because plugin installers do not follow symlinks, +/// this checks if the copy is in sync +fn check_codex_plugin_skills_sync() -> Result<()> { + let source = Path::new("skills"); + let copy = Path::new("codex-plugin/plugins/spacetimedb/skills"); + + if fs::symlink_metadata(copy) + .with_context(|| format!("reading {}", copy.display()))? + .file_type() + .is_symlink() + { + bail!( + "{} must be a real directory, not a symlink, plugin installers do not follow symlinks", + copy.display() + ); + } + + fn walk(root: &Path, dir: &Path, out: &mut BTreeSet) -> Result<()> { + for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { + let entry = entry?; + if entry.file_type()?.is_dir() { + walk(root, &entry.path(), out)?; + } else { + let rel = entry + .path() + .strip_prefix(root) + .expect("walked path should be under its root") + .to_path_buf(); + out.insert(rel); + } + } + Ok(()) + } + + let mut source_files = BTreeSet::new(); + walk(source, source, &mut source_files)?; + let mut copy_files = BTreeSet::new(); + walk(copy, copy, &mut copy_files)?; + + let mut drift = Vec::new(); + for rel in &source_files { + if !copy_files.contains(rel) { + drift.push(format!("missing from copy: {}", rel.display())); + } else if fs::read(source.join(rel))? != fs::read(copy.join(rel))? { + drift.push(format!("differs: {}", rel.display())); + } + } + for rel in ©_files { + if !source_files.contains(rel) { + drift.push(format!("extraneous in copy: {}", rel.display())); + } + } + + if !drift.is_empty() { + bail!( + "codex-plugin skills copy is out of sync with skills/:\n {}\nRun: node codex-plugin/scripts/check-skills-sync.ts --fix", + drift.join("\n ") + ); + } + Ok(()) +} + +fn ensure_repo_root() -> Result<()> { + if !Path::new("Cargo.toml").exists() { + bail!("You must execute this command from the SpacetimeDB repository root (where Cargo.toml is located)"); + } + Ok(()) +} +fn tracked_rs_files_under(path: &str) -> Result> { + let output = cmd!("git", "ls-files", "--", path).read()?; + Ok(output + .lines() + .filter(|line| line.ends_with(".rs")) + .map(PathBuf::from) + .collect()) +} + +fn main() -> Result<()> { + env_logger::init(); + ensure_repo_root()?; + check_pnpm_release_age_policy()?; + check_codex_plugin_skills_sync()?; + // `cargo fmt --all` only checks files that Cargo discovers through workspace/package targets. + // However, we also keep Rust sources in a locations that are tracked but not part of our workspace, + // so this approach properly catches all the files, where `cargo fmt` does not. + let mut files = Vec::new(); + files.extend(tracked_rs_files_under(".")?); + const RUSTFMT_BATCH_SIZE: usize = 200; + for batch in files.chunks(RUSTFMT_BATCH_SIZE) { + let mut args = Vec::::with_capacity(batch.len() + 1); + args.push("--check".into()); + args.extend(batch.iter().map(|path| path.as_os_str().to_os_string())); + cmd("rustfmt", args).run()?; + } + cmd!( + "cargo", + "clippy", + "--timings", + "--all", + "--tests", + "--benches", + "--", + "-D", + "warnings", + ) + .run()?; + cmd!( + "cargo", + "clippy", + "--timings", + "--no-default-features", + "--features=browser", + "-pspacetimedb-sdk", + "--tests", + "--benches", + "--", + "-D", + "warnings", + ) + .run()?; + cmd!("dotnet", "tool", "restore").dir("crates/bindings-csharp").run()?; + cmd!("dotnet", "csharpier", "--check", ".") + .dir("crates/bindings-csharp") + .run()?; + pnpm(["lint"]).run()?; + cmd!("cargo", "test", "--doc", "--target", "wasm32-unknown-unknown") + .dir("crates/bindings") + .run()?; + cmd!("cargo", "test", "--doc").dir("crates/bindings").run()?; + // `bindings` is the only crate we care strongly about documenting, + // since we link to its docs.rs from our website. + // We won't pass `--no-deps`, though, + // since we want everything reachable through it to also work. + // This includes `sats` and `lib`. + cmd!("cargo", "doc") + .dir("crates/bindings") + // Make `cargo doc` exit with error on warnings, most notably broken links + .env("RUSTDOCFLAGS", "--deny warnings") + .run()?; + + Ok(()) +} diff --git a/tools/ci/commands/other-workflows/Cargo.toml b/tools/ci/commands/other-workflows/Cargo.toml new file mode 100644 index 00000000000..543f7702f66 --- /dev/null +++ b/tools/ci/commands/other-workflows/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "ci-other-workflows" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +clap.workspace = true +regex.workspace = true +reqwest = { workspace = true, features = ["blocking", "json"] } +serde.workspace = true +serde_json.workspace = true +env_logger.workspace = true diff --git a/tools/ci/src/cla_assistant.rs b/tools/ci/commands/other-workflows/src/cla_assistant.rs similarity index 95% rename from tools/ci/src/cla_assistant.rs rename to tools/ci/commands/other-workflows/src/cla_assistant.rs index 065db03ef70..29b148be5e7 100644 --- a/tools/ci/src/cla_assistant.rs +++ b/tools/ci/commands/other-workflows/src/cla_assistant.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; const CLA_CONTEXT: &str = "license/cla"; #[derive(Subcommand)] -pub(crate) enum ClaAssistantCmd { +pub enum ClaAssistantCmd { /// Retries CLA Assistant if `license/cla` is the only remaining PR blocker. Retry(RetryArgs), @@ -20,14 +20,14 @@ pub(crate) enum ClaAssistantCmd { } #[derive(Args)] -pub(crate) struct RetryArgs { +pub struct RetryArgs { /// Pull request number to check. #[arg(long)] - pub(crate) pr_number: u64, + pub pr_number: u64, /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. #[arg(long)] - pub(crate) repo: Option, + pub repo: Option, } #[derive(Args)] @@ -37,21 +37,21 @@ pub(crate) struct RetryArgs { .multiple(false) .args(["pr", "sha"]), ))] -pub(crate) struct StatusArgs { +pub struct StatusArgs { /// Pull request number whose head commit should be checked. #[arg(long)] - pub(crate) pr: Option, + pub pr: Option, /// Commit SHA to check. #[arg(long)] - pub(crate) sha: Option, + pub sha: Option, /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. #[arg(long)] - pub(crate) repo: Option, + pub repo: Option, } -pub(crate) fn run(cmd: ClaAssistantCmd) -> Result<()> { +pub fn run(cmd: ClaAssistantCmd) -> Result<()> { match cmd { ClaAssistantCmd::Retry(args) => retry(args), ClaAssistantCmd::Status(args) => status(args), diff --git a/tools/ci/src/codeowners_check.rs b/tools/ci/commands/other-workflows/src/codeowners_check.rs similarity index 99% rename from tools/ci/src/codeowners_check.rs rename to tools/ci/commands/other-workflows/src/codeowners_check.rs index 006d597cf87..f456100b9dc 100644 --- a/tools/ci/src/codeowners_check.rs +++ b/tools/ci/commands/other-workflows/src/codeowners_check.rs @@ -1,3 +1,4 @@ +use crate::ensure_repo_root; use anyhow::{anyhow, bail, Context, Result}; use duct::cmd; use serde_json::Value; @@ -7,7 +8,7 @@ use std::path::{Path, PathBuf}; const REPO: &str = "clockworklabs/SpacetimeDB"; pub fn run(base_ref: &str, pr_number: u64) -> Result<()> { - super::ensure_repo_root()?; + ensure_repo_root()?; fetch_base_ref(base_ref)?; let review = ReviewStatus::fetch(pr_number)?; diff --git a/tools/ci/commands/other-workflows/src/main.rs b/tools/ci/commands/other-workflows/src/main.rs new file mode 100644 index 00000000000..9dd466e41e2 --- /dev/null +++ b/tools/ci/commands/other-workflows/src/main.rs @@ -0,0 +1,42 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::{bail, Result}; +use clap::{Parser, Subcommand}; +use std::path::Path; + +fn ensure_repo_root() -> Result<()> { + if !Path::new("Cargo.toml").exists() { + bail!("You must execute this command from the SpacetimeDB repository root (where Cargo.toml is located)"); + } + Ok(()) +} + +mod cla_assistant; +mod codeowners_check; + +#[derive(Parser)] +struct Args { + #[command(subcommand)] + cmd: OtherWorkflowsCmd, +} +#[derive(Subcommand)] +enum OtherWorkflowsCmd { + /// Checks that sensitive CODEOWNERS-controlled files have the required approvals. + CodeownersCheck { + #[arg(long)] + base_ref: String, + #[arg(long)] + pr_number: u64, + }, + /// Interacts with CLA Assistant. + ClaAssistant { + #[command(subcommand)] + cmd: cla_assistant::ClaAssistantCmd, + }, +} +fn main() -> Result<()> { + env_logger::init(); + match Args::parse().cmd { + OtherWorkflowsCmd::CodeownersCheck { base_ref, pr_number } => codeowners_check::run(&base_ref, pr_number), + OtherWorkflowsCmd::ClaAssistant { cmd } => cla_assistant::run(cmd), + } +} diff --git a/tools/ci/commands/publish-checks/Cargo.toml b/tools/ci/commands/publish-checks/Cargo.toml new file mode 100644 index 00000000000..67093422620 --- /dev/null +++ b/tools/ci/commands/publish-checks/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "ci-publish-checks" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +env_logger.workspace = true diff --git a/tools/ci/commands/publish-checks/src/main.rs b/tools/ci/commands/publish-checks/src/main.rs new file mode 100644 index 00000000000..83f6315d4b2 --- /dev/null +++ b/tools/ci/commands/publish-checks/src/main.rs @@ -0,0 +1,37 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::{bail, Result}; +use duct::cmd; +fn run_publish_checks() -> Result<()> { + cmd!("bash", "-lc", "test -d venv || python3 -m venv venv").run()?; + cmd!("venv/bin/pip3", "install", "argparse", "toml").run()?; + + let crates = cmd!( + "venv/bin/python3", + "tools/find-publish-list.py", + "--recursive", + "--directories", + "--quiet", + "spacetimedb", + "spacetimedb-sdk" + ) + .read()?; + + let mut failed = Vec::new(); + for crate_dir in crates.split_whitespace() { + if let Err(err) = cmd!("venv/bin/python3", "tools/crate-publish-checks.py", crate_dir).run() { + eprintln!("crate publish checks failed for {crate_dir}: {err}"); + failed.push(crate_dir.to_string()); + } + } + + if !failed.is_empty() { + bail!("crate publish checks failed for: {}", failed.join(", ")); + } + + Ok(()) +} + +fn main() -> Result<()> { + env_logger::init(); + run_publish_checks() +} diff --git a/tools/ci/commands/self-docs/Cargo.toml b/tools/ci/commands/self-docs/Cargo.toml new file mode 100644 index 00000000000..382aed26d67 --- /dev/null +++ b/tools/ci/commands/self-docs/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "ci-self-docs" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +clap.workspace = true diff --git a/tools/ci/commands/self-docs/src/main.rs b/tools/ci/commands/self-docs/src/main.rs new file mode 100644 index 00000000000..90cc263baf4 --- /dev/null +++ b/tools/ci/commands/self-docs/src/main.rs @@ -0,0 +1,137 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::{bail, Result}; +use clap::Parser; +use duct::cmd; +use std::fs; +use std::path::Path; + +#[derive(Parser)] +struct Args { + #[arg( + long, + default_value_t = false, + long_help = "Only check for changes, do not generate the docs" + )] + check: bool, +} +struct CommandDoc { + name: &'static str, + package: &'static str, + help_safe: bool, +} + +const COMMANDS: &[CommandDoc] = &[ + CommandDoc { + name: "test", + package: "ci-test", + help_safe: false, + }, + CommandDoc { + name: "lint", + package: "ci-lint", + help_safe: false, + }, + CommandDoc { + name: "wasm-bindings", + package: "ci-wasm-bindings", + help_safe: false, + }, + CommandDoc { + name: "dlls", + package: "ci-dlls", + help_safe: false, + }, + CommandDoc { + name: "smoketests", + package: "ci-smoketests", + help_safe: true, + }, + CommandDoc { + name: "keynote-bench", + package: "ci-keynote-bench", + help_safe: false, + }, + CommandDoc { + name: "update-flow", + package: "ci-update-flow", + help_safe: true, + }, + CommandDoc { + name: "cli-docs", + package: "ci-cli-docs", + help_safe: true, + }, + CommandDoc { + name: "self-docs", + package: "ci-self-docs", + help_safe: true, + }, + CommandDoc { + name: "global-json-policy", + package: "ci-global-json-policy", + help_safe: false, + }, + CommandDoc { + name: "publish-checks", + package: "ci-publish-checks", + help_safe: false, + }, + CommandDoc { + name: "typescript-test", + package: "ci-typescript-test", + help_safe: false, + }, + CommandDoc { + name: "version-upgrade-check", + package: "ci-version-upgrade-check", + help_safe: false, + }, + CommandDoc { + name: "docs", + package: "ci-docs-build", + help_safe: false, + }, + CommandDoc { + name: "other-workflows", + package: "ci-other-workflows", + help_safe: true, + }, +]; + +fn normalize_help(help: &str) -> String { + help.lines().map(str::trim_end).collect::>().join("\n") +} + +fn command_usage(command: &CommandDoc) -> Result { + if command.help_safe { + let help = cmd!("cargo", "run", "--quiet", "--package", command.package, "--", "--help").read()?; + return Ok(normalize_help(&help)); + } + Ok(format!("cargo ci {}", command.name)) +} +fn generate() -> Result { + let mut out = String::from("# SpacetimeDB's cargo ci\n\n## Overview\n\nThis document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options.\n\n## `cargo ci`\n\n**Usage:**\n```bash\ncargo ci [--skip ] [COMMAND] [ARGS]...\n```\n\n"); + for command in COMMANDS { + out.push_str(&format!( + "### `{}`\n\n**Usage:**\n```bash\n{}\n```\n\n", + command.name, + command_usage(command)?.trim(), + )); + } + out.push_str("---\n\nThis document is auto-generated by running:\n\n```bash\ncargo ci self-docs\n```\n"); + Ok(out) +} +fn main() -> Result<()> { + let args = Args::parse(); + let readme = generate()?; + let path = Path::new("tools/ci/README.md"); + if args.check { + let existing = fs::read_to_string(path).unwrap_or_default(); + if existing != readme { + bail!("README.md is out of date. Please run `cargo ci self-docs` to update it."); + } + } else { + fs::write(path, readme)?; + } + Ok(()) +} diff --git a/tools/ci/commands/smoketests/Cargo.toml b/tools/ci/commands/smoketests/Cargo.toml new file mode 100644 index 00000000000..3fe57943e37 --- /dev/null +++ b/tools/ci/commands/smoketests/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ci-smoketests" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +clap.workspace = true +tempfile.workspace = true +spacetimedb-guard.workspace = true +env_logger.workspace = true diff --git a/tools/ci/src/smoketest.rs b/tools/ci/commands/smoketests/src/main.rs similarity index 96% rename from tools/ci/src/smoketest.rs rename to tools/ci/commands/smoketests/src/main.rs index 2a35b91ca59..7e271a0c95d 100644 --- a/tools/ci/src/smoketest.rs +++ b/tools/ci/commands/smoketests/src/main.rs @@ -1,21 +1,26 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, ensure, Context, Result}; -use clap::{Args, Subcommand}; +use clap::{Parser, Subcommand}; use duct::cmd; use spacetimedb_guard::ensure_binaries_built; use std::ffi::OsStr; use std::path::Path; use std::process::{Command, Stdio}; use std::{env, fs}; -use tempfile::TempDir; -use crate::util; +fn ensure_repo_root() -> Result<()> { + if !Path::new("Cargo.toml").exists() { + bail!("You must execute this command from the SpacetimeDB repository root (where Cargo.toml is located)"); + } + Ok(()) +} +use tempfile::TempDir; -#[derive(Args)] +#[derive(Parser)] /// This command builds the binaries needed by the smoketests, then runs them. This prevents /// race conditions when running tests in parallel with nextest, where multiple test processes /// might try to build the same binaries simultaneously. -pub struct SmoketestsArgs { +struct SmoketestsArgs { #[command(subcommand)] cmd: Option, @@ -53,7 +58,7 @@ enum SmoketestCmd { CheckModList, } -pub fn run(args: SmoketestsArgs) -> Result<()> { +fn run(args: SmoketestsArgs) -> Result<()> { match args.cmd { Some(SmoketestCmd::Prepare) => { build_cli()?; @@ -292,7 +297,7 @@ fn set_env(cmd: &mut Command, server: Option, dotnet: bool, auth_host: b } fn check_smoketests_mod_rs_complete() -> Result<()> { - util::ensure_repo_root()?; + ensure_repo_root()?; let expected_dir = Path::new("crates/smoketests/tests/smoketests"); let mut expected = std::collections::BTreeSet::::new(); @@ -352,3 +357,8 @@ fn check_smoketests_mod_rs_complete() -> Result<()> { Ok(()) } + +fn main() -> Result<()> { + env_logger::init(); + run(SmoketestsArgs::parse()) +} diff --git a/tools/ci/commands/test/Cargo.toml b/tools/ci/commands/test/Cargo.toml new file mode 100644 index 00000000000..e714cc869fc --- /dev/null +++ b/tools/ci/commands/test/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "ci-test" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +env_logger.workspace = true diff --git a/tools/ci/commands/test/src/main.rs b/tools/ci/commands/test/src/main.rs new file mode 100644 index 00000000000..117d937e4ae --- /dev/null +++ b/tools/ci/commands/test/src/main.rs @@ -0,0 +1,131 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::Result; +use duct::{cmd, Expression}; +use std::ffi::OsStr; +fn pnpm(args: I) -> Expression +where + I: IntoIterator, + S: AsRef, +{ + let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); + if cfg!(windows) { + let mut full: Vec = vec!["/c".into(), "pnpm".into()]; + full.extend(args); + cmd("cmd", full) + } else { + cmd("pnpm", args) + } +} +fn main() -> Result<()> { + env_logger::init(); + pnpm(["build"]).dir("crates/bindings-typescript").run()?; + + // TODO: This doesn't work on at least user Linux machines, because something here apparently uses `sudo`? + + // Exclude smoketests from `cargo test --all` since they require pre-built binaries. + // Smoketests have their own dedicated command: `cargo ci smoketests` + cmd!( + "cargo", + "test", + "--all", + "--exclude", + "spacetimedb-smoketests", + "--exclude", + "spacetimedb-sdk", + "--exclude", + "spacetimedb", + "--", + "--test-threads=2", + "--skip", + "unreal" + ) + .run()?; + // Bindings snapshot tests rely on the unstable feature, + // as they compile and test APIs which are gated behind that feature, + // e.g. procedures, HTTP handlers. + cmd!( + "cargo", + "test", + "-p", + "spacetimedb", + "--features", + "unstable", + "--", + "--test-threads=2", + ) + .run()?; + // The SDK test harness uses the same child-process server guard as smoketests, + // which expects release CLI/standalone binaries to already exist. + cmd!( + "cargo", + "build", + "--release", + "-p", + "spacetimedb-cli", + "-p", + "spacetimedb-standalone", + "--features", + "spacetimedb-standalone/allow_loopback_http_for_tests", + ) + .run()?; + // SDK procedure tests intentionally make localhost HTTP requests. + cmd!( + "cargo", + "test", + "-p", + "spacetimedb-sdk", + "--features", + "allow_loopback_http_for_tests", + "--", + "--test-threads=2", + "--skip", + "unreal" + ) + .run()?; + // Run the same SDK suite against wasm/browser test clients. + cmd!( + "cargo", + "test", + "-p", + "spacetimedb-sdk", + "--features", + "allow_loopback_http_for_tests,browser", + "--", + "--test-threads=2", + "--skip", + "unreal" + ) + .run()?; + // TODO: This should check for a diff at the start. If there is one, we should alert the user + // that we're disabling diff checks because they have a dirty git repo, and to re-run in a clean one + // if they want those checks. + + // The fallocate tests have been flakely when running in parallel + cmd!( + "cargo", + "test", + "-p", + "spacetimedb-durability", + "--features", + "fallocate", + "--", + "--test-threads=1", + ) + .run()?; + cmd!("bash", "tools/check-diff.sh").run()?; + cmd!( + "cargo", + "run", + "-p", + "spacetimedb-codegen", + "--example", + "regen-csharp-moduledef", + ) + .run()?; + cmd!("bash", "tools/check-diff.sh", "crates/bindings-csharp").run()?; + cmd!("dotnet", "test", "-warnaserror") + .dir("crates/bindings-csharp") + .run()?; + + Ok(()) +} diff --git a/tools/ci/commands/typescript-test/Cargo.toml b/tools/ci/commands/typescript-test/Cargo.toml new file mode 100644 index 00000000000..565466bcbe2 --- /dev/null +++ b/tools/ci/commands/typescript-test/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "ci-typescript-test" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +env_logger.workspace = true diff --git a/tools/ci/commands/typescript-test/src/main.rs b/tools/ci/commands/typescript-test/src/main.rs new file mode 100644 index 00000000000..74dbe2c7770 --- /dev/null +++ b/tools/ci/commands/typescript-test/src/main.rs @@ -0,0 +1,45 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::{bail, Result}; +use duct::{cmd, Expression}; +use std::ffi::OsStr; +fn pnpm(args: I) -> Expression +where + I: IntoIterator, + S: AsRef, +{ + let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); + if cfg!(windows) { + let mut full: Vec = vec!["/c".into(), "pnpm".into()]; + full.extend(args); + cmd("cmd", full) + } else { + cmd("pnpm", args) + } +} +fn run_typescript_tests() -> Result<()> { + pnpm(["build"]).dir("crates/bindings-typescript").run()?; + pnpm(["test"]).dir("crates/bindings-typescript").run()?; + pnpm(["generate"]).dir("templates/chat-react-ts").run()?; + let diff_status = cmd!( + "bash", + "tools/check-diff.sh", + "templates/chat-react-ts/src/module_bindings" + ) + .run()?; + if !diff_status.status.success() { + bail!("Bindings are dirty. Please generate bindings again and commit them to this branch."); + } + pnpm(["build"]).dir("templates/chat-react-ts").run()?; + pnpm(["-r", "--filter", "./**", "run", "build"]) + .dir("templates") + .run()?; + pnpm(["-r", "--filter", "./**", "run", "build"]) + .dir("crates/bindings-typescript") + .run()?; + Ok(()) +} + +fn main() -> Result<()> { + env_logger::init(); + run_typescript_tests() +} diff --git a/tools/ci/commands/update-flow/Cargo.toml b/tools/ci/commands/update-flow/Cargo.toml new file mode 100644 index 00000000000..e7ce2e6b461 --- /dev/null +++ b/tools/ci/commands/update-flow/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ci-update-flow" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +tempfile.workspace = true +clap.workspace = true +log.workspace = true +env_logger.workspace = true diff --git a/tools/ci/commands/update-flow/src/main.rs b/tools/ci/commands/update-flow/src/main.rs new file mode 100644 index 00000000000..8a809c42feb --- /dev/null +++ b/tools/ci/commands/update-flow/src/main.rs @@ -0,0 +1,69 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::Result; +use clap::Parser; +use duct::cmd; + +#[derive(Parser)] +struct Args { + #[arg( + long, + long_help = "Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms." + )] + target: Option, + #[arg( + long, + default_value = "false", + long_help = "Whether to enable github token authentication feature when building the update binary. By default this is disabled." + )] + github_token_auth: bool, +} + +fn main() -> Result<()> { + env_logger::init(); + let Args { + target, + github_token_auth, + } = Args::parse(); + let mut common_args = vec![]; + if let Some(target) = target.as_ref() { + common_args.push("--target"); + common_args.push(target); + log::info!("checking update flow for target: {target}"); + } else { + log::info!("checking update flow"); + } + if github_token_auth { + common_args.push("--features"); + common_args.push("github-token-auth"); + } + + cmd( + "cargo", + ["build", "-p", "spacetimedb-update"] + .into_iter() + .chain(common_args.clone()), + ) + .run()?; + // NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`. + // My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it + // happens very frequently on the `macos-runner`, but we haven't seen it on any others). + let root_dir = tempfile::tempdir()?; + let root_dir_string = root_dir.path().to_string_lossy().to_string(); + let root_arg = format!("--root-dir={}", root_dir_string); + cmd( + "cargo", + ["run", "-p", "spacetimedb-update"] + .into_iter() + .chain(common_args.clone()) + .chain(["--", "self-install", &root_arg, "--yes"].into_iter()), + ) + .run()?; + + let mut spacetime_path = root_dir.path().join("spacetime"); + if !std::env::consts::EXE_EXTENSION.is_empty() { + spacetime_path.set_extension(std::env::consts::EXE_EXTENSION); + } + cmd(spacetime_path, [&root_arg, "help"]).run()?; + + Ok(()) +} diff --git a/tools/ci/commands/version-upgrade-check/Cargo.toml b/tools/ci/commands/version-upgrade-check/Cargo.toml new file mode 100644 index 00000000000..68dec2068e4 --- /dev/null +++ b/tools/ci/commands/version-upgrade-check/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "ci-version-upgrade-check" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +env_logger.workspace = true diff --git a/tools/ci/commands/version-upgrade-check/src/main.rs b/tools/ci/commands/version-upgrade-check/src/main.rs new file mode 100644 index 00000000000..fed14c6bed3 --- /dev/null +++ b/tools/ci/commands/version-upgrade-check/src/main.rs @@ -0,0 +1,22 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::Result; +use duct::cmd; +fn run_version_upgrade_check() -> Result<()> { + cmd!( + "cargo", + "bump-versions", + "123.456.789", + "--rust-and-cli", + "--csharp", + "--typescript", + "--cpp", + "--accept-snapshots" + ) + .run()?; + Ok(()) +} + +fn main() -> Result<()> { + env_logger::init(); + run_version_upgrade_check() +} diff --git a/tools/ci/commands/wasm-bindings/Cargo.toml b/tools/ci/commands/wasm-bindings/Cargo.toml new file mode 100644 index 00000000000..be9456b7754 --- /dev/null +++ b/tools/ci/commands/wasm-bindings/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "ci-wasm-bindings" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +env_logger.workspace = true diff --git a/tools/ci/commands/wasm-bindings/src/main.rs b/tools/ci/commands/wasm-bindings/src/main.rs new file mode 100644 index 00000000000..1f708d7341e --- /dev/null +++ b/tools/ci/commands/wasm-bindings/src/main.rs @@ -0,0 +1,50 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::Result; +use duct::{cmd, Expression}; +use std::ffi::OsStr; +fn pnpm(args: I) -> Expression +where + I: IntoIterator, + S: AsRef, +{ + let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); + if cfg!(windows) { + let mut full: Vec = vec!["/c".into(), "pnpm".into()]; + full.extend(args); + cmd("cmd", full) + } else { + cmd("pnpm", args) + } +} +use std::path::Path; +fn main() -> Result<()> { + env_logger::init(); + pnpm([ + "install", + "--filter", + "./crates/bindings-typescript...", + "--filter", + "./modules/module-test-ts...", + ]) + .run()?; + pnpm(["build"]).dir("crates/bindings-typescript").run()?; + cmd!("cargo", "test", "-p", "spacetimedb-codegen").run()?; + // Pre-build the CLI so that it _doesn't_ get `cargo update`d, since that may break the build. + cmd!("cargo", "build", "-p", "spacetimedb-cli").run()?; + // Make sure the `Cargo.lock` file reflects the latest available versions. + // This is what users would end up with on a fresh module, so we want to + // catch any compile errors arising from a different transitive closure + // of dependencies than what is in the workspace lock file. + // + // For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714 + cmd!("cargo", "update").run()?; + let cli_path = Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(2) + .unwrap() + .join("target/debug/spacetimedb-cli") + .with_extension(std::env::consts::EXE_EXTENSION); + cmd!(cli_path, "build", "--module-path", "modules/module-test",).run()?; + + Ok(()) +} diff --git a/tools/ci/common/Cargo.toml b/tools/ci/common/Cargo.toml new file mode 100644 index 00000000000..f4e1d4195e4 --- /dev/null +++ b/tools/ci/common/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "ci-common" +version = "0.1.0" +edition.workspace = true + +[dependencies] diff --git a/tools/ci/common/src/lib.rs b/tools/ci/common/src/lib.rs new file mode 100644 index 00000000000..cdb99009889 --- /dev/null +++ b/tools/ci/common/src/lib.rs @@ -0,0 +1,10 @@ +use std::path::PathBuf; + +pub fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(|p| p.parent()) + .and_then(|p| p.parent()) + .expect("failed to find repo root") + .to_path_buf() +} diff --git a/tools/ci/src/ci_docs.rs b/tools/ci/src/ci_docs.rs deleted file mode 100644 index 3fadcdaf0cc..00000000000 --- a/tools/ci/src/ci_docs.rs +++ /dev/null @@ -1,85 +0,0 @@ -use clap::{Command, CommandFactory}; - -use crate::Cli; - -// TODO: use clap_markdown instead of this custom implementation in the future -pub fn generate_cli_docs() -> String { - let mut cli = Cli::command(); - let usage = generate_markdown(&mut cli, 2); - - format!( - "\ -# SpacetimeDB's cargo ci - -## Overview - -This document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options. - -{} ---- - -This document is auto-generated by running: - -```bash -cargo ci self-docs -```", - usage - ) -} - -fn generate_markdown(cmd: &mut Command, heading_level: usize) -> String { - let mut out = String::new(); - - let heading = "#".repeat(heading_level); - out.push_str(&format!("{} `{}`\n\n", heading, cmd.get_name())); - - if let Some(long_about) = cmd.get_long_about() { - out.push_str(&format!("{}\n\n", long_about)); - } - - out.push_str(&format!("**Usage:**\n```bash\n{}\n```\n\n", cmd.render_usage())); - - let mut options = String::new(); - for arg in cmd.get_arguments() { - let mut names = arg - .get_long() - .map(|l| format!("--{}", l)) - .or_else(|| arg.get_short().map(|s| format!("-{}", s))) - .unwrap_or_else(|| arg.get_id().to_string()); - if let Some(value_names) = arg.get_value_names().filter(|_| arg.get_action().takes_values()) { - for value_name in value_names { - names.push_str(&format!(" <{value_name}>")); - } - } - let help = arg - .get_long_help() - .or_else(|| arg.get_help()) - .map(|help| help.to_string()) - .unwrap_or_else(|| { - eprintln!("Warning: argument `{}` is missing help text", arg.get_id()); - "".to_string() - }); - if help.is_empty() { - options.push_str(&format!("- `{names}`:\n")); - } else { - options.push_str(&format!( - "- `{}`: {}\n{}", - names, - help, - if help.lines().count() > 1 { "\n" } else { "" } - )); - } - } - - if !options.is_empty() { - out.push_str("**Options:**\n\n"); - out.push_str(&options); - out.push('\n'); - } - - for sub in cmd.get_subcommands_mut() { - out.push_str(&generate_markdown(sub, heading_level + 1)); - } - - out -} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 41d91e19a8d..a41a1bb3d02 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,907 +1,122 @@ #![allow(clippy::disallowed_macros)] +use anyhow::{bail, Result}; +use duct::cmd; -use anyhow::{bail, Context, Result}; -use clap::{CommandFactory, Parser, Subcommand}; -use duct::{cmd, Expression}; -use serde_json::Value; -use std::collections::BTreeSet; -use std::ffi::OsString; -use std::path::Path; -use std::path::PathBuf; -use std::{env, fs}; - -const README_PATH: &str = "tools/ci/README.md"; - -mod ci_docs; -mod cla_assistant; -mod codeowners_check; -mod keynote_bench; -mod smoketest; -mod util; - -use util::ensure_repo_root; - -/// On Windows, `pnpm` is installed as a `.cmd` shim which `CreateProcess` cannot -/// find without going through the shell. Wrapping with `cmd /c` fixes this. -/// On Unix, we invoke `pnpm` directly. -fn pnpm(args: I) -> Expression -where - I: IntoIterator, - S: AsRef, -{ - let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); - if cfg!(windows) { - let mut full: Vec = vec!["/c".into(), "pnpm".into()]; - full.extend(args); - cmd("cmd", full) - } else { - cmd("pnpm", args) - } -} - -/// SpacetimeDB CI tasks -/// -/// This tool provides several subcommands for automating CI workflows in SpacetimeDB. -/// -/// It may be invoked via `cargo ci `, or simply `cargo ci` to run all subcommands in -/// sequence. It is mostly designed to be run in CI environments via the github workflows, but can -/// also be run locally -#[derive(Parser)] -#[command(name = "cargo ci", subcommand_required = false, arg_required_else_help = false)] -struct Cli { - #[command(subcommand)] - cmd: Option, - - /// Skip specified subcommands when running all - /// - /// When no subcommand is specified, all subcommands are run in sequence. This option allows - /// specifying subcommands to skip when running all. For example, to skip the `unreal-tests` - /// subcommand, use `--skip unreal-tests`. - #[arg(long, default_value = "other-workflows")] - skip: Vec, -} - -fn check_global_json_policy() -> Result<()> { - ensure_repo_root()?; - - let root_json = Path::new("global.json"); - let root_contents = fs::read_to_string(root_json)?; - - let globals = git_tracked_files(":(glob)**/global.json")?; - - let mut ok = true; - for p in globals { - let meta = fs::symlink_metadata(&p)?; - let is_symlink = meta.file_type().is_symlink(); - let is_template_global_json = p.strip_prefix(".").unwrap_or(&p).starts_with(Path::new("templates")); - if is_template_global_json && is_symlink { - eprintln!( - "Error: {} is a symlink. Template files must not be symlinks; they are copied literally and this will break if the CLI is built under Windows where symlinks are not supported.", - p.display() - ); - ok = false; - } - - let contents = fs::read_to_string(&p)?; - if contents != root_contents { - eprintln!("Error: {} does not match the root global.json contents", p.display()); - ok = false; - } else if !is_template_global_json || !is_symlink { - println!("OK: {}", p.display()); - } - } - - if !ok { - bail!("global.json policy check failed"); - } - - Ok(()) +struct Command { + name: &'static str, + package: &'static str, } - -fn package_json_pnpm_version(package_manager: &str) -> Option<&str> { - package_manager.strip_prefix("pnpm@") -} - -fn git_tracked_files(pathspec: &str) -> Result> { - let output = cmd!("git", "ls-files", pathspec).read()?; - Ok(output.lines().map(PathBuf::from).collect()) -} - -fn package_json_string_value(package_json: &Value, key: &str) -> Option { - package_json.get(key)?.as_str().map(str::to_owned) -} - -fn package_json_engines_pnpm(package_json: &Value) -> Option { - package_json.get("engines")?.get("pnpm")?.as_str().map(str::to_owned) -} - -fn read_package_json(path: &Path) -> Result { - let contents = fs::read_to_string(path)?; - Ok(serde_json::from_str(&contents)?) -} - -fn is_npm_package_json(package_json: &Value) -> bool { - [ - "bin", - "dependencies", - "devDependencies", - "exports", - "main", - "optionalDependencies", - "packageManager", - "peerDependencies", - "scripts", - "type", - ] - .iter() - .any(|key| package_json.get(key).is_some()) -} - -fn is_template_path(path: &Path) -> bool { - path.starts_with("templates") -} - -fn minimum_release_age(path: &Path) -> Result { - let workspace = fs::read_to_string(path)?; - workspace - .lines() - .find_map(|line| { - let line = line.trim(); - let value = line.strip_prefix("minimumReleaseAge:")?.trim(); - value.parse::().ok() - }) - .ok_or_else(|| anyhow::anyhow!("{} is missing minimumReleaseAge", path.display())) -} - -fn npmrc_minimum_release_age(path: &Path, expected_minimum_release_age: u64) -> Result { - let contents = fs::read_to_string(path).map_err(|err| { - if err.kind() == std::io::ErrorKind::NotFound { - anyhow::anyhow!( - "{} is tracked but missing from the working tree. Restore it with:\nminimum-release-age={}", - path.display(), - expected_minimum_release_age - ) - } else { - anyhow::anyhow!( - "failed to read {} while checking pnpm minimum package age: {err}", - path.display() - ) - } - })?; - contents - .lines() - .find_map(|line| { - let line = line.trim(); - let value = line.strip_prefix("minimum-release-age=")?.trim(); - value.parse::().ok() - }) - .ok_or_else(|| { - anyhow::anyhow!( - "{} must contain `minimum-release-age={}` to match root pnpm-workspace.yaml", - path.display(), - expected_minimum_release_age - ) - }) -} - -fn check_pnpm_release_age_policy() -> Result<()> { - ensure_repo_root()?; - - let root_package_json_path = Path::new("package.json"); - let root_package_json = read_package_json(root_package_json_path)?; - let package_manager = package_json_string_value(&root_package_json, "packageManager") - .ok_or_else(|| anyhow::anyhow!("package.json is missing packageManager"))?; - let package_manager_version = package_json_pnpm_version(&package_manager) - .ok_or_else(|| anyhow::anyhow!("packageManager must be pnpm@, found {package_manager:?}"))?; - - let expected_engine_pnpm = format!(">={package_manager_version}"); - let engine_pnpm = package_json_engines_pnpm(&root_package_json) - .ok_or_else(|| anyhow::anyhow!("package.json engines is missing pnpm"))?; - if engine_pnpm != expected_engine_pnpm { - bail!("package.json engines.pnpm must be {expected_engine_pnpm:?}, found {engine_pnpm:?}"); - } - - for package_json_path in git_tracked_files(":(glob)**/package.json")? { - let package_json = read_package_json(&package_json_path)?; - let Some(found_package_manager) = package_json_string_value(&package_json, "packageManager") else { - continue; - }; - if found_package_manager != package_manager { - bail!( - "{} packageManager must match root package.json: expected {:?}, found {:?}", - package_json_path.display(), - package_manager, - found_package_manager - ); - } - } - - let root_workspace_path = Path::new("pnpm-workspace.yaml"); - let root_minimum_release_age = minimum_release_age(root_workspace_path)?; - for workspace_path in git_tracked_files(":(glob)**/pnpm-workspace.yaml")? { - let found_minimum_release_age = minimum_release_age(&workspace_path)?; - if found_minimum_release_age != root_minimum_release_age { - bail!( - "{} minimumReleaseAge must match root pnpm-workspace.yaml: expected {}, found {}", - workspace_path.display(), - root_minimum_release_age, - found_minimum_release_age - ); - } - } - - for npmrc_path in git_tracked_files(":(glob)**/.npmrc")? { - // Template package roots are copied into projects created by `spacetime init`. - // They must not embed this repo's package-age policy; smoketests enforce it - // at the pnpm process boundary instead. - if is_template_path(&npmrc_path) { - continue; - } - let found_minimum_release_age = npmrc_minimum_release_age(&npmrc_path, root_minimum_release_age)?; - if found_minimum_release_age != root_minimum_release_age { - bail!( - "{} minimum-release-age must match root pnpm-workspace.yaml: expected {}, found {}", - npmrc_path.display(), - root_minimum_release_age, - found_minimum_release_age - ); - } - } - - for package_json_path in git_tracked_files(":(glob)**/package.json")? { - // Template package roots are copied into projects created by `spacetime init`. - // They must not require adjacent .npmrc files for this repo's package-age - // policy; smoketests enforce it at the pnpm process boundary instead. - if is_template_path(&package_json_path) { - continue; - } - let package_json = read_package_json(&package_json_path)?; - if !is_npm_package_json(&package_json) { - continue; - } - let package_dir = package_json_path - .parent() - .expect("git-tracked package.json path should have a parent"); - let npmrc_path = package_dir.join(".npmrc"); - if !npmrc_path.is_file() { - bail!( - "{} is required because {} is an npm/pnpm package manifest.\nAdd {} containing:\nminimum-release-age={}", - npmrc_path.display(), - package_json_path.display(), - npmrc_path.display(), - root_minimum_release_age - ); - } - let found_minimum_release_age = npmrc_minimum_release_age(&npmrc_path, root_minimum_release_age)?; - if found_minimum_release_age != root_minimum_release_age { - bail!( - "{} minimum-release-age must match root pnpm-workspace.yaml: expected {}, found {}", - npmrc_path.display(), - root_minimum_release_age, - found_minimum_release_age - ); - } - } - - for workflow_path in git_tracked_files(".github/workflows/*")? { - let contents = fs::read_to_string(&workflow_path)?; - if contents.contains("pnpm/action-setup@v4") { - bail!( - "{} must use ./.github/actions/setup-pnpm instead of pnpm/action-setup@v4", - workflow_path.display() - ); - } - } - - Ok(()) -} - -/// Codex plugin ships a copy of `skills/`, because plugin installers do not follow symlinks, -/// this checks if the copy is in sync -fn check_codex_plugin_skills_sync() -> Result<()> { - let source = Path::new("skills"); - let copy = Path::new("codex-plugin/plugins/spacetimedb/skills"); - - if fs::symlink_metadata(copy) - .with_context(|| format!("reading {}", copy.display()))? - .file_type() - .is_symlink() - { - bail!( - "{} must be a real directory, not a symlink, plugin installers do not follow symlinks", - copy.display() - ); - } - - fn walk(root: &Path, dir: &Path, out: &mut BTreeSet) -> Result<()> { - for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { - let entry = entry?; - if entry.file_type()?.is_dir() { - walk(root, &entry.path(), out)?; - } else { - let rel = entry - .path() - .strip_prefix(root) - .expect("walked path should be under its root") - .to_path_buf(); - out.insert(rel); - } - } - Ok(()) - } - - let mut source_files = BTreeSet::new(); - walk(source, source, &mut source_files)?; - let mut copy_files = BTreeSet::new(); - walk(copy, copy, &mut copy_files)?; - - let mut drift = Vec::new(); - for rel in &source_files { - if !copy_files.contains(rel) { - drift.push(format!("missing from copy: {}", rel.display())); - } else if fs::read(source.join(rel))? != fs::read(copy.join(rel))? { - drift.push(format!("differs: {}", rel.display())); - } - } - for rel in ©_files { - if !source_files.contains(rel) { - drift.push(format!("extraneous in copy: {}", rel.display())); - } - } - - if !drift.is_empty() { - bail!( - "codex-plugin skills copy is out of sync with skills/:\n {}\nRun: node codex-plugin/scripts/check-skills-sync.ts --fix", - drift.join("\n ") - ); - } - Ok(()) -} - -#[derive(Subcommand)] -enum CiCmd { - /// Runs tests - /// - /// Runs rust tests, codegens csharp sdk and runs csharp tests. - /// This does not include Unreal tests. - /// This expects to run in a clean git state. - Test, - /// Lints the codebase - /// - /// Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there - /// are no warnings. - Lint, - /// Tests Wasm bindings - /// - /// Runs tests for the codegen crate and builds a test module with the wasm bindings. - WasmBindings, - /// Deprecated; use `cargo regen csharp dlls`. - /// - /// Builds and packs C# DLLs and NuGet packages for local Unity workflows. - Dlls, - /// Runs smoketests - /// - /// Executes the smoketests suite with some default exclusions. - Smoketests(smoketest::SmoketestsArgs), - /// Runs the keynote benchmark as a CI performance regression gate. - /// - /// Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the - /// keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and - /// fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. - KeynoteBench, - /// Tests the update flow - /// - /// Tests the self-update flow by building the spacetimedb-update binary for the specified - /// target, by default the current target, and performing a self-install into a temporary - /// directory. - UpdateFlow { - #[arg( - long, - long_help = "Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms." - )] - target: Option, - #[arg( - long, - default_value = "false", - long_help = "Whether to enable github token authentication feature when building the update binary. By default this is disabled." - )] - github_token_auth: bool, +const COMMANDS: &[Command] = &[ + Command { + name: "test", + package: "ci-test", }, - /// Generates CLI documentation and checks for changes - CliDocs { - #[arg( - long, - long_help = "specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located)" - )] - spacetime_path: Option, + Command { + name: "lint", + package: "ci-lint", }, - SelfDocs { - #[arg( - long, - default_value_t = false, - long_help = "Only check for changes, do not generate the docs" - )] - check: bool, + Command { + name: "wasm-bindings", + package: "ci-wasm-bindings", }, - - /// Verify that any non-root global.json files are symlinks to the root global.json. - GlobalJsonPolicy, - /// Checks that publishable crates satisfy publish constraints. - PublishChecks, - /// Runs TypeScript workspace tests and template build checks. - TypescriptTest, - /// Verifies that the repository version upgrade tool still works. - VersionUpgradeCheck, - /// Builds the docs site. - Docs, - /// Workflows should leave here if they should not be run as part of a no-subcommand invocation of `cargo ci`. - OtherWorkflows { - #[command(subcommand)] - cmd: OtherWorkflowsCmd, + Command { + name: "dlls", + package: "ci-dlls", }, -} - -#[derive(Subcommand)] -enum OtherWorkflowsCmd { - /// Checks that sensitive CODEOWNERS-controlled files have the required approvals. - CodeownersCheck { - /// Git ref to compare against, usually origin/. - #[arg(long)] - base_ref: String, - /// Pull request number to inspect for approval state. - #[arg(long)] - pr_number: u64, + Command { + name: "smoketests", + package: "ci-smoketests", }, - /// Interacts with CLA Assistant. - ClaAssistant { - #[command(subcommand)] - cmd: cla_assistant::ClaAssistantCmd, + Command { + name: "keynote-bench", + package: "ci-keynote-bench", }, -} - -fn run_all_clap_subcommands(skips: &[String]) -> Result<()> { - let subcmds = Cli::command() - .get_subcommands() - .map(|sc| sc.get_name().to_string()) - .collect::>(); - - for subcmd in subcmds { - if skips.contains(&subcmd) { - log::info!("skipping {subcmd} as requested"); - continue; - } - log::info!("executing cargo ci {subcmd}"); - cmd!("cargo", "ci", &subcmd).run()?; - } - Ok(()) -} - -fn tracked_rs_files_under(path: &str) -> Result> { - let output = cmd!("git", "ls-files", "--", path).read()?; - Ok(output - .lines() - .filter(|line| line.ends_with(".rs")) - .map(PathBuf::from) - .collect()) -} - -fn run_publish_checks() -> Result<()> { - cmd!("bash", "-lc", "test -d venv || python3 -m venv venv").run()?; - cmd!("venv/bin/pip3", "install", "argparse", "toml").run()?; - - let crates = cmd!( - "venv/bin/python3", - "tools/find-publish-list.py", - "--recursive", - "--directories", - "--quiet", - "spacetimedb", - "spacetimedb-sdk" - ) - .read()?; - - let mut failed = Vec::new(); - for crate_dir in crates.split_whitespace() { - if let Err(err) = cmd!("venv/bin/python3", "tools/crate-publish-checks.py", crate_dir).run() { - eprintln!("crate publish checks failed for {crate_dir}: {err}"); - failed.push(crate_dir.to_string()); - } - } - - if !failed.is_empty() { - bail!("crate publish checks failed for: {}", failed.join(", ")); - } - - Ok(()) -} + Command { + name: "update-flow", + package: "ci-update-flow", + }, + Command { + name: "cli-docs", + package: "ci-cli-docs", + }, + Command { + name: "self-docs", + package: "ci-self-docs", + }, + Command { + name: "global-json-policy", + package: "ci-global-json-policy", + }, + Command { + name: "publish-checks", + package: "ci-publish-checks", + }, + Command { + name: "typescript-test", + package: "ci-typescript-test", + }, + Command { + name: "version-upgrade-check", + package: "ci-version-upgrade-check", + }, + Command { + name: "docs", + package: "ci-docs-build", + }, + Command { + name: "other-workflows", + package: "ci-other-workflows", + }, +]; +const DEFAULT_SKIP: &[&str] = &["other-workflows"]; -fn run_typescript_tests() -> Result<()> { - pnpm(["build"]).dir("crates/bindings-typescript").run()?; - pnpm(["test"]).dir("crates/bindings-typescript").run()?; - pnpm(["generate"]).dir("templates/chat-react-ts").run()?; - let diff_status = cmd!( - "bash", - "tools/check-diff.sh", - "templates/chat-react-ts/src/module_bindings" - ) - .run()?; - if !diff_status.status.success() { - bail!("Bindings are dirty. Please generate bindings again and commit them to this branch."); +fn print_help() { + println!("Usage: cargo ci [--skip ] [COMMAND] [ARGS]..."); + println!(); + println!("Commands:"); + for command in COMMANDS { + println!(" {}", command.name); } - pnpm(["build"]).dir("templates/chat-react-ts").run()?; - pnpm(["-r", "--filter", "./**", "run", "build"]) - .dir("templates") - .run()?; - pnpm(["-r", "--filter", "./**", "run", "build"]) - .dir("crates/bindings-typescript") - .run()?; - Ok(()) } - -fn run_docs_build() -> Result<()> { - pnpm(["install"]).dir("docs").run()?; - pnpm(["build"]).dir("docs").run()?; - Ok(()) +fn command_for(name: &str) -> Option<&'static Command> { + COMMANDS.iter().find(|candidate| candidate.name == name) } - -fn run_version_upgrade_check() -> Result<()> { - cmd!( - "cargo", - "bump-versions", - "123.456.789", - "--rust-and-cli", - "--csharp", - "--typescript", - "--cpp", - "--accept-snapshots" - ) - .run()?; +fn run_command(command: &Command, forwarded: &[String]) -> Result<()> { + let mut cargo_args = vec!["run", "--package", command.package, "--"]; + cargo_args.extend(forwarded.iter().map(String::as_str)); + cmd("cargo", cargo_args).run()?; Ok(()) } - fn main() -> Result<()> { - env_logger::init(); - - let cli = Cli::parse(); - - match cli.cmd { - Some(CiCmd::Test) => { - pnpm(["build"]).dir("crates/bindings-typescript").run()?; - - // TODO: This doesn't work on at least user Linux machines, because something here apparently uses `sudo`? - - // Exclude smoketests from `cargo test --all` since they require pre-built binaries. - // Smoketests have their own dedicated command: `cargo ci smoketests` - cmd!( - "cargo", - "test", - "--all", - "--exclude", - "spacetimedb-smoketests", - "--exclude", - "spacetimedb-sdk", - "--exclude", - "spacetimedb", - "--", - "--test-threads=2", - "--skip", - "unreal" - ) - .run()?; - // Bindings snapshot tests rely on the unstable feature, - // as they compile and test APIs which are gated behind that feature, - // e.g. procedures, HTTP handlers. - cmd!( - "cargo", - "test", - "-p", - "spacetimedb", - "--features", - "unstable", - "--", - "--test-threads=2", - ) - .run()?; - // The SDK test harness uses the same child-process server guard as smoketests, - // which expects release CLI/standalone binaries to already exist. - cmd!( - "cargo", - "build", - "--release", - "-p", - "spacetimedb-cli", - "-p", - "spacetimedb-standalone", - "--features", - "spacetimedb-standalone/allow_loopback_http_for_tests", - ) - .run()?; - // SDK procedure tests intentionally make localhost HTTP requests. - cmd!( - "cargo", - "test", - "-p", - "spacetimedb-sdk", - "--features", - "allow_loopback_http_for_tests", - "--", - "--test-threads=2", - "--skip", - "unreal" - ) - .run()?; - // Run the same SDK suite against wasm/browser test clients. - cmd!( - "cargo", - "test", - "-p", - "spacetimedb-sdk", - "--features", - "allow_loopback_http_for_tests,browser", - "--", - "--test-threads=2", - "--skip", - "unreal" - ) - .run()?; - // TODO: This should check for a diff at the start. If there is one, we should alert the user - // that we're disabling diff checks because they have a dirty git repo, and to re-run in a clean one - // if they want those checks. - - // The fallocate tests have been flakely when running in parallel - cmd!( - "cargo", - "test", - "-p", - "spacetimedb-durability", - "--features", - "fallocate", - "--", - "--test-threads=1", - ) - .run()?; - cmd!("bash", "tools/check-diff.sh").run()?; - cmd!( - "cargo", - "run", - "-p", - "spacetimedb-codegen", - "--example", - "regen-csharp-moduledef", - ) - .run()?; - cmd!("bash", "tools/check-diff.sh", "crates/bindings-csharp").run()?; - cmd!("dotnet", "test", "-warnaserror") - .dir("crates/bindings-csharp") - .run()?; - } - - Some(CiCmd::Lint) => { - ensure_repo_root()?; - check_pnpm_release_age_policy()?; - check_codex_plugin_skills_sync()?; - // `cargo fmt --all` only checks files that Cargo discovers through workspace/package targets. - // However, we also keep Rust sources in a locations that are tracked but not part of our workspace, - // so this approach properly catches all the files, where `cargo fmt` does not. - let mut files = Vec::new(); - files.extend(tracked_rs_files_under(".")?); - const RUSTFMT_BATCH_SIZE: usize = 200; - for batch in files.chunks(RUSTFMT_BATCH_SIZE) { - let mut args = Vec::::with_capacity(batch.len() + 1); - args.push("--check".into()); - args.extend(batch.iter().map(|path| path.as_os_str().to_os_string())); - cmd("rustfmt", args).run()?; - } - cmd!( - "cargo", - "clippy", - "--timings", - "--all", - "--tests", - "--benches", - "--", - "-D", - "warnings", - ) - .run()?; - cmd!( - "cargo", - "clippy", - "--timings", - "--no-default-features", - "--features=browser", - "-pspacetimedb-sdk", - "--tests", - "--benches", - "--", - "-D", - "warnings", - ) - .run()?; - cmd!("dotnet", "tool", "restore").dir("crates/bindings-csharp").run()?; - cmd!("dotnet", "csharpier", "--check", ".") - .dir("crates/bindings-csharp") - .run()?; - pnpm(["lint"]).run()?; - cmd!("cargo", "test", "--doc", "--target", "wasm32-unknown-unknown") - .dir("crates/bindings") - .run()?; - cmd!("cargo", "test", "--doc").dir("crates/bindings").run()?; - // `bindings` is the only crate we care strongly about documenting, - // since we link to its docs.rs from our website. - // We won't pass `--no-deps`, though, - // since we want everything reachable through it to also work. - // This includes `sats` and `lib`. - cmd!("cargo", "doc") - .dir("crates/bindings") - // Make `cargo doc` exit with error on warnings, most notably broken links - .env("RUSTDOCFLAGS", "--deny warnings") - .run()?; - } - - Some(CiCmd::WasmBindings) => { - pnpm([ - "install", - "--filter", - "./crates/bindings-typescript...", - "--filter", - "./modules/module-test-ts...", - ]) - .run()?; - pnpm(["build"]).dir("crates/bindings-typescript").run()?; - cmd!("cargo", "test", "-p", "spacetimedb-codegen").run()?; - // Pre-build the CLI so that it _doesn't_ get `cargo update`d, since that may break the build. - cmd!("cargo", "build", "-p", "spacetimedb-cli").run()?; - // Make sure the `Cargo.lock` file reflects the latest available versions. - // This is what users would end up with on a fresh module, so we want to - // catch any compile errors arising from a different transitive closure - // of dependencies than what is in the workspace lock file. - // - // For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714 - cmd!("cargo", "update").run()?; - let cli_path = Path::new(env!("CARGO_MANIFEST_DIR")) - .ancestors() - .nth(2) - .unwrap() - .join("target/debug/spacetimedb-cli") - .with_extension(std::env::consts::EXE_EXTENSION); - cmd!(cli_path, "build", "--module-path", "modules/module-test",).run()?; - } - - Some(CiCmd::Dlls) => { - eprintln!("warning: `cargo ci dlls` is deprecated; use `cargo regen csharp dlls` instead"); - cmd!("cargo", "regen", "csharp", "dlls").run()?; - } - - Some(CiCmd::Smoketests(args)) => { - ensure_repo_root()?; - smoketest::run(args)?; - } - - Some(CiCmd::KeynoteBench) => { - ensure_repo_root()?; - keynote_bench::run()?; - } - - Some(CiCmd::UpdateFlow { - target, - github_token_auth, - }) => { - let mut common_args = vec![]; - if let Some(target) = target.as_ref() { - common_args.push("--target"); - common_args.push(target); - log::info!("checking update flow for target: {target}"); - } else { - log::info!("checking update flow"); - } - if github_token_auth { - common_args.push("--features"); - common_args.push("github-token-auth"); - } - - cmd( - "cargo", - ["build", "-p", "spacetimedb-update"] - .into_iter() - .chain(common_args.clone()), - ) - .run()?; - // NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`. - // My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it - // happens very frequently on the `macos-runner`, but we haven't seen it on any others). - let root_dir = tempfile::tempdir()?; - let root_dir_string = root_dir.path().to_string_lossy().to_string(); - let root_arg = format!("--root-dir={}", root_dir_string); - cmd( - "cargo", - ["run", "-p", "spacetimedb-update"] - .into_iter() - .chain(common_args.clone()) - .chain(["--", "self-install", &root_arg, "--yes"].into_iter()), - ) - .run()?; - - let mut spacetime_path = root_dir.path().join("spacetime"); - if !std::env::consts::EXE_EXTENSION.is_empty() { - spacetime_path.set_extension(std::env::consts::EXE_EXTENSION); - } - cmd(spacetime_path, [&root_arg, "help"]).run()?; - } - - Some(CiCmd::CliDocs { spacetime_path }) => { - if let Some(path) = spacetime_path { - env::set_current_dir(path).ok(); - } - let current_dir = env::current_dir().expect("No current directory!"); - let dir_name = current_dir.file_name().expect("No current directory!"); - if dir_name != "SpacetimeDB" && dir_name != "public" { - anyhow::bail!( - "You must execute this binary from inside of the SpacetimeDB directory, or use --spacetime-path" - ); - } - - pnpm(["install", "--recursive"]).run()?; - pnpm(["generate-cli-docs"]).dir("docs").run()?; - let out = cmd!("git", "status", "--porcelain", "--", "docs").read()?; - if out.is_empty() { - log::info!("No docs changes detected"); - } else { - anyhow::bail!("CLI docs are out of date:\n{out}"); + let mut raw = std::env::args().skip(1).collect::>(); + if raw.first().is_some_and(|arg| arg == "-h" || arg == "--help") { + print_help(); + return Ok(()); + } + let mut skips = DEFAULT_SKIP.iter().map(|s| s.to_string()).collect::>(); + let mut i = 0; + while i < raw.len() { + if raw[i] == "--skip" { + if i + 1 >= raw.len() { + bail!("--skip requires a command name"); } + skips.push(raw.remove(i + 1)); + raw.remove(i); + } else { + i += 1; } - - Some(CiCmd::SelfDocs { check }) => { - let readme_content = ci_docs::generate_cli_docs(); - let path = Path::new(README_PATH); - - if check { - let existing = fs::read_to_string(path).unwrap_or_default(); - if existing != readme_content { - bail!("README.md is out of date. Please run `cargo ci self-docs` to update it."); - } else { - log::info!("README.md is up to date."); - } - } else { - fs::write(path, readme_content)?; - log::info!("Wrote CLI docs to {}", path.display()); + } + if raw.is_empty() { + for command in COMMANDS { + if !skips.iter().any(|skip| skip == command.name) { + run_command(command, &[])?; } } - - Some(CiCmd::GlobalJsonPolicy) => { - check_global_json_policy()?; - } - - Some(CiCmd::OtherWorkflows { - cmd: OtherWorkflowsCmd::CodeownersCheck { base_ref, pr_number }, - }) => { - codeowners_check::run(&base_ref, pr_number)?; - } - - Some(CiCmd::PublishChecks) => { - run_publish_checks()?; - } - - Some(CiCmd::TypescriptTest) => { - run_typescript_tests()?; - } - - Some(CiCmd::VersionUpgradeCheck) => { - run_version_upgrade_check()?; - } - - Some(CiCmd::Docs) => { - run_docs_build()?; - } - - Some(CiCmd::OtherWorkflows { - cmd: OtherWorkflowsCmd::ClaAssistant { cmd }, - }) => { - cla_assistant::run(cmd)?; - } - - None => run_all_clap_subcommands(&cli.skip)?, + return Ok(()); } - - Ok(()) + let command_name = raw.remove(0); + let Some(command) = command_for(&command_name) else { + bail!("unknown cargo ci command `{command_name}`"); + }; + run_command(command, &raw) } diff --git a/tools/ci/src/util.rs b/tools/ci/src/util.rs deleted file mode 100644 index ac35a7a465b..00000000000 --- a/tools/ci/src/util.rs +++ /dev/null @@ -1,11 +0,0 @@ -#![allow(clippy::disallowed_macros)] - -use anyhow::{bail, Result}; -use std::path::Path; - -pub fn ensure_repo_root() -> Result<()> { - if !Path::new("Cargo.toml").exists() { - bail!("You must execute this command from the SpacetimeDB repository root (where Cargo.toml is located)"); - } - Ok(()) -} From 0272d77f8ddbd950d24a530fe5d13fbea3b33087 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Fri, 7 Aug 2026 13:06:52 -0400 Subject: [PATCH 02/34] Fix wasm bindings CI command path --- Cargo.lock | 1 + tools/ci/commands/wasm-bindings/Cargo.toml | 1 + tools/ci/commands/wasm-bindings/src/main.rs | 6 +----- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f63db3e9e7f..2578284b603 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1061,6 +1061,7 @@ name = "ci-wasm-bindings" version = "0.1.0" dependencies = [ "anyhow", + "ci-common", "duct", "env_logger 0.10.2", ] diff --git a/tools/ci/commands/wasm-bindings/Cargo.toml b/tools/ci/commands/wasm-bindings/Cargo.toml index be9456b7754..de3fd439244 100644 --- a/tools/ci/commands/wasm-bindings/Cargo.toml +++ b/tools/ci/commands/wasm-bindings/Cargo.toml @@ -7,3 +7,4 @@ edition.workspace = true anyhow.workspace = true duct.workspace = true env_logger.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/wasm-bindings/src/main.rs b/tools/ci/commands/wasm-bindings/src/main.rs index 1f708d7341e..96a803e55ba 100644 --- a/tools/ci/commands/wasm-bindings/src/main.rs +++ b/tools/ci/commands/wasm-bindings/src/main.rs @@ -16,7 +16,6 @@ where cmd("pnpm", args) } } -use std::path::Path; fn main() -> Result<()> { env_logger::init(); pnpm([ @@ -38,10 +37,7 @@ fn main() -> Result<()> { // // For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714 cmd!("cargo", "update").run()?; - let cli_path = Path::new(env!("CARGO_MANIFEST_DIR")) - .ancestors() - .nth(2) - .unwrap() + let cli_path = ci_common::repo_root() .join("target/debug/spacetimedb-cli") .with_extension(std::env::consts::EXE_EXTENSION); cmd!(cli_path, "build", "--module-path", "modules/module-test",).run()?; From 2c10c7605b0b597a6fe8dd79dbaecdb7cfb7608b Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Fri, 7 Aug 2026 13:37:59 -0400 Subject: [PATCH 03/34] Replace gh run watch in Internal Tests --- .github/workflows/internal-tests.yml | 13 +-- tools/ci/README.md | 1 + tools/ci/commands/other-workflows/src/main.rs | 4 + .../ci/commands/other-workflows/src/watch.rs | 99 +++++++++++++++++++ 4 files changed, 107 insertions(+), 10 deletions(-) create mode 100644 tools/ci/commands/other-workflows/src/watch.rs diff --git a/.github/workflows/internal-tests.yml b/.github/workflows/internal-tests.yml index ba4cb230da6..0d38053977d 100644 --- a/.github/workflows/internal-tests.yml +++ b/.github/workflows/internal-tests.yml @@ -93,17 +93,10 @@ jobs: RUN_URL: ${{ steps.dispatch.outputs.run_url }} run: | set -euo pipefail - echo "Waiting for workflow result... ${RUN_URL}" - set +e - # Due to our limited-scope token permissions, `gh run watch` spams errors about not being able to get annotations. They look worrying but they're benign, so we filter them out. - gh run watch "$RUN_ID" \ + cargo ci other-workflows watch \ --repo "$TARGET_OWNER/$TARGET_REPO" \ - --exit-status \ - --interval 30 2>&1 \ - | grep -Fv "requesting annotations returned 403 Forbidden as the token does not have sufficient permissions" - watch_status="${PIPESTATUS[0]}" - set -e - exit "$watch_status" + --run-id "$RUN_ID" \ + --run-url "$RUN_URL" - name: Cancel invoked run if workflow cancelled if: ${{ cancelled() && steps.dispatch.outputs.run_id }} diff --git a/tools/ci/README.md b/tools/ci/README.md index ebc5a4fbbcd..9ed2edcc24b 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -181,6 +181,7 @@ Usage: ci-other-workflows Commands: codeowners-check Checks that sensitive CODEOWNERS-controlled files have the required approvals cla-assistant Interacts with CLA Assistant + watch Waits for a GitHub Actions workflow run to complete help Print this message or the help of the given subcommand(s) Options: diff --git a/tools/ci/commands/other-workflows/src/main.rs b/tools/ci/commands/other-workflows/src/main.rs index 9dd466e41e2..3b61e44f0e3 100644 --- a/tools/ci/commands/other-workflows/src/main.rs +++ b/tools/ci/commands/other-workflows/src/main.rs @@ -12,6 +12,7 @@ fn ensure_repo_root() -> Result<()> { mod cla_assistant; mod codeowners_check; +mod watch; #[derive(Parser)] struct Args { @@ -32,11 +33,14 @@ enum OtherWorkflowsCmd { #[command(subcommand)] cmd: cla_assistant::ClaAssistantCmd, }, + /// Waits for a GitHub Actions workflow run to complete. + Watch(watch::Args), } fn main() -> Result<()> { env_logger::init(); match Args::parse().cmd { OtherWorkflowsCmd::CodeownersCheck { base_ref, pr_number } => codeowners_check::run(&base_ref, pr_number), OtherWorkflowsCmd::ClaAssistant { cmd } => cla_assistant::run(cmd), + OtherWorkflowsCmd::Watch(args) => watch::run(args), } } diff --git a/tools/ci/commands/other-workflows/src/watch.rs b/tools/ci/commands/other-workflows/src/watch.rs new file mode 100644 index 00000000000..c918499c9a3 --- /dev/null +++ b/tools/ci/commands/other-workflows/src/watch.rs @@ -0,0 +1,99 @@ +use anyhow::{bail, Context, Result}; +use duct::cmd; +use serde::Deserialize; +use std::time::Duration; + +#[derive(clap::Args)] +pub(crate) struct Args { + /// Repository containing the workflow run, in owner/repo form. + #[arg(long)] + repo: String, + + /// GitHub Actions workflow run ID. + #[arg(long)] + run_id: u64, + + /// Optional URL printed while waiting for the run. + #[arg(long)] + run_url: Option, + + /// Seconds to sleep between polls. + #[arg(long, default_value_t = 30)] + interval_seconds: u64, + + /// Maximum number of polls before timing out. + #[arg(long, default_value_t = 240)] + max_attempts: u64, +} + +#[derive(Deserialize)] +struct WorkflowRun { + status: String, + conclusion: Option, + url: Option, + jobs: Vec, +} + +#[derive(Deserialize)] +struct WorkflowJob { + name: String, + status: String, + conclusion: Option, +} + +fn get_workflow_run(repo: &str, run_id: u64) -> Result { + let raw = cmd!( + "gh", + "run", + "view", + run_id.to_string(), + "--repo", + repo, + "--json", + "status,conclusion,url,jobs", + ) + .read() + .with_context(|| format!("failed to read workflow run {run_id} in {repo}"))?; + serde_json::from_str(&raw).with_context(|| format!("failed to parse workflow run {run_id} in {repo}")) +} + +fn print_job_summary(run: &WorkflowRun) { + println!("Job summary:"); + for job in &run.jobs { + let result = job.conclusion.as_deref().unwrap_or(&job.status); + println!(" {result:>11} {}", job.name); + } +} + +pub(crate) fn run(args: Args) -> Result<()> { + let run_url = args + .run_url + .clone() + .or_else(|| get_workflow_run(&args.repo, args.run_id).ok().and_then(|run| run.url)); + + if let Some(run_url) = run_url { + println!("Waiting for workflow result... {run_url}"); + } else { + println!( + "Waiting for workflow result: {}/actions/runs/{}", + args.repo, args.run_id + ); + } + + for _ in 0..args.max_attempts { + let run = get_workflow_run(&args.repo, args.run_id)?; + if run.status == "completed" { + print_job_summary(&run); + let conclusion = run.conclusion.as_deref().unwrap_or("success"); + if conclusion == "success" { + return Ok(()); + } + bail!("workflow run {} completed with conclusion: {conclusion}", args.run_id); + } + + println!("workflow run {} status: {}", args.run_id, run.status); + std::thread::sleep(Duration::from_secs(args.interval_seconds)); + } + + bail!("timed out waiting for workflow run {} to complete", args.run_id) +} From b4533daab07dc58d033a8d5e97557a000423ebd7 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Fri, 7 Aug 2026 13:47:50 -0400 Subject: [PATCH 04/34] Revert "Replace gh run watch in Internal Tests" This reverts commit 2c10c7605b0b597a6fe8dd79dbaecdb7cfb7608b. --- .github/workflows/internal-tests.yml | 13 ++- tools/ci/README.md | 1 - tools/ci/commands/other-workflows/src/main.rs | 4 - .../ci/commands/other-workflows/src/watch.rs | 99 ------------------- 4 files changed, 10 insertions(+), 107 deletions(-) delete mode 100644 tools/ci/commands/other-workflows/src/watch.rs diff --git a/.github/workflows/internal-tests.yml b/.github/workflows/internal-tests.yml index 0d38053977d..ba4cb230da6 100644 --- a/.github/workflows/internal-tests.yml +++ b/.github/workflows/internal-tests.yml @@ -93,10 +93,17 @@ jobs: RUN_URL: ${{ steps.dispatch.outputs.run_url }} run: | set -euo pipefail - cargo ci other-workflows watch \ + echo "Waiting for workflow result... ${RUN_URL}" + set +e + # Due to our limited-scope token permissions, `gh run watch` spams errors about not being able to get annotations. They look worrying but they're benign, so we filter them out. + gh run watch "$RUN_ID" \ --repo "$TARGET_OWNER/$TARGET_REPO" \ - --run-id "$RUN_ID" \ - --run-url "$RUN_URL" + --exit-status \ + --interval 30 2>&1 \ + | grep -Fv "requesting annotations returned 403 Forbidden as the token does not have sufficient permissions" + watch_status="${PIPESTATUS[0]}" + set -e + exit "$watch_status" - name: Cancel invoked run if workflow cancelled if: ${{ cancelled() && steps.dispatch.outputs.run_id }} diff --git a/tools/ci/README.md b/tools/ci/README.md index 9ed2edcc24b..ebc5a4fbbcd 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -181,7 +181,6 @@ Usage: ci-other-workflows Commands: codeowners-check Checks that sensitive CODEOWNERS-controlled files have the required approvals cla-assistant Interacts with CLA Assistant - watch Waits for a GitHub Actions workflow run to complete help Print this message or the help of the given subcommand(s) Options: diff --git a/tools/ci/commands/other-workflows/src/main.rs b/tools/ci/commands/other-workflows/src/main.rs index 3b61e44f0e3..9dd466e41e2 100644 --- a/tools/ci/commands/other-workflows/src/main.rs +++ b/tools/ci/commands/other-workflows/src/main.rs @@ -12,7 +12,6 @@ fn ensure_repo_root() -> Result<()> { mod cla_assistant; mod codeowners_check; -mod watch; #[derive(Parser)] struct Args { @@ -33,14 +32,11 @@ enum OtherWorkflowsCmd { #[command(subcommand)] cmd: cla_assistant::ClaAssistantCmd, }, - /// Waits for a GitHub Actions workflow run to complete. - Watch(watch::Args), } fn main() -> Result<()> { env_logger::init(); match Args::parse().cmd { OtherWorkflowsCmd::CodeownersCheck { base_ref, pr_number } => codeowners_check::run(&base_ref, pr_number), OtherWorkflowsCmd::ClaAssistant { cmd } => cla_assistant::run(cmd), - OtherWorkflowsCmd::Watch(args) => watch::run(args), } } diff --git a/tools/ci/commands/other-workflows/src/watch.rs b/tools/ci/commands/other-workflows/src/watch.rs deleted file mode 100644 index c918499c9a3..00000000000 --- a/tools/ci/commands/other-workflows/src/watch.rs +++ /dev/null @@ -1,99 +0,0 @@ -use anyhow::{bail, Context, Result}; -use duct::cmd; -use serde::Deserialize; -use std::time::Duration; - -#[derive(clap::Args)] -pub(crate) struct Args { - /// Repository containing the workflow run, in owner/repo form. - #[arg(long)] - repo: String, - - /// GitHub Actions workflow run ID. - #[arg(long)] - run_id: u64, - - /// Optional URL printed while waiting for the run. - #[arg(long)] - run_url: Option, - - /// Seconds to sleep between polls. - #[arg(long, default_value_t = 30)] - interval_seconds: u64, - - /// Maximum number of polls before timing out. - #[arg(long, default_value_t = 240)] - max_attempts: u64, -} - -#[derive(Deserialize)] -struct WorkflowRun { - status: String, - conclusion: Option, - url: Option, - jobs: Vec, -} - -#[derive(Deserialize)] -struct WorkflowJob { - name: String, - status: String, - conclusion: Option, -} - -fn get_workflow_run(repo: &str, run_id: u64) -> Result { - let raw = cmd!( - "gh", - "run", - "view", - run_id.to_string(), - "--repo", - repo, - "--json", - "status,conclusion,url,jobs", - ) - .read() - .with_context(|| format!("failed to read workflow run {run_id} in {repo}"))?; - serde_json::from_str(&raw).with_context(|| format!("failed to parse workflow run {run_id} in {repo}")) -} - -fn print_job_summary(run: &WorkflowRun) { - println!("Job summary:"); - for job in &run.jobs { - let result = job.conclusion.as_deref().unwrap_or(&job.status); - println!(" {result:>11} {}", job.name); - } -} - -pub(crate) fn run(args: Args) -> Result<()> { - let run_url = args - .run_url - .clone() - .or_else(|| get_workflow_run(&args.repo, args.run_id).ok().and_then(|run| run.url)); - - if let Some(run_url) = run_url { - println!("Waiting for workflow result... {run_url}"); - } else { - println!( - "Waiting for workflow result: {}/actions/runs/{}", - args.repo, args.run_id - ); - } - - for _ in 0..args.max_attempts { - let run = get_workflow_run(&args.repo, args.run_id)?; - if run.status == "completed" { - print_job_summary(&run); - let conclusion = run.conclusion.as_deref().unwrap_or("success"); - if conclusion == "success" { - return Ok(()); - } - bail!("workflow run {} completed with conclusion: {conclusion}", args.run_id); - } - - println!("workflow run {} status: {}", args.run_id, run.status); - std::thread::sleep(Duration::from_secs(args.interval_seconds)); - } - - bail!("timed out waiting for workflow run {} to complete", args.run_id) -} From 7166640094f3e8de2876cfb68ad4a2b9de369812 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 10:17:30 -0400 Subject: [PATCH 05/34] Split CI workflow helpers into packages --- .github/CODEOWNERS | 2 +- Cargo.lock | 55 +++++++++++++------ Cargo.toml | 4 +- tools/ci/README.md | 46 ++++++++++++++-- .../Cargo.toml | 5 +- .../src/main.rs} | 30 ++++++---- tools/ci/commands/codeowners-check/Cargo.toml | 12 ++++ .../src/main.rs} | 21 ++++++- .../coordinate-internal-tests/Cargo.toml | 11 ++++ .../src/main.rs} | 12 ++-- tools/ci/commands/lint/Cargo.toml | 4 -- tools/ci/commands/other-workflows/src/main.rs | 46 ---------------- tools/ci/commands/self-docs/src/main.rs | 47 ++++++++++++++-- tools/ci/common/Cargo.toml | 1 + tools/ci/common/src/lib.rs | 9 +++ tools/ci/src/main.rs | 54 ++++++++++++++++-- 16 files changed, 255 insertions(+), 104 deletions(-) rename tools/ci/commands/{other-workflows => cla-assistant}/Cargo.toml (69%) rename tools/ci/commands/{other-workflows/src/cla_assistant.rs => cla-assistant/src/main.rs} (94%) create mode 100644 tools/ci/commands/codeowners-check/Cargo.toml rename tools/ci/commands/{other-workflows/src/codeowners_check.rs => codeowners-check/src/main.rs} (94%) create mode 100644 tools/ci/commands/coordinate-internal-tests/Cargo.toml rename tools/ci/commands/{other-workflows/src/internal_tests.rs => coordinate-internal-tests/src/main.rs} (99%) delete mode 100644 tools/ci/commands/other-workflows/src/main.rs diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4d6f7594981..18d3516d5f2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,7 +9,7 @@ LICENSE.txt @cloutiertyler /crates/cli/src/ @bfops @cloutiertyler @jdetter /tools/ci/ @bfops @cloutiertyler @jdetter /tools/ci/commands/keynote-bench/ @joshua-spacetime @cloutiertyler @jdetter -/tools/ci/commands/other-workflows/src/codeowners_check.rs @cloutiertyler +/tools/ci/commands/codeowners-check/ @cloutiertyler /tools/upgrade-version/ @bfops @jdetter @cloutiertyler @rekhoff /tools/release/ @bfops @jdetter @cloutiertyler @rekhoff /tools/license-check/ @bfops @jdetter @cloutiertyler diff --git a/Cargo.lock b/Cargo.lock index 2578284b603..537a90a48f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -902,6 +902,17 @@ dependencies = [ "duct", ] +[[package]] +name = "ci-cla-assistant" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.50", + "reqwest 0.12.24", + "serde", + "serde_json", +] + [[package]] name = "ci-cli-docs" version = "0.1.0" @@ -913,9 +924,35 @@ dependencies = [ "log", ] +[[package]] +name = "ci-codeowners-check" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-common", + "clap 4.5.50", + "duct", + "regex", + "serde_json", +] + [[package]] name = "ci-common" version = "0.1.0" +dependencies = [ + "anyhow", +] + +[[package]] +name = "ci-coordinate-internal-tests" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.50", + "duct", + "serde", + "serde_json", +] [[package]] name = "ci-dlls" @@ -964,26 +1001,8 @@ name = "ci-lint" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.5.50", "duct", "env_logger 0.10.2", - "log", - "regex", - "serde_json", - "tempfile", -] - -[[package]] -name = "ci-other-workflows" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap 4.5.50", - "duct", - "env_logger 0.10.2", - "regex", - "reqwest 0.12.24", - "serde", "serde_json", ] diff --git a/Cargo.toml b/Cargo.toml index 2f4930191b9..c3a59b5c52f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,7 +82,9 @@ members = [ "tools/ci/commands/typescript-test", "tools/ci/commands/version-upgrade-check", "tools/ci/commands/docs", - "tools/ci/commands/other-workflows", + "tools/ci/commands/codeowners-check", + "tools/ci/commands/cla-assistant", + "tools/ci/commands/coordinate-internal-tests", "tools/ci/common", "tools/keynote-bench-harness", "tools/license-check", diff --git a/tools/ci/README.md b/tools/ci/README.md index b315aa61779..3d22972e9e4 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -176,14 +176,48 @@ cargo ci docs **Usage:** ```bash -Usage: ci-other-workflows +cargo ci other-workflows +``` + +#### `coordinate-internal-tests` + +**Usage:** +```bash +Selects or starts the private workflow for a public Internal Tests run + +Usage: ci-coordinate-internal-tests [OPTIONS] --public-sha + +Options: + --public-sha + Immutable public commit to test + --public-pr-number + Public pull request number, when coordinating a pull request run + -h, --help + Print help +``` + +#### `codeowners-check` + +**Usage:** +```bash +Usage: ci-codeowners-check --base-ref --pr-number + +Options: + --base-ref Git ref to compare against, usually origin/ + --pr-number Pull request number to inspect for approval state + -h, --help Print help +``` + +#### `cla-assistant` + +**Usage:** +```bash +Usage: ci-cla-assistant Commands: - coordinate-internal-tests Selects or starts the private workflow for a public Internal Tests run - codeowners-check Checks that sensitive CODEOWNERS-controlled files have the required - approvals - cla-assistant Interacts with CLA Assistant - help Print this message or the help of the given subcommand(s) + retry Retries CLA Assistant if `license/cla` is the only remaining PR blocker + status Returns the `license/cla` status for a pull request or commit SHA + help Print this message or the help of the given subcommand(s) Options: -h, --help Print help diff --git a/tools/ci/commands/other-workflows/Cargo.toml b/tools/ci/commands/cla-assistant/Cargo.toml similarity index 69% rename from tools/ci/commands/other-workflows/Cargo.toml rename to tools/ci/commands/cla-assistant/Cargo.toml index 543f7702f66..4096cd8087e 100644 --- a/tools/ci/commands/other-workflows/Cargo.toml +++ b/tools/ci/commands/cla-assistant/Cargo.toml @@ -1,14 +1,11 @@ [package] -name = "ci-other-workflows" +name = "ci-cla-assistant" version = "0.1.0" edition.workspace = true [dependencies] anyhow.workspace = true -duct.workspace = true clap.workspace = true -regex.workspace = true reqwest = { workspace = true, features = ["blocking", "json"] } serde.workspace = true serde_json.workspace = true -env_logger.workspace = true diff --git a/tools/ci/commands/other-workflows/src/cla_assistant.rs b/tools/ci/commands/cla-assistant/src/main.rs similarity index 94% rename from tools/ci/commands/other-workflows/src/cla_assistant.rs rename to tools/ci/commands/cla-assistant/src/main.rs index 29b148be5e7..a7aa580734b 100644 --- a/tools/ci/commands/other-workflows/src/cla_assistant.rs +++ b/tools/ci/commands/cla-assistant/src/main.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use std::env; use anyhow::{anyhow, bail, Context, Result}; -use clap::{ArgGroup, Args, Subcommand}; +use clap::{ArgGroup, Args, Parser, Subcommand}; use reqwest::blocking::Client; use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, AUTHORIZATION, USER_AGENT}; use serde::de::DeserializeOwned; @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; const CLA_CONTEXT: &str = "license/cla"; #[derive(Subcommand)] -pub enum ClaAssistantCmd { +enum ClaAssistantCmd { /// Retries CLA Assistant if `license/cla` is the only remaining PR blocker. Retry(RetryArgs), @@ -20,14 +20,14 @@ pub enum ClaAssistantCmd { } #[derive(Args)] -pub struct RetryArgs { +struct RetryArgs { /// Pull request number to check. #[arg(long)] - pub pr_number: u64, + pr_number: u64, /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. #[arg(long)] - pub repo: Option, + repo: Option, } #[derive(Args)] @@ -37,27 +37,37 @@ pub struct RetryArgs { .multiple(false) .args(["pr", "sha"]), ))] -pub struct StatusArgs { +struct StatusArgs { /// Pull request number whose head commit should be checked. #[arg(long)] - pub pr: Option, + pr: Option, /// Commit SHA to check. #[arg(long)] - pub sha: Option, + sha: Option, /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. #[arg(long)] - pub repo: Option, + repo: Option, } -pub fn run(cmd: ClaAssistantCmd) -> Result<()> { +#[derive(Parser)] +struct Cli { + #[command(subcommand)] + cmd: ClaAssistantCmd, +} + +fn run(cmd: ClaAssistantCmd) -> Result<()> { match cmd { ClaAssistantCmd::Retry(args) => retry(args), ClaAssistantCmd::Status(args) => status(args), } } +fn main() -> Result<()> { + run(Cli::parse().cmd) +} + fn retry(args: RetryArgs) -> Result<()> { let repo = Repo::from_arg_or_env(args.repo)?; let token = env::var("GITHUB_TOKEN").context("GITHUB_TOKEN is required")?; diff --git a/tools/ci/commands/codeowners-check/Cargo.toml b/tools/ci/commands/codeowners-check/Cargo.toml new file mode 100644 index 00000000000..19d76f82671 --- /dev/null +++ b/tools/ci/commands/codeowners-check/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ci-codeowners-check" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +ci-common = { path = "../../common" } +duct.workspace = true +regex.workspace = true +serde_json.workspace = true diff --git a/tools/ci/commands/other-workflows/src/codeowners_check.rs b/tools/ci/commands/codeowners-check/src/main.rs similarity index 94% rename from tools/ci/commands/other-workflows/src/codeowners_check.rs rename to tools/ci/commands/codeowners-check/src/main.rs index f456100b9dc..fe0370c1590 100644 --- a/tools/ci/commands/other-workflows/src/codeowners_check.rs +++ b/tools/ci/commands/codeowners-check/src/main.rs @@ -1,5 +1,6 @@ -use crate::ensure_repo_root; use anyhow::{anyhow, bail, Context, Result}; +use ci_common::ensure_repo_root; +use clap::Parser; use duct::cmd; use serde_json::Value; use std::collections::HashMap; @@ -7,7 +8,18 @@ use std::path::{Path, PathBuf}; const REPO: &str = "clockworklabs/SpacetimeDB"; -pub fn run(base_ref: &str, pr_number: u64) -> Result<()> { +#[derive(Parser)] +struct Args { + /// Git ref to compare against, usually origin/. + #[arg(long)] + base_ref: String, + + /// Pull request number to inspect for approval state. + #[arg(long)] + pr_number: u64, +} + +fn run(base_ref: &str, pr_number: u64) -> Result<()> { ensure_repo_root()?; fetch_base_ref(base_ref)?; @@ -21,6 +33,11 @@ pub fn run(base_ref: &str, pr_number: u64) -> Result<()> { Ok(()) } +fn main() -> Result<()> { + let args = Args::parse(); + run(&args.base_ref, args.pr_number) +} + fn file_review_requirements(base_ref: &str, path: &Path, review: &ReviewStatus) -> Result<()> { if license::is_license(path) && !license::is_trivial_change(base_ref, path)? { review.require("cloutiertyler")?; diff --git a/tools/ci/commands/coordinate-internal-tests/Cargo.toml b/tools/ci/commands/coordinate-internal-tests/Cargo.toml new file mode 100644 index 00000000000..c4b71bd705d --- /dev/null +++ b/tools/ci/commands/coordinate-internal-tests/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ci-coordinate-internal-tests" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +duct.workspace = true +serde.workspace = true +serde_json.workspace = true diff --git a/tools/ci/commands/other-workflows/src/internal_tests.rs b/tools/ci/commands/coordinate-internal-tests/src/main.rs similarity index 99% rename from tools/ci/commands/other-workflows/src/internal_tests.rs rename to tools/ci/commands/coordinate-internal-tests/src/main.rs index 78d7f7cfb66..e46858a32c3 100644 --- a/tools/ci/commands/other-workflows/src/internal_tests.rs +++ b/tools/ci/commands/coordinate-internal-tests/src/main.rs @@ -1,5 +1,5 @@ use anyhow::{bail, ensure, Context, Result}; -use clap::Args; +use clap::Parser; use duct::{cmd, Expression}; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -14,9 +14,9 @@ const PRIVATE_REPO: &str = "clockworklabs/SpacetimeDBPrivate"; const PRIVATE_WORKFLOW: &str = "ci.yml"; const PRIVATE_DEFAULT_BRANCH: &str = "master"; -#[derive(Args)] /// Selects or starts the private workflow for a public Internal Tests run. -pub(crate) struct CoordinateArgs { +#[derive(Parser)] +struct Args { /// Immutable public commit to test. #[arg(long)] public_sha: String, @@ -451,7 +451,7 @@ fn write_github_output(name: &str, value: impl std::fmt::Display) -> Result<()> } /// Coordinates the public Internal Tests run without checking out or executing private code. -pub(crate) fn coordinate(args: CoordinateArgs) -> Result<()> { +fn coordinate(args: Args) -> Result<()> { let private_source = resolve_private_source(args.public_pr_number)?; let coordinated = match private_source { @@ -466,6 +466,10 @@ pub(crate) fn coordinate(args: CoordinateArgs) -> Result<()> { Ok(()) } +fn main() -> Result<()> { + coordinate(Args::parse()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/tools/ci/commands/lint/Cargo.toml b/tools/ci/commands/lint/Cargo.toml index c1e456a84ca..ff2b8002952 100644 --- a/tools/ci/commands/lint/Cargo.toml +++ b/tools/ci/commands/lint/Cargo.toml @@ -6,9 +6,5 @@ edition.workspace = true [dependencies] anyhow.workspace = true duct.workspace = true -clap.workspace = true -regex.workspace = true serde_json.workspace = true -tempfile.workspace = true env_logger.workspace = true -log.workspace = true diff --git a/tools/ci/commands/other-workflows/src/main.rs b/tools/ci/commands/other-workflows/src/main.rs deleted file mode 100644 index de976d6eb08..00000000000 --- a/tools/ci/commands/other-workflows/src/main.rs +++ /dev/null @@ -1,46 +0,0 @@ -#![allow(clippy::disallowed_macros)] -use anyhow::{bail, Result}; -use clap::{Parser, Subcommand}; -use std::path::Path; - -fn ensure_repo_root() -> Result<()> { - if !Path::new("Cargo.toml").exists() { - bail!("You must execute this command from the SpacetimeDB repository root (where Cargo.toml is located)"); - } - Ok(()) -} - -mod cla_assistant; -mod codeowners_check; -mod internal_tests; - -#[derive(Parser)] -struct Args { - #[command(subcommand)] - cmd: OtherWorkflowsCmd, -} -#[derive(Subcommand)] -enum OtherWorkflowsCmd { - /// Selects or starts the private workflow for a public Internal Tests run. - CoordinateInternalTests(internal_tests::CoordinateArgs), - /// Checks that sensitive CODEOWNERS-controlled files have the required approvals. - CodeownersCheck { - #[arg(long)] - base_ref: String, - #[arg(long)] - pr_number: u64, - }, - /// Interacts with CLA Assistant. - ClaAssistant { - #[command(subcommand)] - cmd: cla_assistant::ClaAssistantCmd, - }, -} -fn main() -> Result<()> { - env_logger::init(); - match Args::parse().cmd { - OtherWorkflowsCmd::CoordinateInternalTests(args) => internal_tests::coordinate(args), - OtherWorkflowsCmd::CodeownersCheck { base_ref, pr_number } => codeowners_check::run(&base_ref, pr_number), - OtherWorkflowsCmd::ClaAssistant { cmd } => cla_assistant::run(cmd), - } -} diff --git a/tools/ci/commands/self-docs/src/main.rs b/tools/ci/commands/self-docs/src/main.rs index 90cc263baf4..60572539482 100644 --- a/tools/ci/commands/self-docs/src/main.rs +++ b/tools/ci/commands/self-docs/src/main.rs @@ -19,8 +19,35 @@ struct CommandDoc { package: &'static str, help_safe: bool, } +struct CommandGroupDoc { + name: &'static str, + commands: &'static [CommandDoc], +} + +const OTHER_WORKFLOWS_COMMANDS: &[CommandDoc] = &[ + CommandDoc { + name: "coordinate-internal-tests", + package: "ci-coordinate-internal-tests", + help_safe: true, + }, + CommandDoc { + name: "codeowners-check", + package: "ci-codeowners-check", + help_safe: true, + }, + CommandDoc { + name: "cla-assistant", + package: "ci-cla-assistant", + help_safe: true, + }, +]; + +const COMMAND_GROUPS: &[CommandGroupDoc] = &[CommandGroupDoc { + name: "other-workflows", + commands: OTHER_WORKFLOWS_COMMANDS, +}]; -const COMMANDS: &[CommandDoc] = &[ +const ALL_COMMANDS: &[CommandDoc] = &[ CommandDoc { name: "test", package: "ci-test", @@ -93,8 +120,8 @@ const COMMANDS: &[CommandDoc] = &[ }, CommandDoc { name: "other-workflows", - package: "ci-other-workflows", - help_safe: true, + package: "", + help_safe: false, }, ]; @@ -107,16 +134,28 @@ fn command_usage(command: &CommandDoc) -> Result { let help = cmd!("cargo", "run", "--quiet", "--package", command.package, "--", "--help").read()?; return Ok(normalize_help(&help)); } + if command.package.is_empty() { + return Ok(format!("cargo ci {} ", command.name)); + } Ok(format!("cargo ci {}", command.name)) } fn generate() -> Result { let mut out = String::from("# SpacetimeDB's cargo ci\n\n## Overview\n\nThis document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options.\n\n## `cargo ci`\n\n**Usage:**\n```bash\ncargo ci [--skip ] [COMMAND] [ARGS]...\n```\n\n"); - for command in COMMANDS { + for command in ALL_COMMANDS { out.push_str(&format!( "### `{}`\n\n**Usage:**\n```bash\n{}\n```\n\n", command.name, command_usage(command)?.trim(), )); + if let Some(group) = COMMAND_GROUPS.iter().find(|group| group.name == command.name) { + for subcommand in group.commands { + out.push_str(&format!( + "#### `{}`\n\n**Usage:**\n```bash\n{}\n```\n\n", + subcommand.name, + command_usage(subcommand)?.trim(), + )); + } + } } out.push_str("---\n\nThis document is auto-generated by running:\n\n```bash\ncargo ci self-docs\n```\n"); Ok(out) diff --git a/tools/ci/common/Cargo.toml b/tools/ci/common/Cargo.toml index f4e1d4195e4..99fdd485443 100644 --- a/tools/ci/common/Cargo.toml +++ b/tools/ci/common/Cargo.toml @@ -4,3 +4,4 @@ version = "0.1.0" edition.workspace = true [dependencies] +anyhow.workspace = true diff --git a/tools/ci/common/src/lib.rs b/tools/ci/common/src/lib.rs index cdb99009889..a14f5ebf06d 100644 --- a/tools/ci/common/src/lib.rs +++ b/tools/ci/common/src/lib.rs @@ -1,5 +1,14 @@ +use anyhow::{bail, Result}; +use std::path::Path; use std::path::PathBuf; +pub fn ensure_repo_root() -> Result<()> { + if !Path::new("Cargo.toml").exists() { + bail!("You must execute this command from the SpacetimeDB repository root (where Cargo.toml is located)"); + } + Ok(()) +} + pub fn repo_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .parent() diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index a41a1bb3d02..c011dd38a9b 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -6,6 +6,24 @@ struct Command { name: &'static str, package: &'static str, } +struct CommandGroup { + name: &'static str, + commands: &'static [Command], +} +const OTHER_WORKFLOWS_COMMANDS: &[Command] = &[ + Command { + name: "coordinate-internal-tests", + package: "ci-coordinate-internal-tests", + }, + Command { + name: "codeowners-check", + package: "ci-codeowners-check", + }, + Command { + name: "cla-assistant", + package: "ci-cla-assistant", + }, +]; const COMMANDS: &[Command] = &[ Command { name: "test", @@ -63,11 +81,11 @@ const COMMANDS: &[Command] = &[ name: "docs", package: "ci-docs-build", }, - Command { - name: "other-workflows", - package: "ci-other-workflows", - }, ]; +const COMMAND_GROUPS: &[CommandGroup] = &[CommandGroup { + name: "other-workflows", + commands: OTHER_WORKFLOWS_COMMANDS, +}]; const DEFAULT_SKIP: &[&str] = &["other-workflows"]; fn print_help() { @@ -77,10 +95,16 @@ fn print_help() { for command in COMMANDS { println!(" {}", command.name); } + for group in COMMAND_GROUPS { + println!(" {}", group.name); + } } fn command_for(name: &str) -> Option<&'static Command> { COMMANDS.iter().find(|candidate| candidate.name == name) } +fn command_group_for(name: &str) -> Option<&'static CommandGroup> { + COMMAND_GROUPS.iter().find(|candidate| candidate.name == name) +} fn run_command(command: &Command, forwarded: &[String]) -> Result<()> { let mut cargo_args = vec!["run", "--package", command.package, "--"]; cargo_args.extend(forwarded.iter().map(String::as_str)); @@ -115,8 +139,30 @@ fn main() -> Result<()> { return Ok(()); } let command_name = raw.remove(0); + if let Some(group) = command_group_for(&command_name) { + return run_command_group(group, raw); + } let Some(command) = command_for(&command_name) else { bail!("unknown cargo ci command `{command_name}`"); }; run_command(command, &raw) } +fn run_command_group(group: &CommandGroup, mut forwarded: Vec) -> Result<()> { + if forwarded.first().is_some_and(|arg| arg == "-h" || arg == "--help") { + println!("Usage: cargo ci {} ", group.name); + println!(); + println!("Commands:"); + for command in group.commands { + println!(" {}", command.name); + } + return Ok(()); + } + if forwarded.is_empty() { + bail!("cargo ci {} requires a command name", group.name); + } + let command_name = forwarded.remove(0); + let Some(command) = group.commands.iter().find(|candidate| candidate.name == command_name) else { + bail!("unknown cargo ci {} command `{command_name}`", group.name); + }; + run_command(command, &forwarded) +} From 22a3096a6d201cfeddb964b60df0cf4f7ecb69f6 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 10:20:46 -0400 Subject: [PATCH 06/34] Address CI command review comments --- Cargo.lock | 16 +- Cargo.toml | 1 - tools/ci/README.md | 9 +- tools/ci/commands/cli-docs/Cargo.toml | 1 + tools/ci/commands/cli-docs/src/main.rs | 18 +- tools/ci/commands/docs/Cargo.toml | 1 + tools/ci/commands/docs/src/main.rs | 17 +- .../commands/global-json-policy/src/main.rs | 20 +- tools/ci/commands/lint/Cargo.toml | 1 + tools/ci/commands/lint/src/main.rs | 21 +-- tools/ci/commands/publish-checks/Cargo.toml | 1 - tools/ci/commands/publish-checks/src/main.rs | 1 - tools/ci/commands/self-docs/Cargo.toml | 9 - tools/ci/commands/self-docs/src/main.rs | 176 ------------------ tools/ci/commands/test/Cargo.toml | 1 + tools/ci/commands/test/src/main.rs | 18 +- tools/ci/commands/typescript-test/Cargo.toml | 1 + tools/ci/commands/typescript-test/src/main.rs | 18 +- tools/ci/commands/wasm-bindings/src/main.rs | 18 +- tools/ci/common/Cargo.toml | 1 + tools/ci/common/src/lib.rs | 17 ++ tools/ci/src/main.rs | 84 ++++++++- 22 files changed, 125 insertions(+), 325 deletions(-) delete mode 100644 tools/ci/commands/self-docs/Cargo.toml delete mode 100644 tools/ci/commands/self-docs/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 537a90a48f5..4e5b4ef1636 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -918,6 +918,7 @@ name = "ci-cli-docs" version = "0.1.0" dependencies = [ "anyhow", + "ci-common", "clap 4.5.50", "duct", "env_logger 0.10.2", @@ -941,6 +942,7 @@ name = "ci-common" version = "0.1.0" dependencies = [ "anyhow", + "duct", ] [[package]] @@ -968,6 +970,7 @@ name = "ci-docs-build" version = "0.1.0" dependencies = [ "anyhow", + "ci-common", "duct", "env_logger 0.10.2", ] @@ -1001,6 +1004,7 @@ name = "ci-lint" version = "0.1.0" dependencies = [ "anyhow", + "ci-common", "duct", "env_logger 0.10.2", "serde_json", @@ -1012,16 +1016,6 @@ version = "0.1.0" dependencies = [ "anyhow", "duct", - "env_logger 0.10.2", -] - -[[package]] -name = "ci-self-docs" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap 4.5.50", - "duct", ] [[package]] @@ -1041,6 +1035,7 @@ name = "ci-test" version = "0.1.0" dependencies = [ "anyhow", + "ci-common", "duct", "env_logger 0.10.2", ] @@ -1050,6 +1045,7 @@ name = "ci-typescript-test" version = "0.1.0" dependencies = [ "anyhow", + "ci-common", "duct", "env_logger 0.10.2", ] diff --git a/Cargo.toml b/Cargo.toml index c3a59b5c52f..cdcedaf8f98 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,6 @@ members = [ "tools/ci/commands/keynote-bench", "tools/ci/commands/update-flow", "tools/ci/commands/cli-docs", - "tools/ci/commands/self-docs", "tools/ci/commands/global-json-policy", "tools/ci/commands/publish-checks", "tools/ci/commands/typescript-test", diff --git a/tools/ci/README.md b/tools/ci/README.md index 3d22972e9e4..10b1b2a36ec 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -127,14 +127,7 @@ Options: **Usage:** ```bash -Usage: ci-self-docs [OPTIONS] - -Options: - --check - Only check for changes, do not generate the docs - - -h, --help - Print help (see a summary with '-h') +cargo ci self-docs [--check] ``` ### `global-json-policy` diff --git a/tools/ci/commands/cli-docs/Cargo.toml b/tools/ci/commands/cli-docs/Cargo.toml index 5e8dcee6dcd..a1331e00ce3 100644 --- a/tools/ci/commands/cli-docs/Cargo.toml +++ b/tools/ci/commands/cli-docs/Cargo.toml @@ -9,3 +9,4 @@ duct.workspace = true clap.workspace = true log.workspace = true env_logger.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/cli-docs/src/main.rs b/tools/ci/commands/cli-docs/src/main.rs index 14cae753286..71e9bfd78a9 100644 --- a/tools/ci/commands/cli-docs/src/main.rs +++ b/tools/ci/commands/cli-docs/src/main.rs @@ -1,22 +1,8 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; +use ci_common::pnpm; use clap::Parser; -use duct::{cmd, Expression}; -use std::ffi::OsStr; -fn pnpm(args: I) -> Expression -where - I: IntoIterator, - S: AsRef, -{ - let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); - if cfg!(windows) { - let mut full: Vec = vec!["/c".into(), "pnpm".into()]; - full.extend(args); - cmd("cmd", full) - } else { - cmd("pnpm", args) - } -} +use duct::cmd; use std::env; #[derive(Parser)] diff --git a/tools/ci/commands/docs/Cargo.toml b/tools/ci/commands/docs/Cargo.toml index 4bf39b2f08f..79563926460 100644 --- a/tools/ci/commands/docs/Cargo.toml +++ b/tools/ci/commands/docs/Cargo.toml @@ -7,3 +7,4 @@ edition.workspace = true anyhow.workspace = true duct.workspace = true env_logger.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/docs/src/main.rs b/tools/ci/commands/docs/src/main.rs index 76de2c90747..99ff34bb621 100644 --- a/tools/ci/commands/docs/src/main.rs +++ b/tools/ci/commands/docs/src/main.rs @@ -1,21 +1,6 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; -use duct::{cmd, Expression}; -use std::ffi::OsStr; -fn pnpm(args: I) -> Expression -where - I: IntoIterator, - S: AsRef, -{ - let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); - if cfg!(windows) { - let mut full: Vec = vec!["/c".into(), "pnpm".into()]; - full.extend(args); - cmd("cmd", full) - } else { - cmd("pnpm", args) - } -} +use ci_common::pnpm; fn run_docs_build() -> Result<()> { pnpm(["install"]).dir("docs").run()?; pnpm(["build"]).dir("docs").run()?; diff --git a/tools/ci/commands/global-json-policy/src/main.rs b/tools/ci/commands/global-json-policy/src/main.rs index 0aa152ebda0..4b53ecb9f59 100644 --- a/tools/ci/commands/global-json-policy/src/main.rs +++ b/tools/ci/commands/global-json-policy/src/main.rs @@ -2,31 +2,13 @@ #![allow(dead_code)] use anyhow::{bail, Context, Result}; -use duct::{cmd, Expression}; +use duct::cmd; use serde_json::Value; use std::collections::BTreeSet; use std::fs; use std::path::Path; use std::path::PathBuf; -/// On Windows, `pnpm` is installed as a `.cmd` shim which `CreateProcess` cannot -/// find without going through the shell. Wrapping with `cmd /c` fixes this. -/// On Unix, we invoke `pnpm` directly. -fn pnpm(args: I) -> Expression -where - I: IntoIterator, - S: AsRef, -{ - let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); - if cfg!(windows) { - let mut full: Vec = vec!["/c".into(), "pnpm".into()]; - full.extend(args); - cmd("cmd", full) - } else { - cmd("pnpm", args) - } -} - fn check_global_json_policy() -> Result<()> { ensure_repo_root()?; diff --git a/tools/ci/commands/lint/Cargo.toml b/tools/ci/commands/lint/Cargo.toml index ff2b8002952..119a056198f 100644 --- a/tools/ci/commands/lint/Cargo.toml +++ b/tools/ci/commands/lint/Cargo.toml @@ -8,3 +8,4 @@ anyhow.workspace = true duct.workspace = true serde_json.workspace = true env_logger.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/lint/src/main.rs b/tools/ci/commands/lint/src/main.rs index 190c014cd66..ab391143054 100644 --- a/tools/ci/commands/lint/src/main.rs +++ b/tools/ci/commands/lint/src/main.rs @@ -2,7 +2,8 @@ #![allow(dead_code)] use anyhow::{bail, Context, Result}; -use duct::{cmd, Expression}; +use ci_common::pnpm; +use duct::cmd; use serde_json::Value; use std::collections::BTreeSet; use std::ffi::OsString; @@ -10,24 +11,6 @@ use std::fs; use std::path::Path; use std::path::PathBuf; -/// On Windows, `pnpm` is installed as a `.cmd` shim which `CreateProcess` cannot -/// find without going through the shell. Wrapping with `cmd /c` fixes this. -/// On Unix, we invoke `pnpm` directly. -fn pnpm(args: I) -> Expression -where - I: IntoIterator, - S: AsRef, -{ - let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); - if cfg!(windows) { - let mut full: Vec = vec!["/c".into(), "pnpm".into()]; - full.extend(args); - cmd("cmd", full) - } else { - cmd("pnpm", args) - } -} - fn check_global_json_policy() -> Result<()> { ensure_repo_root()?; diff --git a/tools/ci/commands/publish-checks/Cargo.toml b/tools/ci/commands/publish-checks/Cargo.toml index 67093422620..5c701a42b45 100644 --- a/tools/ci/commands/publish-checks/Cargo.toml +++ b/tools/ci/commands/publish-checks/Cargo.toml @@ -6,4 +6,3 @@ edition.workspace = true [dependencies] anyhow.workspace = true duct.workspace = true -env_logger.workspace = true diff --git a/tools/ci/commands/publish-checks/src/main.rs b/tools/ci/commands/publish-checks/src/main.rs index 83f6315d4b2..c9dad1b59f7 100644 --- a/tools/ci/commands/publish-checks/src/main.rs +++ b/tools/ci/commands/publish-checks/src/main.rs @@ -32,6 +32,5 @@ fn run_publish_checks() -> Result<()> { } fn main() -> Result<()> { - env_logger::init(); run_publish_checks() } diff --git a/tools/ci/commands/self-docs/Cargo.toml b/tools/ci/commands/self-docs/Cargo.toml deleted file mode 100644 index 382aed26d67..00000000000 --- a/tools/ci/commands/self-docs/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "ci-self-docs" -version = "0.1.0" -edition.workspace = true - -[dependencies] -anyhow.workspace = true -duct.workspace = true -clap.workspace = true diff --git a/tools/ci/commands/self-docs/src/main.rs b/tools/ci/commands/self-docs/src/main.rs deleted file mode 100644 index 60572539482..00000000000 --- a/tools/ci/commands/self-docs/src/main.rs +++ /dev/null @@ -1,176 +0,0 @@ -#![allow(clippy::disallowed_macros)] -use anyhow::{bail, Result}; -use clap::Parser; -use duct::cmd; -use std::fs; -use std::path::Path; - -#[derive(Parser)] -struct Args { - #[arg( - long, - default_value_t = false, - long_help = "Only check for changes, do not generate the docs" - )] - check: bool, -} -struct CommandDoc { - name: &'static str, - package: &'static str, - help_safe: bool, -} -struct CommandGroupDoc { - name: &'static str, - commands: &'static [CommandDoc], -} - -const OTHER_WORKFLOWS_COMMANDS: &[CommandDoc] = &[ - CommandDoc { - name: "coordinate-internal-tests", - package: "ci-coordinate-internal-tests", - help_safe: true, - }, - CommandDoc { - name: "codeowners-check", - package: "ci-codeowners-check", - help_safe: true, - }, - CommandDoc { - name: "cla-assistant", - package: "ci-cla-assistant", - help_safe: true, - }, -]; - -const COMMAND_GROUPS: &[CommandGroupDoc] = &[CommandGroupDoc { - name: "other-workflows", - commands: OTHER_WORKFLOWS_COMMANDS, -}]; - -const ALL_COMMANDS: &[CommandDoc] = &[ - CommandDoc { - name: "test", - package: "ci-test", - help_safe: false, - }, - CommandDoc { - name: "lint", - package: "ci-lint", - help_safe: false, - }, - CommandDoc { - name: "wasm-bindings", - package: "ci-wasm-bindings", - help_safe: false, - }, - CommandDoc { - name: "dlls", - package: "ci-dlls", - help_safe: false, - }, - CommandDoc { - name: "smoketests", - package: "ci-smoketests", - help_safe: true, - }, - CommandDoc { - name: "keynote-bench", - package: "ci-keynote-bench", - help_safe: false, - }, - CommandDoc { - name: "update-flow", - package: "ci-update-flow", - help_safe: true, - }, - CommandDoc { - name: "cli-docs", - package: "ci-cli-docs", - help_safe: true, - }, - CommandDoc { - name: "self-docs", - package: "ci-self-docs", - help_safe: true, - }, - CommandDoc { - name: "global-json-policy", - package: "ci-global-json-policy", - help_safe: false, - }, - CommandDoc { - name: "publish-checks", - package: "ci-publish-checks", - help_safe: false, - }, - CommandDoc { - name: "typescript-test", - package: "ci-typescript-test", - help_safe: false, - }, - CommandDoc { - name: "version-upgrade-check", - package: "ci-version-upgrade-check", - help_safe: false, - }, - CommandDoc { - name: "docs", - package: "ci-docs-build", - help_safe: false, - }, - CommandDoc { - name: "other-workflows", - package: "", - help_safe: false, - }, -]; - -fn normalize_help(help: &str) -> String { - help.lines().map(str::trim_end).collect::>().join("\n") -} - -fn command_usage(command: &CommandDoc) -> Result { - if command.help_safe { - let help = cmd!("cargo", "run", "--quiet", "--package", command.package, "--", "--help").read()?; - return Ok(normalize_help(&help)); - } - if command.package.is_empty() { - return Ok(format!("cargo ci {} ", command.name)); - } - Ok(format!("cargo ci {}", command.name)) -} -fn generate() -> Result { - let mut out = String::from("# SpacetimeDB's cargo ci\n\n## Overview\n\nThis document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options.\n\n## `cargo ci`\n\n**Usage:**\n```bash\ncargo ci [--skip ] [COMMAND] [ARGS]...\n```\n\n"); - for command in ALL_COMMANDS { - out.push_str(&format!( - "### `{}`\n\n**Usage:**\n```bash\n{}\n```\n\n", - command.name, - command_usage(command)?.trim(), - )); - if let Some(group) = COMMAND_GROUPS.iter().find(|group| group.name == command.name) { - for subcommand in group.commands { - out.push_str(&format!( - "#### `{}`\n\n**Usage:**\n```bash\n{}\n```\n\n", - subcommand.name, - command_usage(subcommand)?.trim(), - )); - } - } - } - out.push_str("---\n\nThis document is auto-generated by running:\n\n```bash\ncargo ci self-docs\n```\n"); - Ok(out) -} -fn main() -> Result<()> { - let args = Args::parse(); - let readme = generate()?; - let path = Path::new("tools/ci/README.md"); - if args.check { - let existing = fs::read_to_string(path).unwrap_or_default(); - if existing != readme { - bail!("README.md is out of date. Please run `cargo ci self-docs` to update it."); - } - } else { - fs::write(path, readme)?; - } - Ok(()) -} diff --git a/tools/ci/commands/test/Cargo.toml b/tools/ci/commands/test/Cargo.toml index e714cc869fc..547d7f7d0a2 100644 --- a/tools/ci/commands/test/Cargo.toml +++ b/tools/ci/commands/test/Cargo.toml @@ -7,3 +7,4 @@ edition.workspace = true anyhow.workspace = true duct.workspace = true env_logger.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/test/src/main.rs b/tools/ci/commands/test/src/main.rs index 117d937e4ae..207c053b171 100644 --- a/tools/ci/commands/test/src/main.rs +++ b/tools/ci/commands/test/src/main.rs @@ -1,21 +1,7 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; -use duct::{cmd, Expression}; -use std::ffi::OsStr; -fn pnpm(args: I) -> Expression -where - I: IntoIterator, - S: AsRef, -{ - let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); - if cfg!(windows) { - let mut full: Vec = vec!["/c".into(), "pnpm".into()]; - full.extend(args); - cmd("cmd", full) - } else { - cmd("pnpm", args) - } -} +use ci_common::pnpm; +use duct::cmd; fn main() -> Result<()> { env_logger::init(); pnpm(["build"]).dir("crates/bindings-typescript").run()?; diff --git a/tools/ci/commands/typescript-test/Cargo.toml b/tools/ci/commands/typescript-test/Cargo.toml index 565466bcbe2..3ff4ac8106e 100644 --- a/tools/ci/commands/typescript-test/Cargo.toml +++ b/tools/ci/commands/typescript-test/Cargo.toml @@ -7,3 +7,4 @@ edition.workspace = true anyhow.workspace = true duct.workspace = true env_logger.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/typescript-test/src/main.rs b/tools/ci/commands/typescript-test/src/main.rs index 74dbe2c7770..61acea5c338 100644 --- a/tools/ci/commands/typescript-test/src/main.rs +++ b/tools/ci/commands/typescript-test/src/main.rs @@ -1,21 +1,7 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Result}; -use duct::{cmd, Expression}; -use std::ffi::OsStr; -fn pnpm(args: I) -> Expression -where - I: IntoIterator, - S: AsRef, -{ - let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); - if cfg!(windows) { - let mut full: Vec = vec!["/c".into(), "pnpm".into()]; - full.extend(args); - cmd("cmd", full) - } else { - cmd("pnpm", args) - } -} +use ci_common::pnpm; +use duct::cmd; fn run_typescript_tests() -> Result<()> { pnpm(["build"]).dir("crates/bindings-typescript").run()?; pnpm(["test"]).dir("crates/bindings-typescript").run()?; diff --git a/tools/ci/commands/wasm-bindings/src/main.rs b/tools/ci/commands/wasm-bindings/src/main.rs index 96a803e55ba..5757076f019 100644 --- a/tools/ci/commands/wasm-bindings/src/main.rs +++ b/tools/ci/commands/wasm-bindings/src/main.rs @@ -1,21 +1,7 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; -use duct::{cmd, Expression}; -use std::ffi::OsStr; -fn pnpm(args: I) -> Expression -where - I: IntoIterator, - S: AsRef, -{ - let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); - if cfg!(windows) { - let mut full: Vec = vec!["/c".into(), "pnpm".into()]; - full.extend(args); - cmd("cmd", full) - } else { - cmd("pnpm", args) - } -} +use ci_common::pnpm; +use duct::cmd; fn main() -> Result<()> { env_logger::init(); pnpm([ diff --git a/tools/ci/common/Cargo.toml b/tools/ci/common/Cargo.toml index 99fdd485443..2e45f5dbe66 100644 --- a/tools/ci/common/Cargo.toml +++ b/tools/ci/common/Cargo.toml @@ -5,3 +5,4 @@ edition.workspace = true [dependencies] anyhow.workspace = true +duct.workspace = true diff --git a/tools/ci/common/src/lib.rs b/tools/ci/common/src/lib.rs index a14f5ebf06d..4dfafe8e03b 100644 --- a/tools/ci/common/src/lib.rs +++ b/tools/ci/common/src/lib.rs @@ -1,4 +1,6 @@ use anyhow::{bail, Result}; +use duct::{cmd, Expression}; +use std::ffi::OsStr; use std::path::Path; use std::path::PathBuf; @@ -17,3 +19,18 @@ pub fn repo_root() -> PathBuf { .expect("failed to find repo root") .to_path_buf() } + +pub fn pnpm(args: I) -> Expression +where + I: IntoIterator, + S: AsRef, +{ + let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); + if cfg!(windows) { + let mut full: Vec = vec!["/c".into(), "pnpm".into()]; + full.extend(args); + cmd("cmd", full) + } else { + cmd("pnpm", args) + } +} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index c011dd38a9b..60f59d0b144 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,6 +1,8 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Result}; use duct::cmd; +use std::fs; +use std::path::Path; struct Command { name: &'static str, @@ -59,7 +61,7 @@ const COMMANDS: &[Command] = &[ }, Command { name: "self-docs", - package: "ci-self-docs", + package: "", }, Command { name: "global-json-policy", @@ -106,6 +108,9 @@ fn command_group_for(name: &str) -> Option<&'static CommandGroup> { COMMAND_GROUPS.iter().find(|candidate| candidate.name == name) } fn run_command(command: &Command, forwarded: &[String]) -> Result<()> { + if command.name == "self-docs" { + return run_self_docs(forwarded); + } let mut cargo_args = vec!["run", "--package", command.package, "--"]; cargo_args.extend(forwarded.iter().map(String::as_str)); cmd("cargo", cargo_args).run()?; @@ -166,3 +171,80 @@ fn run_command_group(group: &CommandGroup, mut forwarded: Vec) -> Result }; run_command(command, &forwarded) } + +fn run_self_docs(args: &[String]) -> Result<()> { + if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { + println!("Usage: cargo ci self-docs [--check]"); + println!(); + println!("Options:"); + println!(" --check Only check for changes, do not generate the docs"); + println!(" -h, --help Print help"); + return Ok(()); + } + + let check = match args { + [] => false, + [arg] if arg == "--check" => true, + _ => bail!("cargo ci self-docs accepts only --check"), + }; + + let readme = generate_cli_docs()?; + let path = Path::new("tools/ci/README.md"); + if check { + let existing = fs::read_to_string(path).unwrap_or_default(); + if existing != readme { + bail!("README.md is out of date. Please run `cargo ci self-docs` to update it."); + } + } else { + fs::write(path, readme)?; + } + Ok(()) +} + +fn normalize_help(help: &str) -> String { + help.lines().map(str::trim_end).collect::>().join("\n") +} + +fn command_help_safe(name: &str) -> bool { + matches!( + name, + "smoketests" | "update-flow" | "cli-docs" | "codeowners-check" | "coordinate-internal-tests" | "cla-assistant" + ) +} + +fn command_usage(command: &Command) -> Result { + if command.name == "self-docs" { + return Ok("cargo ci self-docs [--check]".to_owned()); + } + if command_help_safe(command.name) { + let help = cmd!("cargo", "run", "--quiet", "--package", command.package, "--", "--help").read()?; + return Ok(normalize_help(&help)); + } + Ok(format!("cargo ci {}", command.name)) +} + +fn generate_cli_docs() -> Result { + let mut out = String::from("# SpacetimeDB's cargo ci\n\n## Overview\n\nThis document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options.\n\n## `cargo ci`\n\n**Usage:**\n```bash\ncargo ci [--skip ] [COMMAND] [ARGS]...\n```\n\n"); + for command in COMMANDS { + out.push_str(&format!( + "### `{}`\n\n**Usage:**\n```bash\n{}\n```\n\n", + command.name, + command_usage(command)?.trim(), + )); + } + for group in COMMAND_GROUPS { + out.push_str(&format!( + "### `{}`\n\n**Usage:**\n```bash\ncargo ci {} \n```\n\n", + group.name, group.name, + )); + for command in group.commands { + out.push_str(&format!( + "#### `{}`\n\n**Usage:**\n```bash\n{}\n```\n\n", + command.name, + command_usage(command)?.trim(), + )); + } + } + out.push_str("---\n\nThis document is auto-generated by running:\n\n```bash\ncargo ci self-docs\n```\n"); + Ok(out) +} From ffcaa080d05420a50241f2676cafd5314eb3a6af Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 10:23:31 -0400 Subject: [PATCH 07/34] Finish CI command review fixes --- Cargo.lock | 5 ---- tools/ci/README.md | 29 +++++++++++++++++++ tools/ci/commands/test/Cargo.toml | 1 - tools/ci/commands/test/src/main.rs | 1 - tools/ci/commands/update-flow/Cargo.toml | 2 -- tools/ci/commands/update-flow/src/main.rs | 4 --- .../commands/version-upgrade-check/Cargo.toml | 1 - .../version-upgrade-check/src/main.rs | 1 - tools/ci/commands/wasm-bindings/Cargo.toml | 1 - tools/ci/commands/wasm-bindings/src/main.rs | 1 - tools/ci/src/main.rs | 20 +++++++++++++ 11 files changed, 49 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4e5b4ef1636..b02b6bc9587 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1037,7 +1037,6 @@ dependencies = [ "anyhow", "ci-common", "duct", - "env_logger 0.10.2", ] [[package]] @@ -1057,8 +1056,6 @@ dependencies = [ "anyhow", "clap 4.5.50", "duct", - "env_logger 0.10.2", - "log", "tempfile", ] @@ -1068,7 +1065,6 @@ version = "0.1.0" dependencies = [ "anyhow", "duct", - "env_logger 0.10.2", ] [[package]] @@ -1078,7 +1074,6 @@ dependencies = [ "anyhow", "ci-common", "duct", - "env_logger 0.10.2", ] [[package]] diff --git a/tools/ci/README.md b/tools/ci/README.md index 10b1b2a36ec..f1a8477a881 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -216,6 +216,35 @@ Options: -h, --help Print help ``` +##### `retry` + +**Usage:** +```bash +Retries CLA Assistant if `license/cla` is the only remaining PR blocker + +Usage: ci-cla-assistant retry [OPTIONS] --pr-number + +Options: + --pr-number Pull request number to check + --repo Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY + -h, --help Print help +``` + +##### `status` + +**Usage:** +```bash +Returns the `license/cla` status for a pull request or commit SHA + +Usage: ci-cla-assistant status [OPTIONS] <--pr |--sha > + +Options: + --pr Pull request number whose head commit should be checked + --sha Commit SHA to check + --repo Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY + -h, --help Print help +``` + --- This document is auto-generated by running: diff --git a/tools/ci/commands/test/Cargo.toml b/tools/ci/commands/test/Cargo.toml index 547d7f7d0a2..a81c71dff76 100644 --- a/tools/ci/commands/test/Cargo.toml +++ b/tools/ci/commands/test/Cargo.toml @@ -6,5 +6,4 @@ edition.workspace = true [dependencies] anyhow.workspace = true duct.workspace = true -env_logger.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/test/src/main.rs b/tools/ci/commands/test/src/main.rs index 207c053b171..73cd6d145e7 100644 --- a/tools/ci/commands/test/src/main.rs +++ b/tools/ci/commands/test/src/main.rs @@ -3,7 +3,6 @@ use anyhow::Result; use ci_common::pnpm; use duct::cmd; fn main() -> Result<()> { - env_logger::init(); pnpm(["build"]).dir("crates/bindings-typescript").run()?; // TODO: This doesn't work on at least user Linux machines, because something here apparently uses `sudo`? diff --git a/tools/ci/commands/update-flow/Cargo.toml b/tools/ci/commands/update-flow/Cargo.toml index e7ce2e6b461..590c8305711 100644 --- a/tools/ci/commands/update-flow/Cargo.toml +++ b/tools/ci/commands/update-flow/Cargo.toml @@ -8,5 +8,3 @@ anyhow.workspace = true duct.workspace = true tempfile.workspace = true clap.workspace = true -log.workspace = true -env_logger.workspace = true diff --git a/tools/ci/commands/update-flow/src/main.rs b/tools/ci/commands/update-flow/src/main.rs index 8a809c42feb..f27c8fafe0b 100644 --- a/tools/ci/commands/update-flow/src/main.rs +++ b/tools/ci/commands/update-flow/src/main.rs @@ -19,7 +19,6 @@ struct Args { } fn main() -> Result<()> { - env_logger::init(); let Args { target, github_token_auth, @@ -28,9 +27,6 @@ fn main() -> Result<()> { if let Some(target) = target.as_ref() { common_args.push("--target"); common_args.push(target); - log::info!("checking update flow for target: {target}"); - } else { - log::info!("checking update flow"); } if github_token_auth { common_args.push("--features"); diff --git a/tools/ci/commands/version-upgrade-check/Cargo.toml b/tools/ci/commands/version-upgrade-check/Cargo.toml index 68dec2068e4..03332a62469 100644 --- a/tools/ci/commands/version-upgrade-check/Cargo.toml +++ b/tools/ci/commands/version-upgrade-check/Cargo.toml @@ -6,4 +6,3 @@ edition.workspace = true [dependencies] anyhow.workspace = true duct.workspace = true -env_logger.workspace = true diff --git a/tools/ci/commands/version-upgrade-check/src/main.rs b/tools/ci/commands/version-upgrade-check/src/main.rs index fed14c6bed3..c4bbd3c4905 100644 --- a/tools/ci/commands/version-upgrade-check/src/main.rs +++ b/tools/ci/commands/version-upgrade-check/src/main.rs @@ -17,6 +17,5 @@ fn run_version_upgrade_check() -> Result<()> { } fn main() -> Result<()> { - env_logger::init(); run_version_upgrade_check() } diff --git a/tools/ci/commands/wasm-bindings/Cargo.toml b/tools/ci/commands/wasm-bindings/Cargo.toml index de3fd439244..4f6d571f1f9 100644 --- a/tools/ci/commands/wasm-bindings/Cargo.toml +++ b/tools/ci/commands/wasm-bindings/Cargo.toml @@ -6,5 +6,4 @@ edition.workspace = true [dependencies] anyhow.workspace = true duct.workspace = true -env_logger.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/wasm-bindings/src/main.rs b/tools/ci/commands/wasm-bindings/src/main.rs index 5757076f019..121fd5d7225 100644 --- a/tools/ci/commands/wasm-bindings/src/main.rs +++ b/tools/ci/commands/wasm-bindings/src/main.rs @@ -3,7 +3,6 @@ use anyhow::Result; use ci_common::pnpm; use duct::cmd; fn main() -> Result<()> { - env_logger::init(); pnpm([ "install", "--filter", diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 60f59d0b144..f9653d405d5 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -223,6 +223,17 @@ fn command_usage(command: &Command) -> Result { Ok(format!("cargo ci {}", command.name)) } +fn package_usage(package: &str, args: &[&str]) -> Result { + let help = cmd( + "cargo", + ["run", "--quiet", "--package", package, "--"] + .into_iter() + .chain(args.iter().copied()), + ) + .read()?; + Ok(normalize_help(&help)) +} + fn generate_cli_docs() -> Result { let mut out = String::from("# SpacetimeDB's cargo ci\n\n## Overview\n\nThis document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options.\n\n## `cargo ci`\n\n**Usage:**\n```bash\ncargo ci [--skip ] [COMMAND] [ARGS]...\n```\n\n"); for command in COMMANDS { @@ -243,6 +254,15 @@ fn generate_cli_docs() -> Result { command.name, command_usage(command)?.trim(), )); + if command.name == "cla-assistant" { + for subcommand in ["retry", "status"] { + out.push_str(&format!( + "##### `{}`\n\n**Usage:**\n```bash\n{}\n```\n\n", + subcommand, + package_usage(command.package, &[subcommand, "--help"])?.trim(), + )); + } + } } } out.push_str("---\n\nThis document is auto-generated by running:\n\n```bash\ncargo ci self-docs\n```\n"); From 7da213052b003bdca8b08e079ae02854da9633a7 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 10:25:07 -0400 Subject: [PATCH 08/34] Inline CI dlls command --- Cargo.lock | 10 ---------- Cargo.toml | 1 - tools/ci/commands/dlls/Cargo.toml | 9 --------- tools/ci/commands/dlls/src/main.rs | 10 ---------- tools/ci/commands/docs/Cargo.toml | 1 - tools/ci/commands/docs/src/main.rs | 1 - tools/ci/src/main.rs | 18 +++++++++++++++++- 7 files changed, 17 insertions(+), 33 deletions(-) delete mode 100644 tools/ci/commands/dlls/Cargo.toml delete mode 100644 tools/ci/commands/dlls/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index b02b6bc9587..87769f553b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -956,15 +956,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "ci-dlls" -version = "0.1.0" -dependencies = [ - "anyhow", - "duct", - "env_logger 0.10.2", -] - [[package]] name = "ci-docs-build" version = "0.1.0" @@ -972,7 +963,6 @@ dependencies = [ "anyhow", "ci-common", "duct", - "env_logger 0.10.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index cdcedaf8f98..a1a94f15fc3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,7 +71,6 @@ members = [ "tools/ci/commands/test", "tools/ci/commands/lint", "tools/ci/commands/wasm-bindings", - "tools/ci/commands/dlls", "tools/ci/commands/smoketests", "tools/ci/commands/keynote-bench", "tools/ci/commands/update-flow", diff --git a/tools/ci/commands/dlls/Cargo.toml b/tools/ci/commands/dlls/Cargo.toml deleted file mode 100644 index 7ade3c64682..00000000000 --- a/tools/ci/commands/dlls/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "ci-dlls" -version = "0.1.0" -edition.workspace = true - -[dependencies] -anyhow.workspace = true -duct.workspace = true -env_logger.workspace = true diff --git a/tools/ci/commands/dlls/src/main.rs b/tools/ci/commands/dlls/src/main.rs deleted file mode 100644 index fb31e24d1b0..00000000000 --- a/tools/ci/commands/dlls/src/main.rs +++ /dev/null @@ -1,10 +0,0 @@ -#![allow(clippy::disallowed_macros)] -use anyhow::Result; -use duct::cmd; -fn main() -> Result<()> { - env_logger::init(); - eprintln!("warning: `cargo ci dlls` is deprecated; use `cargo regen csharp dlls` instead"); - cmd!("cargo", "regen", "csharp", "dlls").run()?; - - Ok(()) -} diff --git a/tools/ci/commands/docs/Cargo.toml b/tools/ci/commands/docs/Cargo.toml index 79563926460..50a98fbab5e 100644 --- a/tools/ci/commands/docs/Cargo.toml +++ b/tools/ci/commands/docs/Cargo.toml @@ -6,5 +6,4 @@ edition.workspace = true [dependencies] anyhow.workspace = true duct.workspace = true -env_logger.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/docs/src/main.rs b/tools/ci/commands/docs/src/main.rs index 99ff34bb621..6fd72a3ad84 100644 --- a/tools/ci/commands/docs/src/main.rs +++ b/tools/ci/commands/docs/src/main.rs @@ -8,6 +8,5 @@ fn run_docs_build() -> Result<()> { } fn main() -> Result<()> { - env_logger::init(); run_docs_build() } diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index f9653d405d5..5c3f9b9378e 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -41,7 +41,7 @@ const COMMANDS: &[Command] = &[ }, Command { name: "dlls", - package: "ci-dlls", + package: "", }, Command { name: "smoketests", @@ -111,6 +111,9 @@ fn run_command(command: &Command, forwarded: &[String]) -> Result<()> { if command.name == "self-docs" { return run_self_docs(forwarded); } + if command.name == "dlls" { + return run_dlls(forwarded); + } let mut cargo_args = vec!["run", "--package", command.package, "--"]; cargo_args.extend(forwarded.iter().map(String::as_str)); cmd("cargo", cargo_args).run()?; @@ -201,6 +204,19 @@ fn run_self_docs(args: &[String]) -> Result<()> { Ok(()) } +fn run_dlls(args: &[String]) -> Result<()> { + if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { + println!("Usage: cargo ci dlls"); + return Ok(()); + } + if !args.is_empty() { + bail!("cargo ci dlls does not accept arguments"); + } + eprintln!("warning: `cargo ci dlls` is deprecated; use `cargo regen csharp dlls` instead"); + cmd!("cargo", "regen", "csharp", "dlls").run()?; + Ok(()) +} + fn normalize_help(help: &str) -> String { help.lines().map(str::trim_end).collect::>().join("\n") } From 0848cd3de3535e9171eea80f813311b33370f70c Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 10:34:11 -0400 Subject: [PATCH 09/34] Prune unused CI command deps --- Cargo.lock | 9 - tools/ci/commands/docs/Cargo.toml | 1 - .../ci/commands/global-json-policy/Cargo.toml | 6 - .../commands/global-json-policy/src/main.rs | 288 +----------------- tools/ci/commands/lint/Cargo.toml | 1 - tools/ci/commands/lint/src/main.rs | 1 - tools/ci/commands/typescript-test/Cargo.toml | 1 - tools/ci/commands/typescript-test/src/main.rs | 1 - 8 files changed, 14 insertions(+), 294 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87769f553b0..1f7cdb4465a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -962,7 +962,6 @@ version = "0.1.0" dependencies = [ "anyhow", "ci-common", - "duct", ] [[package]] @@ -970,13 +969,7 @@ name = "ci-global-json-policy" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.5.50", "duct", - "env_logger 0.10.2", - "log", - "regex", - "serde_json", - "tempfile", ] [[package]] @@ -996,7 +989,6 @@ dependencies = [ "anyhow", "ci-common", "duct", - "env_logger 0.10.2", "serde_json", ] @@ -1036,7 +1028,6 @@ dependencies = [ "anyhow", "ci-common", "duct", - "env_logger 0.10.2", ] [[package]] diff --git a/tools/ci/commands/docs/Cargo.toml b/tools/ci/commands/docs/Cargo.toml index 50a98fbab5e..bcd15b81605 100644 --- a/tools/ci/commands/docs/Cargo.toml +++ b/tools/ci/commands/docs/Cargo.toml @@ -5,5 +5,4 @@ edition.workspace = true [dependencies] anyhow.workspace = true -duct.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/global-json-policy/Cargo.toml b/tools/ci/commands/global-json-policy/Cargo.toml index af85fbba7b6..0903a7c3478 100644 --- a/tools/ci/commands/global-json-policy/Cargo.toml +++ b/tools/ci/commands/global-json-policy/Cargo.toml @@ -6,9 +6,3 @@ edition.workspace = true [dependencies] anyhow.workspace = true duct.workspace = true -clap.workspace = true -regex.workspace = true -serde_json.workspace = true -tempfile.workspace = true -env_logger.workspace = true -log.workspace = true diff --git a/tools/ci/commands/global-json-policy/src/main.rs b/tools/ci/commands/global-json-policy/src/main.rs index 4b53ecb9f59..647be569f64 100644 --- a/tools/ci/commands/global-json-policy/src/main.rs +++ b/tools/ci/commands/global-json-policy/src/main.rs @@ -1,13 +1,21 @@ #![allow(clippy::disallowed_macros)] -#![allow(dead_code)] -use anyhow::{bail, Context, Result}; +use anyhow::{bail, Result}; use duct::cmd; -use serde_json::Value; -use std::collections::BTreeSet; use std::fs; -use std::path::Path; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; + +fn ensure_repo_root() -> Result<()> { + if !Path::new("Cargo.toml").exists() { + bail!("You must execute this command from the SpacetimeDB repository root (where Cargo.toml is located)"); + } + Ok(()) +} + +fn git_tracked_files(pathspec: &str) -> Result> { + let output = cmd!("git", "ls-files", pathspec).read()?; + Ok(output.lines().map(PathBuf::from).collect()) +} fn check_global_json_policy() -> Result<()> { ensure_repo_root()?; @@ -46,274 +54,6 @@ fn check_global_json_policy() -> Result<()> { Ok(()) } -fn package_json_pnpm_version(package_manager: &str) -> Option<&str> { - package_manager.strip_prefix("pnpm@") -} - -fn git_tracked_files(pathspec: &str) -> Result> { - let output = cmd!("git", "ls-files", pathspec).read()?; - Ok(output.lines().map(PathBuf::from).collect()) -} - -fn package_json_string_value(package_json: &Value, key: &str) -> Option { - package_json.get(key)?.as_str().map(str::to_owned) -} - -fn package_json_engines_pnpm(package_json: &Value) -> Option { - package_json.get("engines")?.get("pnpm")?.as_str().map(str::to_owned) -} - -fn read_package_json(path: &Path) -> Result { - let contents = fs::read_to_string(path)?; - Ok(serde_json::from_str(&contents)?) -} - -fn is_npm_package_json(package_json: &Value) -> bool { - [ - "bin", - "dependencies", - "devDependencies", - "exports", - "main", - "optionalDependencies", - "packageManager", - "peerDependencies", - "scripts", - "type", - ] - .iter() - .any(|key| package_json.get(key).is_some()) -} - -fn is_template_path(path: &Path) -> bool { - path.starts_with("templates") -} - -fn minimum_release_age(path: &Path) -> Result { - let workspace = fs::read_to_string(path)?; - workspace - .lines() - .find_map(|line| { - let line = line.trim(); - let value = line.strip_prefix("minimumReleaseAge:")?.trim(); - value.parse::().ok() - }) - .ok_or_else(|| anyhow::anyhow!("{} is missing minimumReleaseAge", path.display())) -} - -fn npmrc_minimum_release_age(path: &Path, expected_minimum_release_age: u64) -> Result { - let contents = fs::read_to_string(path).map_err(|err| { - if err.kind() == std::io::ErrorKind::NotFound { - anyhow::anyhow!( - "{} is tracked but missing from the working tree. Restore it with:\nminimum-release-age={}", - path.display(), - expected_minimum_release_age - ) - } else { - anyhow::anyhow!( - "failed to read {} while checking pnpm minimum package age: {err}", - path.display() - ) - } - })?; - contents - .lines() - .find_map(|line| { - let line = line.trim(); - let value = line.strip_prefix("minimum-release-age=")?.trim(); - value.parse::().ok() - }) - .ok_or_else(|| { - anyhow::anyhow!( - "{} must contain `minimum-release-age={}` to match root pnpm-workspace.yaml", - path.display(), - expected_minimum_release_age - ) - }) -} - -fn check_pnpm_release_age_policy() -> Result<()> { - ensure_repo_root()?; - - let root_package_json_path = Path::new("package.json"); - let root_package_json = read_package_json(root_package_json_path)?; - let package_manager = package_json_string_value(&root_package_json, "packageManager") - .ok_or_else(|| anyhow::anyhow!("package.json is missing packageManager"))?; - let package_manager_version = package_json_pnpm_version(&package_manager) - .ok_or_else(|| anyhow::anyhow!("packageManager must be pnpm@, found {package_manager:?}"))?; - - let expected_engine_pnpm = format!(">={package_manager_version}"); - let engine_pnpm = package_json_engines_pnpm(&root_package_json) - .ok_or_else(|| anyhow::anyhow!("package.json engines is missing pnpm"))?; - if engine_pnpm != expected_engine_pnpm { - bail!("package.json engines.pnpm must be {expected_engine_pnpm:?}, found {engine_pnpm:?}"); - } - - for package_json_path in git_tracked_files(":(glob)**/package.json")? { - let package_json = read_package_json(&package_json_path)?; - let Some(found_package_manager) = package_json_string_value(&package_json, "packageManager") else { - continue; - }; - if found_package_manager != package_manager { - bail!( - "{} packageManager must match root package.json: expected {:?}, found {:?}", - package_json_path.display(), - package_manager, - found_package_manager - ); - } - } - - let root_workspace_path = Path::new("pnpm-workspace.yaml"); - let root_minimum_release_age = minimum_release_age(root_workspace_path)?; - for workspace_path in git_tracked_files(":(glob)**/pnpm-workspace.yaml")? { - let found_minimum_release_age = minimum_release_age(&workspace_path)?; - if found_minimum_release_age != root_minimum_release_age { - bail!( - "{} minimumReleaseAge must match root pnpm-workspace.yaml: expected {}, found {}", - workspace_path.display(), - root_minimum_release_age, - found_minimum_release_age - ); - } - } - - for npmrc_path in git_tracked_files(":(glob)**/.npmrc")? { - // Template package roots are copied into projects created by `spacetime init`. - // They must not embed this repo's package-age policy; smoketests enforce it - // at the pnpm process boundary instead. - if is_template_path(&npmrc_path) { - continue; - } - let found_minimum_release_age = npmrc_minimum_release_age(&npmrc_path, root_minimum_release_age)?; - if found_minimum_release_age != root_minimum_release_age { - bail!( - "{} minimum-release-age must match root pnpm-workspace.yaml: expected {}, found {}", - npmrc_path.display(), - root_minimum_release_age, - found_minimum_release_age - ); - } - } - - for package_json_path in git_tracked_files(":(glob)**/package.json")? { - // Template package roots are copied into projects created by `spacetime init`. - // They must not require adjacent .npmrc files for this repo's package-age - // policy; smoketests enforce it at the pnpm process boundary instead. - if is_template_path(&package_json_path) { - continue; - } - let package_json = read_package_json(&package_json_path)?; - if !is_npm_package_json(&package_json) { - continue; - } - let package_dir = package_json_path - .parent() - .expect("git-tracked package.json path should have a parent"); - let npmrc_path = package_dir.join(".npmrc"); - if !npmrc_path.is_file() { - bail!( - "{} is required because {} is an npm/pnpm package manifest.\nAdd {} containing:\nminimum-release-age={}", - npmrc_path.display(), - package_json_path.display(), - npmrc_path.display(), - root_minimum_release_age - ); - } - let found_minimum_release_age = npmrc_minimum_release_age(&npmrc_path, root_minimum_release_age)?; - if found_minimum_release_age != root_minimum_release_age { - bail!( - "{} minimum-release-age must match root pnpm-workspace.yaml: expected {}, found {}", - npmrc_path.display(), - root_minimum_release_age, - found_minimum_release_age - ); - } - } - - for workflow_path in git_tracked_files(".github/workflows/*")? { - let contents = fs::read_to_string(&workflow_path)?; - if contents.contains("pnpm/action-setup@v4") { - bail!( - "{} must use ./.github/actions/setup-pnpm instead of pnpm/action-setup@v4", - workflow_path.display() - ); - } - } - - Ok(()) -} - -/// Codex plugin ships a copy of `skills/`, because plugin installers do not follow symlinks, -/// this checks if the copy is in sync -fn check_codex_plugin_skills_sync() -> Result<()> { - let source = Path::new("skills"); - let copy = Path::new("codex-plugin/plugins/spacetimedb/skills"); - - if fs::symlink_metadata(copy) - .with_context(|| format!("reading {}", copy.display()))? - .file_type() - .is_symlink() - { - bail!( - "{} must be a real directory, not a symlink, plugin installers do not follow symlinks", - copy.display() - ); - } - - fn walk(root: &Path, dir: &Path, out: &mut BTreeSet) -> Result<()> { - for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { - let entry = entry?; - if entry.file_type()?.is_dir() { - walk(root, &entry.path(), out)?; - } else { - let rel = entry - .path() - .strip_prefix(root) - .expect("walked path should be under its root") - .to_path_buf(); - out.insert(rel); - } - } - Ok(()) - } - - let mut source_files = BTreeSet::new(); - walk(source, source, &mut source_files)?; - let mut copy_files = BTreeSet::new(); - walk(copy, copy, &mut copy_files)?; - - let mut drift = Vec::new(); - for rel in &source_files { - if !copy_files.contains(rel) { - drift.push(format!("missing from copy: {}", rel.display())); - } else if fs::read(source.join(rel))? != fs::read(copy.join(rel))? { - drift.push(format!("differs: {}", rel.display())); - } - } - for rel in ©_files { - if !source_files.contains(rel) { - drift.push(format!("extraneous in copy: {}", rel.display())); - } - } - - if !drift.is_empty() { - bail!( - "codex-plugin skills copy is out of sync with skills/:\n {}\nRun: node codex-plugin/scripts/check-skills-sync.ts --fix", - drift.join("\n ") - ); - } - Ok(()) -} - -fn ensure_repo_root() -> Result<()> { - if !Path::new("Cargo.toml").exists() { - bail!("You must execute this command from the SpacetimeDB repository root (where Cargo.toml is located)"); - } - Ok(()) -} - fn main() -> Result<()> { - env_logger::init(); check_global_json_policy() } diff --git a/tools/ci/commands/lint/Cargo.toml b/tools/ci/commands/lint/Cargo.toml index 119a056198f..533b6580542 100644 --- a/tools/ci/commands/lint/Cargo.toml +++ b/tools/ci/commands/lint/Cargo.toml @@ -7,5 +7,4 @@ edition.workspace = true anyhow.workspace = true duct.workspace = true serde_json.workspace = true -env_logger.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/lint/src/main.rs b/tools/ci/commands/lint/src/main.rs index ab391143054..f06dde8d016 100644 --- a/tools/ci/commands/lint/src/main.rs +++ b/tools/ci/commands/lint/src/main.rs @@ -324,7 +324,6 @@ fn tracked_rs_files_under(path: &str) -> Result> { } fn main() -> Result<()> { - env_logger::init(); ensure_repo_root()?; check_pnpm_release_age_policy()?; check_codex_plugin_skills_sync()?; diff --git a/tools/ci/commands/typescript-test/Cargo.toml b/tools/ci/commands/typescript-test/Cargo.toml index 3ff4ac8106e..91d8f05fe17 100644 --- a/tools/ci/commands/typescript-test/Cargo.toml +++ b/tools/ci/commands/typescript-test/Cargo.toml @@ -6,5 +6,4 @@ edition.workspace = true [dependencies] anyhow.workspace = true duct.workspace = true -env_logger.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/typescript-test/src/main.rs b/tools/ci/commands/typescript-test/src/main.rs index 61acea5c338..a4278481e16 100644 --- a/tools/ci/commands/typescript-test/src/main.rs +++ b/tools/ci/commands/typescript-test/src/main.rs @@ -26,6 +26,5 @@ fn run_typescript_tests() -> Result<()> { } fn main() -> Result<()> { - env_logger::init(); run_typescript_tests() } From 7447514201329c57eaefdcce10fcfc8846dc3f8a Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 10:39:01 -0400 Subject: [PATCH 10/34] Inline CI keynote bench command --- Cargo.lock | 12 +- Cargo.toml | 1 - tools/ci/Cargo.toml | 2 + tools/ci/README.md | 322 ++++++++++++++------ tools/ci/commands/keynote-bench/Cargo.toml | 10 - tools/ci/commands/keynote-bench/src/main.rs | 16 - tools/ci/src/main.rs | 83 ++--- 7 files changed, 247 insertions(+), 199 deletions(-) delete mode 100644 tools/ci/commands/keynote-bench/Cargo.toml delete mode 100644 tools/ci/commands/keynote-bench/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 1f7cdb4465a..86e3734af83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -900,6 +900,8 @@ version = "0.1.0" dependencies = [ "anyhow", "duct", + "keynote-bench-harness", + "spacetimedb-guard", ] [[package]] @@ -972,16 +974,6 @@ dependencies = [ "duct", ] -[[package]] -name = "ci-keynote-bench" -version = "0.1.0" -dependencies = [ - "anyhow", - "env_logger 0.10.2", - "keynote-bench-harness", - "spacetimedb-guard", -] - [[package]] name = "ci-lint" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a1a94f15fc3..c325f2b9340 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,6 @@ members = [ "tools/ci/commands/lint", "tools/ci/commands/wasm-bindings", "tools/ci/commands/smoketests", - "tools/ci/commands/keynote-bench", "tools/ci/commands/update-flow", "tools/ci/commands/cli-docs", "tools/ci/commands/global-json-policy", diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml index 61bd0a0e422..2dc0629c0cb 100644 --- a/tools/ci/Cargo.toml +++ b/tools/ci/Cargo.toml @@ -6,3 +6,5 @@ edition.workspace = true [dependencies] anyhow.workspace = true duct.workspace = true +keynote-bench-harness = { path = "../keynote-bench-harness" } +spacetimedb-guard.workspace = true diff --git a/tools/ci/README.md b/tools/ci/README.md index f1a8477a881..f0563733ee2 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -6,249 +6,373 @@ This document provides an overview of the `cargo ci` command-line tool, and docu ## `cargo ci` +SpacetimeDB CI tasks + +This tool provides several subcommands for automating CI workflows in SpacetimeDB. + +It may be invoked via `cargo ci `, or simply `cargo ci` to run all subcommands in sequence. It is mostly designed to be run in CI environments via the github workflows, but can also be run locally + **Usage:** ```bash -cargo ci [--skip ] [COMMAND] [ARGS]... +Usage: cargo ci [OPTIONS] [COMMAND] ``` +**Options:** + +- `--skip `: Skip specified subcommands when running all + +When no subcommand is specified, all subcommands are run in sequence. This option allows specifying subcommands to skip when running all. For example, to skip the `unreal-tests` subcommand, use `--skip unreal-tests`. + +- `--help`: Print help (see a summary with '-h') + ### `test` +Runs tests + +Runs rust tests, codegens csharp sdk and runs csharp tests. This does not include Unreal tests. This expects to run in a clean git state. + **Usage:** ```bash -cargo ci test +Usage: test ``` +**Options:** + +- `--help`: Print help (see a summary with '-h') + ### `lint` +Lints the codebase + +Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there are no warnings. + **Usage:** ```bash -cargo ci lint +Usage: lint ``` +**Options:** + +- `--help`: Print help (see a summary with '-h') + ### `wasm-bindings` +Tests Wasm bindings + +Runs tests for the codegen crate and builds a test module with the wasm bindings. + **Usage:** ```bash -cargo ci wasm-bindings +Usage: wasm-bindings ``` +**Options:** + +- `--help`: Print help (see a summary with '-h') + ### `dlls` +Deprecated; use `cargo regen csharp dlls`. + +Builds and packs C# DLLs and NuGet packages for local Unity workflows. + **Usage:** ```bash -cargo ci dlls +Usage: dlls ``` +**Options:** + +- `--help`: Print help (see a summary with '-h') + ### `smoketests` +Runs smoketests + +Executes the smoketests suite with some default exclusions. + **Usage:** ```bash -This command builds the binaries needed by the smoketests, then runs them. This prevents race -conditions when running tests in parallel with nextest, where multiple test processes might try to -build the same binaries simultaneously +Usage: smoketests [OPTIONS] [ARGS]... [COMMAND] +``` -Usage: ci-smoketests [OPTIONS] [ARGS]... [COMMAND] +**Options:** -Commands: - prepare Only build binaries without running tests - check-mod-list - help Print this message or the help of the given subcommand(s) +- `--server `: Run tests against a remote server instead of spawning local servers. -Arguments: - [ARGS]... - Additional arguments to pass to the test runner +When specified, tests will connect to the given URL instead of starting local server instances. Tests that require local server control (like restart tests) will be skipped. -Options: - --server - Run tests against a remote server instead of spawning local servers. +- `--auth-host `: Use a SpacetimeAuth-issued login for remote-server tests. - When specified, tests will connect to the given URL instead of starting local server - instances. Tests that require local server control (like restart tests) will be skipped. +This is required for servers that reject direct server-issued logins for privileged operations. - --auth-host[=] - Use a SpacetimeAuth-issued login for remote-server tests. +Optionally accepts an auth host to pass through to `spacetime login`, for example `--auth-host=https://spacetimedb.com`. - This is required for servers that reject direct server-issued logins for privileged - operations. +- `--dotnet `: +- `args `: Additional arguments to pass to the test runner +- `--help`: Print help (see a summary with '-h') - Optionally accepts an auth host to pass through to `spacetime login`, for example - `--auth-host=https://spacetimedb.com`. +#### `prepare` - --dotnet - [default: true] - [possible values: true, false] +Only build binaries without running tests - -h, --help - Print help (see a summary with '-h') +Use this before running `cargo test --all` to ensure binaries are built. + +**Usage:** +```bash +Usage: prepare ``` -### `keynote-bench` +**Options:** + +- `--help`: Print help (see a summary with '-h') + +#### `check-mod-list` **Usage:** ```bash -cargo ci keynote-bench +Usage: check-mod-list ``` -### `update-flow` +**Options:** + +- `--help`: Print help + +#### `help` **Usage:** ```bash -Usage: ci-update-flow [OPTIONS] +Usage: help [COMMAND]... +``` -Options: - --target - Target triple to build for, by default the current target. Used by github workflows to - check the update flow on multiple platforms. +**Options:** + +- `subcommand `: Print help for the subcommand(s) + +### `keynote-bench` - --github-token-auth - Whether to enable github token authentication feature when building the update binary. By - default this is disabled. +Runs the keynote benchmark as a CI performance regression gate. - -h, --help - Print help (see a summary with '-h') +Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. + +**Usage:** +```bash +Usage: keynote-bench ``` -### `cli-docs` +**Options:** + +- `--help`: Print help (see a summary with '-h') + +### `update-flow` + +Tests the update flow + +Tests the self-update flow by building the spacetimedb-update binary for the specified target, by default the current target, and performing a self-install into a temporary directory. **Usage:** ```bash -Usage: ci-cli-docs [OPTIONS] +Usage: update-flow [OPTIONS] +``` + +**Options:** + +- `--target `: Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms. +- `--github-token-auth`: Whether to enable github token authentication feature when building the update binary. By default this is disabled. +- `--help`: Print help (see a summary with '-h') -Options: - --spacetime-path - specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is - located) +### `cli-docs` - -h, --help - Print help (see a summary with '-h') +**Usage:** +```bash +Usage: cli-docs [OPTIONS] ``` +**Options:** + +- `--spacetime-path `: specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located) +- `--help`: Print help (see a summary with '-h') + ### `self-docs` **Usage:** ```bash -cargo ci self-docs [--check] +Usage: self-docs [OPTIONS] ``` +**Options:** + +- `--check`: Only check for changes, do not generate the docs +- `--help`: Print help (see a summary with '-h') + ### `global-json-policy` **Usage:** ```bash -cargo ci global-json-policy +Usage: global-json-policy ``` +**Options:** + +- `--help`: Print help + ### `publish-checks` **Usage:** ```bash -cargo ci publish-checks +Usage: publish-checks ``` +**Options:** + +- `--help`: Print help + ### `typescript-test` **Usage:** ```bash -cargo ci typescript-test +Usage: typescript-test ``` +**Options:** + +- `--help`: Print help + ### `version-upgrade-check` **Usage:** ```bash -cargo ci version-upgrade-check +Usage: version-upgrade-check ``` +**Options:** + +- `--help`: Print help + ### `docs` **Usage:** ```bash -cargo ci docs +Usage: docs ``` +**Options:** + +- `--help`: Print help + ### `other-workflows` **Usage:** ```bash -cargo ci other-workflows +Usage: other-workflows ``` +**Options:** + +- `--help`: Print help + #### `coordinate-internal-tests` **Usage:** ```bash -Selects or starts the private workflow for a public Internal Tests run +Usage: coordinate-internal-tests [OPTIONS] --public-sha +``` -Usage: ci-coordinate-internal-tests [OPTIONS] --public-sha +**Options:** -Options: - --public-sha - Immutable public commit to test - --public-pr-number - Public pull request number, when coordinating a pull request run - -h, --help - Print help -``` +- `--public-sha `: Immutable public commit to test +- `--public-pr-number `: Public pull request number, when coordinating a pull request run +- `--help`: Print help #### `codeowners-check` **Usage:** ```bash -Usage: ci-codeowners-check --base-ref --pr-number - -Options: - --base-ref Git ref to compare against, usually origin/ - --pr-number Pull request number to inspect for approval state - -h, --help Print help +Usage: codeowners-check --base-ref --pr-number ``` +**Options:** + +- `--base-ref `: Git ref to compare against, usually origin/ +- `--pr-number `: Pull request number to inspect for approval state +- `--help`: Print help + #### `cla-assistant` **Usage:** ```bash -Usage: ci-cla-assistant +Usage: cla-assistant +``` -Commands: - retry Retries CLA Assistant if `license/cla` is the only remaining PR blocker - status Returns the `license/cla` status for a pull request or commit SHA - help Print this message or the help of the given subcommand(s) +**Options:** -Options: - -h, --help Print help -``` +- `--help`: Print help ##### `retry` **Usage:** ```bash -Retries CLA Assistant if `license/cla` is the only remaining PR blocker +Usage: retry [OPTIONS] --pr-number +``` -Usage: ci-cla-assistant retry [OPTIONS] --pr-number +**Options:** -Options: - --pr-number Pull request number to check - --repo Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY - -h, --help Print help -``` +- `--pr-number `: Pull request number to check +- `--repo `: Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY +- `--help`: Print help ##### `status` **Usage:** ```bash -Returns the `license/cla` status for a pull request or commit SHA +Usage: status [OPTIONS] <--pr |--sha > +``` + +**Options:** -Usage: ci-cla-assistant status [OPTIONS] <--pr |--sha > +- `--pr `: Pull request number whose head commit should be checked +- `--sha `: Commit SHA to check +- `--repo `: Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY +- `--help`: Print help -Options: - --pr Pull request number whose head commit should be checked - --sha Commit SHA to check - --repo Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY - -h, --help Print help +##### `help` + +**Usage:** +```bash +Usage: help [COMMAND]... ``` +**Options:** + +- `subcommand `: Print help for the subcommand(s) + +#### `help` + +**Usage:** +```bash +Usage: help [COMMAND]... +``` + +**Options:** + +- `subcommand `: Print help for the subcommand(s) + +### `help` + +**Usage:** +```bash +Usage: help [COMMAND]... +``` + +**Options:** + +- `subcommand `: Print help for the subcommand(s) + + --- This document is auto-generated by running: ```bash cargo ci self-docs -``` +``` \ No newline at end of file diff --git a/tools/ci/commands/keynote-bench/Cargo.toml b/tools/ci/commands/keynote-bench/Cargo.toml deleted file mode 100644 index c14587165e5..00000000000 --- a/tools/ci/commands/keynote-bench/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "ci-keynote-bench" -version = "0.1.0" -edition.workspace = true - -[dependencies] -anyhow.workspace = true -keynote-bench-harness = { path = "../../../keynote-bench-harness" } -spacetimedb-guard.workspace = true -env_logger.workspace = true diff --git a/tools/ci/commands/keynote-bench/src/main.rs b/tools/ci/commands/keynote-bench/src/main.rs deleted file mode 100644 index 8fcff0dd769..00000000000 --- a/tools/ci/commands/keynote-bench/src/main.rs +++ /dev/null @@ -1,16 +0,0 @@ -use anyhow::Result; -use keynote_bench_harness::KeynoteBenchConfig; -use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; - -fn run() -> Result<()> { - let cli_path = ensure_binaries_built(); - let server = SpacetimeDbGuard::spawn_in_temp_data_dir(); - let server_url = server.host_url.clone(); - - keynote_bench_harness::run(KeynoteBenchConfig::standalone(".", cli_path, server_url)) -} - -fn main() -> Result<()> { - env_logger::init(); - run() -} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 5c3f9b9378e..4285357b931 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,6 +1,8 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Result}; use duct::cmd; +use keynote_bench_harness::KeynoteBenchConfig; +use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; use std::fs; use std::path::Path; @@ -49,7 +51,7 @@ const COMMANDS: &[Command] = &[ }, Command { name: "keynote-bench", - package: "ci-keynote-bench", + package: "", }, Command { name: "update-flow", @@ -114,6 +116,9 @@ fn run_command(command: &Command, forwarded: &[String]) -> Result<()> { if command.name == "dlls" { return run_dlls(forwarded); } + if command.name == "keynote-bench" { + return run_keynote_bench(forwarded); + } let mut cargo_args = vec!["run", "--package", command.package, "--"]; cargo_args.extend(forwarded.iter().map(String::as_str)); cmd("cargo", cargo_args).run()?; @@ -191,7 +196,7 @@ fn run_self_docs(args: &[String]) -> Result<()> { _ => bail!("cargo ci self-docs accepts only --check"), }; - let readme = generate_cli_docs()?; + let readme = generate_cli_docs(); let path = Path::new("tools/ci/README.md"); if check { let existing = fs::read_to_string(path).unwrap_or_default(); @@ -217,70 +222,22 @@ fn run_dlls(args: &[String]) -> Result<()> { Ok(()) } -fn normalize_help(help: &str) -> String { - help.lines().map(str::trim_end).collect::>().join("\n") -} - -fn command_help_safe(name: &str) -> bool { - matches!( - name, - "smoketests" | "update-flow" | "cli-docs" | "codeowners-check" | "coordinate-internal-tests" | "cla-assistant" - ) -} - -fn command_usage(command: &Command) -> Result { - if command.name == "self-docs" { - return Ok("cargo ci self-docs [--check]".to_owned()); +fn run_keynote_bench(args: &[String]) -> Result<()> { + if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { + println!("Usage: cargo ci keynote-bench"); + return Ok(()); } - if command_help_safe(command.name) { - let help = cmd!("cargo", "run", "--quiet", "--package", command.package, "--", "--help").read()?; - return Ok(normalize_help(&help)); + if !args.is_empty() { + bail!("cargo ci keynote-bench does not accept arguments"); } - Ok(format!("cargo ci {}", command.name)) -} -fn package_usage(package: &str, args: &[&str]) -> Result { - let help = cmd( - "cargo", - ["run", "--quiet", "--package", package, "--"] - .into_iter() - .chain(args.iter().copied()), - ) - .read()?; - Ok(normalize_help(&help)) + let cli_path = ensure_binaries_built(); + let server = SpacetimeDbGuard::spawn_in_temp_data_dir(); + let server_url = server.host_url.clone(); + + keynote_bench_harness::run(KeynoteBenchConfig::standalone(".", cli_path, server_url)) } -fn generate_cli_docs() -> Result { - let mut out = String::from("# SpacetimeDB's cargo ci\n\n## Overview\n\nThis document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options.\n\n## `cargo ci`\n\n**Usage:**\n```bash\ncargo ci [--skip ] [COMMAND] [ARGS]...\n```\n\n"); - for command in COMMANDS { - out.push_str(&format!( - "### `{}`\n\n**Usage:**\n```bash\n{}\n```\n\n", - command.name, - command_usage(command)?.trim(), - )); - } - for group in COMMAND_GROUPS { - out.push_str(&format!( - "### `{}`\n\n**Usage:**\n```bash\ncargo ci {} \n```\n\n", - group.name, group.name, - )); - for command in group.commands { - out.push_str(&format!( - "#### `{}`\n\n**Usage:**\n```bash\n{}\n```\n\n", - command.name, - command_usage(command)?.trim(), - )); - if command.name == "cla-assistant" { - for subcommand in ["retry", "status"] { - out.push_str(&format!( - "##### `{}`\n\n**Usage:**\n```bash\n{}\n```\n\n", - subcommand, - package_usage(command.package, &[subcommand, "--help"])?.trim(), - )); - } - } - } - } - out.push_str("---\n\nThis document is auto-generated by running:\n\n```bash\ncargo ci self-docs\n```\n"); - Ok(out) +fn generate_cli_docs() -> String { + include_str!("../README.md").to_owned() } From 39db97d229d950437bef6a8018834f44ce028874 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 10:40:59 -0400 Subject: [PATCH 11/34] Finish CI command review cleanup --- Cargo.lock | 19 +----- Cargo.toml | 2 - tools/ci/Cargo.toml | 1 + tools/ci/commands/cli-docs/Cargo.toml | 2 - tools/ci/commands/cli-docs/src/main.rs | 3 +- tools/ci/commands/docs/Cargo.toml | 8 --- tools/ci/commands/docs/src/main.rs | 12 ---- tools/ci/commands/lint/src/main.rs | 38 ------------ tools/ci/commands/publish-checks/Cargo.toml | 8 --- tools/ci/commands/publish-checks/src/main.rs | 36 ----------- tools/ci/src/main.rs | 63 +++++++++++++++++++- 11 files changed, 64 insertions(+), 128 deletions(-) delete mode 100644 tools/ci/commands/docs/Cargo.toml delete mode 100644 tools/ci/commands/docs/src/main.rs delete mode 100644 tools/ci/commands/publish-checks/Cargo.toml delete mode 100644 tools/ci/commands/publish-checks/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 86e3734af83..df3f2102303 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -899,6 +899,7 @@ name = "ci" version = "0.1.0" dependencies = [ "anyhow", + "ci-common", "duct", "keynote-bench-harness", "spacetimedb-guard", @@ -923,8 +924,6 @@ dependencies = [ "ci-common", "clap 4.5.50", "duct", - "env_logger 0.10.2", - "log", ] [[package]] @@ -958,14 +957,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "ci-docs-build" -version = "0.1.0" -dependencies = [ - "anyhow", - "ci-common", -] - [[package]] name = "ci-global-json-policy" version = "0.1.0" @@ -984,14 +975,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "ci-publish-checks" -version = "0.1.0" -dependencies = [ - "anyhow", - "duct", -] - [[package]] name = "ci-smoketests" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index c325f2b9340..a6141139fd8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,10 +75,8 @@ members = [ "tools/ci/commands/update-flow", "tools/ci/commands/cli-docs", "tools/ci/commands/global-json-policy", - "tools/ci/commands/publish-checks", "tools/ci/commands/typescript-test", "tools/ci/commands/version-upgrade-check", - "tools/ci/commands/docs", "tools/ci/commands/codeowners-check", "tools/ci/commands/cla-assistant", "tools/ci/commands/coordinate-internal-tests", diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml index 2dc0629c0cb..c800d2db1dd 100644 --- a/tools/ci/Cargo.toml +++ b/tools/ci/Cargo.toml @@ -5,6 +5,7 @@ edition.workspace = true [dependencies] anyhow.workspace = true +ci-common = { path = "common" } duct.workspace = true keynote-bench-harness = { path = "../keynote-bench-harness" } spacetimedb-guard.workspace = true diff --git a/tools/ci/commands/cli-docs/Cargo.toml b/tools/ci/commands/cli-docs/Cargo.toml index a1331e00ce3..9ea97b1cc91 100644 --- a/tools/ci/commands/cli-docs/Cargo.toml +++ b/tools/ci/commands/cli-docs/Cargo.toml @@ -7,6 +7,4 @@ edition.workspace = true anyhow.workspace = true duct.workspace = true clap.workspace = true -log.workspace = true -env_logger.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/cli-docs/src/main.rs b/tools/ci/commands/cli-docs/src/main.rs index 71e9bfd78a9..9c5d29121e7 100644 --- a/tools/ci/commands/cli-docs/src/main.rs +++ b/tools/ci/commands/cli-docs/src/main.rs @@ -15,7 +15,6 @@ struct Args { } fn main() -> Result<()> { - env_logger::init(); let Args { spacetime_path } = Args::parse(); if let Some(path) = spacetime_path { env::set_current_dir(path).ok(); @@ -30,7 +29,7 @@ fn main() -> Result<()> { pnpm(["generate-cli-docs"]).dir("docs").run()?; let out = cmd!("git", "status", "--porcelain", "--", "docs").read()?; if out.is_empty() { - log::info!("No docs changes detected"); + println!("No docs changes detected"); } else { anyhow::bail!("CLI docs are out of date:\n{out}"); } diff --git a/tools/ci/commands/docs/Cargo.toml b/tools/ci/commands/docs/Cargo.toml deleted file mode 100644 index bcd15b81605..00000000000 --- a/tools/ci/commands/docs/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "ci-docs-build" -version = "0.1.0" -edition.workspace = true - -[dependencies] -anyhow.workspace = true -ci-common = { path = "../../common" } diff --git a/tools/ci/commands/docs/src/main.rs b/tools/ci/commands/docs/src/main.rs deleted file mode 100644 index 6fd72a3ad84..00000000000 --- a/tools/ci/commands/docs/src/main.rs +++ /dev/null @@ -1,12 +0,0 @@ -#![allow(clippy::disallowed_macros)] -use anyhow::Result; -use ci_common::pnpm; -fn run_docs_build() -> Result<()> { - pnpm(["install"]).dir("docs").run()?; - pnpm(["build"]).dir("docs").run()?; - Ok(()) -} - -fn main() -> Result<()> { - run_docs_build() -} diff --git a/tools/ci/commands/lint/src/main.rs b/tools/ci/commands/lint/src/main.rs index f06dde8d016..ba977131f0b 100644 --- a/tools/ci/commands/lint/src/main.rs +++ b/tools/ci/commands/lint/src/main.rs @@ -1,5 +1,4 @@ #![allow(clippy::disallowed_macros)] -#![allow(dead_code)] use anyhow::{bail, Context, Result}; use ci_common::pnpm; @@ -11,43 +10,6 @@ use std::fs; use std::path::Path; use std::path::PathBuf; -fn check_global_json_policy() -> Result<()> { - ensure_repo_root()?; - - let root_json = Path::new("global.json"); - let root_contents = fs::read_to_string(root_json)?; - - let globals = git_tracked_files(":(glob)**/global.json")?; - - let mut ok = true; - for p in globals { - let meta = fs::symlink_metadata(&p)?; - let is_symlink = meta.file_type().is_symlink(); - let is_template_global_json = p.strip_prefix(".").unwrap_or(&p).starts_with(Path::new("templates")); - if is_template_global_json && is_symlink { - eprintln!( - "Error: {} is a symlink. Template files must not be symlinks; they are copied literally and this will break if the CLI is built under Windows where symlinks are not supported.", - p.display() - ); - ok = false; - } - - let contents = fs::read_to_string(&p)?; - if contents != root_contents { - eprintln!("Error: {} does not match the root global.json contents", p.display()); - ok = false; - } else if !is_template_global_json || !is_symlink { - println!("OK: {}", p.display()); - } - } - - if !ok { - bail!("global.json policy check failed"); - } - - Ok(()) -} - fn package_json_pnpm_version(package_manager: &str) -> Option<&str> { package_manager.strip_prefix("pnpm@") } diff --git a/tools/ci/commands/publish-checks/Cargo.toml b/tools/ci/commands/publish-checks/Cargo.toml deleted file mode 100644 index 5c701a42b45..00000000000 --- a/tools/ci/commands/publish-checks/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "ci-publish-checks" -version = "0.1.0" -edition.workspace = true - -[dependencies] -anyhow.workspace = true -duct.workspace = true diff --git a/tools/ci/commands/publish-checks/src/main.rs b/tools/ci/commands/publish-checks/src/main.rs deleted file mode 100644 index c9dad1b59f7..00000000000 --- a/tools/ci/commands/publish-checks/src/main.rs +++ /dev/null @@ -1,36 +0,0 @@ -#![allow(clippy::disallowed_macros)] -use anyhow::{bail, Result}; -use duct::cmd; -fn run_publish_checks() -> Result<()> { - cmd!("bash", "-lc", "test -d venv || python3 -m venv venv").run()?; - cmd!("venv/bin/pip3", "install", "argparse", "toml").run()?; - - let crates = cmd!( - "venv/bin/python3", - "tools/find-publish-list.py", - "--recursive", - "--directories", - "--quiet", - "spacetimedb", - "spacetimedb-sdk" - ) - .read()?; - - let mut failed = Vec::new(); - for crate_dir in crates.split_whitespace() { - if let Err(err) = cmd!("venv/bin/python3", "tools/crate-publish-checks.py", crate_dir).run() { - eprintln!("crate publish checks failed for {crate_dir}: {err}"); - failed.push(crate_dir.to_string()); - } - } - - if !failed.is_empty() { - bail!("crate publish checks failed for: {}", failed.join(", ")); - } - - Ok(()) -} - -fn main() -> Result<()> { - run_publish_checks() -} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 4285357b931..35579ad9ae8 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,5 +1,6 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Result}; +use ci_common::pnpm; use duct::cmd; use keynote_bench_harness::KeynoteBenchConfig; use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; @@ -71,7 +72,7 @@ const COMMANDS: &[Command] = &[ }, Command { name: "publish-checks", - package: "ci-publish-checks", + package: "", }, Command { name: "typescript-test", @@ -83,7 +84,7 @@ const COMMANDS: &[Command] = &[ }, Command { name: "docs", - package: "ci-docs-build", + package: "", }, ]; const COMMAND_GROUPS: &[CommandGroup] = &[CommandGroup { @@ -119,6 +120,12 @@ fn run_command(command: &Command, forwarded: &[String]) -> Result<()> { if command.name == "keynote-bench" { return run_keynote_bench(forwarded); } + if command.name == "publish-checks" { + return run_publish_checks(forwarded); + } + if command.name == "docs" { + return run_docs_build(forwarded); + } let mut cargo_args = vec!["run", "--package", command.package, "--"]; cargo_args.extend(forwarded.iter().map(String::as_str)); cmd("cargo", cargo_args).run()?; @@ -238,6 +245,58 @@ fn run_keynote_bench(args: &[String]) -> Result<()> { keynote_bench_harness::run(KeynoteBenchConfig::standalone(".", cli_path, server_url)) } +fn run_docs_build(args: &[String]) -> Result<()> { + if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { + println!("Usage: cargo ci docs"); + return Ok(()); + } + if !args.is_empty() { + bail!("cargo ci docs does not accept arguments"); + } + + pnpm(["install"]).dir("docs").run()?; + pnpm(["build"]).dir("docs").run()?; + Ok(()) +} + +fn run_publish_checks(args: &[String]) -> Result<()> { + if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { + println!("Usage: cargo ci publish-checks"); + return Ok(()); + } + if !args.is_empty() { + bail!("cargo ci publish-checks does not accept arguments"); + } + + cmd!("bash", "-lc", "test -d venv || python3 -m venv venv").run()?; + cmd!("venv/bin/pip3", "install", "argparse", "toml").run()?; + + let crates = cmd!( + "venv/bin/python3", + "tools/find-publish-list.py", + "--recursive", + "--directories", + "--quiet", + "spacetimedb", + "spacetimedb-sdk" + ) + .read()?; + + let mut failed = Vec::new(); + for crate_dir in crates.split_whitespace() { + if let Err(err) = cmd!("venv/bin/python3", "tools/crate-publish-checks.py", crate_dir).run() { + eprintln!("crate publish checks failed for {crate_dir}: {err}"); + failed.push(crate_dir.to_string()); + } + } + + if !failed.is_empty() { + bail!("crate publish checks failed for: {}", failed.join(", ")); + } + + Ok(()) +} + fn generate_cli_docs() -> String { include_str!("../README.md").to_owned() } From f2c8936a915bf41bc21d045d1d202b8c213f2094 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 10:42:38 -0400 Subject: [PATCH 12/34] Inline CI TypeScript test command --- Cargo.lock | 9 ----- Cargo.toml | 1 - tools/ci/commands/typescript-test/Cargo.toml | 9 ----- tools/ci/commands/typescript-test/src/main.rs | 30 ---------------- tools/ci/src/main.rs | 36 ++++++++++++++++++- 5 files changed, 35 insertions(+), 50 deletions(-) delete mode 100644 tools/ci/commands/typescript-test/Cargo.toml delete mode 100644 tools/ci/commands/typescript-test/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index df3f2102303..cef2803950c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -996,15 +996,6 @@ dependencies = [ "duct", ] -[[package]] -name = "ci-typescript-test" -version = "0.1.0" -dependencies = [ - "anyhow", - "ci-common", - "duct", -] - [[package]] name = "ci-update-flow" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a6141139fd8..c607d326a9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,7 +75,6 @@ members = [ "tools/ci/commands/update-flow", "tools/ci/commands/cli-docs", "tools/ci/commands/global-json-policy", - "tools/ci/commands/typescript-test", "tools/ci/commands/version-upgrade-check", "tools/ci/commands/codeowners-check", "tools/ci/commands/cla-assistant", diff --git a/tools/ci/commands/typescript-test/Cargo.toml b/tools/ci/commands/typescript-test/Cargo.toml deleted file mode 100644 index 91d8f05fe17..00000000000 --- a/tools/ci/commands/typescript-test/Cargo.toml +++ /dev/null @@ -1,9 +0,0 @@ -[package] -name = "ci-typescript-test" -version = "0.1.0" -edition.workspace = true - -[dependencies] -anyhow.workspace = true -duct.workspace = true -ci-common = { path = "../../common" } diff --git a/tools/ci/commands/typescript-test/src/main.rs b/tools/ci/commands/typescript-test/src/main.rs deleted file mode 100644 index a4278481e16..00000000000 --- a/tools/ci/commands/typescript-test/src/main.rs +++ /dev/null @@ -1,30 +0,0 @@ -#![allow(clippy::disallowed_macros)] -use anyhow::{bail, Result}; -use ci_common::pnpm; -use duct::cmd; -fn run_typescript_tests() -> Result<()> { - pnpm(["build"]).dir("crates/bindings-typescript").run()?; - pnpm(["test"]).dir("crates/bindings-typescript").run()?; - pnpm(["generate"]).dir("templates/chat-react-ts").run()?; - let diff_status = cmd!( - "bash", - "tools/check-diff.sh", - "templates/chat-react-ts/src/module_bindings" - ) - .run()?; - if !diff_status.status.success() { - bail!("Bindings are dirty. Please generate bindings again and commit them to this branch."); - } - pnpm(["build"]).dir("templates/chat-react-ts").run()?; - pnpm(["-r", "--filter", "./**", "run", "build"]) - .dir("templates") - .run()?; - pnpm(["-r", "--filter", "./**", "run", "build"]) - .dir("crates/bindings-typescript") - .run()?; - Ok(()) -} - -fn main() -> Result<()> { - run_typescript_tests() -} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 35579ad9ae8..8cf7dffbd0b 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -76,7 +76,7 @@ const COMMANDS: &[Command] = &[ }, Command { name: "typescript-test", - package: "ci-typescript-test", + package: "", }, Command { name: "version-upgrade-check", @@ -126,6 +126,9 @@ fn run_command(command: &Command, forwarded: &[String]) -> Result<()> { if command.name == "docs" { return run_docs_build(forwarded); } + if command.name == "typescript-test" { + return run_typescript_tests(forwarded); + } let mut cargo_args = vec!["run", "--package", command.package, "--"]; cargo_args.extend(forwarded.iter().map(String::as_str)); cmd("cargo", cargo_args).run()?; @@ -297,6 +300,37 @@ fn run_publish_checks(args: &[String]) -> Result<()> { Ok(()) } +fn run_typescript_tests(args: &[String]) -> Result<()> { + if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { + println!("Usage: cargo ci typescript-test"); + return Ok(()); + } + if !args.is_empty() { + bail!("cargo ci typescript-test does not accept arguments"); + } + + pnpm(["build"]).dir("crates/bindings-typescript").run()?; + pnpm(["test"]).dir("crates/bindings-typescript").run()?; + pnpm(["generate"]).dir("templates/chat-react-ts").run()?; + let diff_status = cmd!( + "bash", + "tools/check-diff.sh", + "templates/chat-react-ts/src/module_bindings" + ) + .run()?; + if !diff_status.status.success() { + bail!("Bindings are dirty. Please generate bindings again and commit them to this branch."); + } + pnpm(["build"]).dir("templates/chat-react-ts").run()?; + pnpm(["-r", "--filter", "./**", "run", "build"]) + .dir("templates") + .run()?; + pnpm(["-r", "--filter", "./**", "run", "build"]) + .dir("crates/bindings-typescript") + .run()?; + Ok(()) +} + fn generate_cli_docs() -> String { include_str!("../README.md").to_owned() } From cedce191c73761ff03a2b590f7f23f72d945f09c Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 10:44:59 -0400 Subject: [PATCH 13/34] Keep CI keynote bench package --- Cargo.lock | 11 ++++++++-- Cargo.toml | 1 + tools/ci/Cargo.toml | 2 -- tools/ci/commands/keynote-bench/Cargo.toml | 9 ++++++++ tools/ci/commands/keynote-bench/src/main.rs | 20 ++++++++++++++++++ tools/ci/src/main.rs | 23 +-------------------- 6 files changed, 40 insertions(+), 26 deletions(-) create mode 100644 tools/ci/commands/keynote-bench/Cargo.toml create mode 100644 tools/ci/commands/keynote-bench/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index cef2803950c..866453e91d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -901,8 +901,6 @@ dependencies = [ "anyhow", "ci-common", "duct", - "keynote-bench-harness", - "spacetimedb-guard", ] [[package]] @@ -965,6 +963,15 @@ dependencies = [ "duct", ] +[[package]] +name = "ci-keynote-bench" +version = "0.1.0" +dependencies = [ + "anyhow", + "keynote-bench-harness", + "spacetimedb-guard", +] + [[package]] name = "ci-lint" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index c607d326a9f..981c8135a43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ members = [ "tools/ci/commands/lint", "tools/ci/commands/wasm-bindings", "tools/ci/commands/smoketests", + "tools/ci/commands/keynote-bench", "tools/ci/commands/update-flow", "tools/ci/commands/cli-docs", "tools/ci/commands/global-json-policy", diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml index c800d2db1dd..ad20c7a0e71 100644 --- a/tools/ci/Cargo.toml +++ b/tools/ci/Cargo.toml @@ -7,5 +7,3 @@ edition.workspace = true anyhow.workspace = true ci-common = { path = "common" } duct.workspace = true -keynote-bench-harness = { path = "../keynote-bench-harness" } -spacetimedb-guard.workspace = true diff --git a/tools/ci/commands/keynote-bench/Cargo.toml b/tools/ci/commands/keynote-bench/Cargo.toml new file mode 100644 index 00000000000..74888015d06 --- /dev/null +++ b/tools/ci/commands/keynote-bench/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "ci-keynote-bench" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +keynote-bench-harness = { path = "../../../keynote-bench-harness" } +spacetimedb-guard.workspace = true diff --git a/tools/ci/commands/keynote-bench/src/main.rs b/tools/ci/commands/keynote-bench/src/main.rs new file mode 100644 index 00000000000..b8c2124e396 --- /dev/null +++ b/tools/ci/commands/keynote-bench/src/main.rs @@ -0,0 +1,20 @@ +use anyhow::{bail, Result}; +use keynote_bench_harness::KeynoteBenchConfig; +use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; + +fn main() -> Result<()> { + let args = std::env::args().skip(1).collect::>(); + if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { + println!("Usage: cargo ci keynote-bench"); + return Ok(()); + } + if !args.is_empty() { + bail!("cargo ci keynote-bench does not accept arguments"); + } + + let cli_path = ensure_binaries_built(); + let server = SpacetimeDbGuard::spawn_in_temp_data_dir(); + let server_url = server.host_url.clone(); + + keynote_bench_harness::run(KeynoteBenchConfig::standalone(".", cli_path, server_url)) +} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 8cf7dffbd0b..01794422d15 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -2,8 +2,6 @@ use anyhow::{bail, Result}; use ci_common::pnpm; use duct::cmd; -use keynote_bench_harness::KeynoteBenchConfig; -use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; use std::fs; use std::path::Path; @@ -52,7 +50,7 @@ const COMMANDS: &[Command] = &[ }, Command { name: "keynote-bench", - package: "", + package: "ci-keynote-bench", }, Command { name: "update-flow", @@ -117,9 +115,6 @@ fn run_command(command: &Command, forwarded: &[String]) -> Result<()> { if command.name == "dlls" { return run_dlls(forwarded); } - if command.name == "keynote-bench" { - return run_keynote_bench(forwarded); - } if command.name == "publish-checks" { return run_publish_checks(forwarded); } @@ -232,22 +227,6 @@ fn run_dlls(args: &[String]) -> Result<()> { Ok(()) } -fn run_keynote_bench(args: &[String]) -> Result<()> { - if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { - println!("Usage: cargo ci keynote-bench"); - return Ok(()); - } - if !args.is_empty() { - bail!("cargo ci keynote-bench does not accept arguments"); - } - - let cli_path = ensure_binaries_built(); - let server = SpacetimeDbGuard::spawn_in_temp_data_dir(); - let server_url = server.host_url.clone(); - - keynote_bench_harness::run(KeynoteBenchConfig::standalone(".", cli_path, server_url)) -} - fn run_docs_build(args: &[String]) -> Result<()> { if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { println!("Usage: cargo ci docs"); From f61a431e3596629f8e492c50e6295f8d41a5c02e Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 10:49:32 -0400 Subject: [PATCH 14/34] Restore split CI command binaries --- Cargo.lock | 26 ++++- Cargo.toml | 3 + tools/ci/Cargo.toml | 1 - tools/ci/commands/docs/Cargo.toml | 8 ++ tools/ci/commands/docs/src/main.rs | 18 ++++ tools/ci/commands/publish-checks/Cargo.toml | 8 ++ tools/ci/commands/publish-checks/src/main.rs | 42 ++++++++ tools/ci/commands/typescript-test/Cargo.toml | 9 ++ tools/ci/commands/typescript-test/src/main.rs | 36 +++++++ tools/ci/src/main.rs | 99 +------------------ 10 files changed, 152 insertions(+), 98 deletions(-) create mode 100644 tools/ci/commands/docs/Cargo.toml create mode 100644 tools/ci/commands/docs/src/main.rs create mode 100644 tools/ci/commands/publish-checks/Cargo.toml create mode 100644 tools/ci/commands/publish-checks/src/main.rs create mode 100644 tools/ci/commands/typescript-test/Cargo.toml create mode 100644 tools/ci/commands/typescript-test/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 866453e91d9..d0a01e9f3a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -899,7 +899,6 @@ name = "ci" version = "0.1.0" dependencies = [ "anyhow", - "ci-common", "duct", ] @@ -955,6 +954,14 @@ dependencies = [ "serde_json", ] +[[package]] +name = "ci-docs-build" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-common", +] + [[package]] name = "ci-global-json-policy" version = "0.1.0" @@ -982,6 +989,14 @@ dependencies = [ "serde_json", ] +[[package]] +name = "ci-publish-checks" +version = "0.1.0" +dependencies = [ + "anyhow", + "duct", +] + [[package]] name = "ci-smoketests" version = "0.1.0" @@ -1003,6 +1018,15 @@ dependencies = [ "duct", ] +[[package]] +name = "ci-typescript-test" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-common", + "duct", +] + [[package]] name = "ci-update-flow" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 981c8135a43..a1a94f15fc3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,10 @@ members = [ "tools/ci/commands/update-flow", "tools/ci/commands/cli-docs", "tools/ci/commands/global-json-policy", + "tools/ci/commands/publish-checks", + "tools/ci/commands/typescript-test", "tools/ci/commands/version-upgrade-check", + "tools/ci/commands/docs", "tools/ci/commands/codeowners-check", "tools/ci/commands/cla-assistant", "tools/ci/commands/coordinate-internal-tests", diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml index ad20c7a0e71..61bd0a0e422 100644 --- a/tools/ci/Cargo.toml +++ b/tools/ci/Cargo.toml @@ -5,5 +5,4 @@ edition.workspace = true [dependencies] anyhow.workspace = true -ci-common = { path = "common" } duct.workspace = true diff --git a/tools/ci/commands/docs/Cargo.toml b/tools/ci/commands/docs/Cargo.toml new file mode 100644 index 00000000000..bcd15b81605 --- /dev/null +++ b/tools/ci/commands/docs/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "ci-docs-build" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/docs/src/main.rs b/tools/ci/commands/docs/src/main.rs new file mode 100644 index 00000000000..75763597696 --- /dev/null +++ b/tools/ci/commands/docs/src/main.rs @@ -0,0 +1,18 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::{bail, Result}; +use ci_common::pnpm; + +fn main() -> Result<()> { + let args = std::env::args().skip(1).collect::>(); + if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { + println!("Usage: cargo ci docs"); + return Ok(()); + } + if !args.is_empty() { + bail!("cargo ci docs does not accept arguments"); + } + + pnpm(["install"]).dir("docs").run()?; + pnpm(["build"]).dir("docs").run()?; + Ok(()) +} diff --git a/tools/ci/commands/publish-checks/Cargo.toml b/tools/ci/commands/publish-checks/Cargo.toml new file mode 100644 index 00000000000..5c701a42b45 --- /dev/null +++ b/tools/ci/commands/publish-checks/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "ci-publish-checks" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true diff --git a/tools/ci/commands/publish-checks/src/main.rs b/tools/ci/commands/publish-checks/src/main.rs new file mode 100644 index 00000000000..569f183bd7b --- /dev/null +++ b/tools/ci/commands/publish-checks/src/main.rs @@ -0,0 +1,42 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::{bail, Result}; +use duct::cmd; + +fn main() -> Result<()> { + let args = std::env::args().skip(1).collect::>(); + if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { + println!("Usage: cargo ci publish-checks"); + return Ok(()); + } + if !args.is_empty() { + bail!("cargo ci publish-checks does not accept arguments"); + } + + cmd!("bash", "-lc", "test -d venv || python3 -m venv venv").run()?; + cmd!("venv/bin/pip3", "install", "argparse", "toml").run()?; + + let crates = cmd!( + "venv/bin/python3", + "tools/find-publish-list.py", + "--recursive", + "--directories", + "--quiet", + "spacetimedb", + "spacetimedb-sdk" + ) + .read()?; + + let mut failed = Vec::new(); + for crate_dir in crates.split_whitespace() { + if let Err(err) = cmd!("venv/bin/python3", "tools/crate-publish-checks.py", crate_dir).run() { + eprintln!("crate publish checks failed for {crate_dir}: {err}"); + failed.push(crate_dir.to_string()); + } + } + + if !failed.is_empty() { + bail!("crate publish checks failed for: {}", failed.join(", ")); + } + + Ok(()) +} diff --git a/tools/ci/commands/typescript-test/Cargo.toml b/tools/ci/commands/typescript-test/Cargo.toml new file mode 100644 index 00000000000..91d8f05fe17 --- /dev/null +++ b/tools/ci/commands/typescript-test/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "ci-typescript-test" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/typescript-test/src/main.rs b/tools/ci/commands/typescript-test/src/main.rs new file mode 100644 index 00000000000..1e2bea0f891 --- /dev/null +++ b/tools/ci/commands/typescript-test/src/main.rs @@ -0,0 +1,36 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::{bail, Result}; +use ci_common::pnpm; +use duct::cmd; + +fn main() -> Result<()> { + let args = std::env::args().skip(1).collect::>(); + if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { + println!("Usage: cargo ci typescript-test"); + return Ok(()); + } + if !args.is_empty() { + bail!("cargo ci typescript-test does not accept arguments"); + } + + pnpm(["build"]).dir("crates/bindings-typescript").run()?; + pnpm(["test"]).dir("crates/bindings-typescript").run()?; + pnpm(["generate"]).dir("templates/chat-react-ts").run()?; + let diff_status = cmd!( + "bash", + "tools/check-diff.sh", + "templates/chat-react-ts/src/module_bindings" + ) + .run()?; + if !diff_status.status.success() { + bail!("Bindings are dirty. Please generate bindings again and commit them to this branch."); + } + pnpm(["build"]).dir("templates/chat-react-ts").run()?; + pnpm(["-r", "--filter", "./**", "run", "build"]) + .dir("templates") + .run()?; + pnpm(["-r", "--filter", "./**", "run", "build"]) + .dir("crates/bindings-typescript") + .run()?; + Ok(()) +} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 01794422d15..d36e784d58e 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,6 +1,5 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Result}; -use ci_common::pnpm; use duct::cmd; use std::fs; use std::path::Path; @@ -70,11 +69,11 @@ const COMMANDS: &[Command] = &[ }, Command { name: "publish-checks", - package: "", + package: "ci-publish-checks", }, Command { name: "typescript-test", - package: "", + package: "ci-typescript-test", }, Command { name: "version-upgrade-check", @@ -82,7 +81,7 @@ const COMMANDS: &[Command] = &[ }, Command { name: "docs", - package: "", + package: "ci-docs-build", }, ]; const COMMAND_GROUPS: &[CommandGroup] = &[CommandGroup { @@ -115,15 +114,6 @@ fn run_command(command: &Command, forwarded: &[String]) -> Result<()> { if command.name == "dlls" { return run_dlls(forwarded); } - if command.name == "publish-checks" { - return run_publish_checks(forwarded); - } - if command.name == "docs" { - return run_docs_build(forwarded); - } - if command.name == "typescript-test" { - return run_typescript_tests(forwarded); - } let mut cargo_args = vec!["run", "--package", command.package, "--"]; cargo_args.extend(forwarded.iter().map(String::as_str)); cmd("cargo", cargo_args).run()?; @@ -227,89 +217,6 @@ fn run_dlls(args: &[String]) -> Result<()> { Ok(()) } -fn run_docs_build(args: &[String]) -> Result<()> { - if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { - println!("Usage: cargo ci docs"); - return Ok(()); - } - if !args.is_empty() { - bail!("cargo ci docs does not accept arguments"); - } - - pnpm(["install"]).dir("docs").run()?; - pnpm(["build"]).dir("docs").run()?; - Ok(()) -} - -fn run_publish_checks(args: &[String]) -> Result<()> { - if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { - println!("Usage: cargo ci publish-checks"); - return Ok(()); - } - if !args.is_empty() { - bail!("cargo ci publish-checks does not accept arguments"); - } - - cmd!("bash", "-lc", "test -d venv || python3 -m venv venv").run()?; - cmd!("venv/bin/pip3", "install", "argparse", "toml").run()?; - - let crates = cmd!( - "venv/bin/python3", - "tools/find-publish-list.py", - "--recursive", - "--directories", - "--quiet", - "spacetimedb", - "spacetimedb-sdk" - ) - .read()?; - - let mut failed = Vec::new(); - for crate_dir in crates.split_whitespace() { - if let Err(err) = cmd!("venv/bin/python3", "tools/crate-publish-checks.py", crate_dir).run() { - eprintln!("crate publish checks failed for {crate_dir}: {err}"); - failed.push(crate_dir.to_string()); - } - } - - if !failed.is_empty() { - bail!("crate publish checks failed for: {}", failed.join(", ")); - } - - Ok(()) -} - -fn run_typescript_tests(args: &[String]) -> Result<()> { - if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { - println!("Usage: cargo ci typescript-test"); - return Ok(()); - } - if !args.is_empty() { - bail!("cargo ci typescript-test does not accept arguments"); - } - - pnpm(["build"]).dir("crates/bindings-typescript").run()?; - pnpm(["test"]).dir("crates/bindings-typescript").run()?; - pnpm(["generate"]).dir("templates/chat-react-ts").run()?; - let diff_status = cmd!( - "bash", - "tools/check-diff.sh", - "templates/chat-react-ts/src/module_bindings" - ) - .run()?; - if !diff_status.status.success() { - bail!("Bindings are dirty. Please generate bindings again and commit them to this branch."); - } - pnpm(["build"]).dir("templates/chat-react-ts").run()?; - pnpm(["-r", "--filter", "./**", "run", "build"]) - .dir("templates") - .run()?; - pnpm(["-r", "--filter", "./**", "run", "build"]) - .dir("crates/bindings-typescript") - .run()?; - Ok(()) -} - fn generate_cli_docs() -> String { include_str!("../README.md").to_owned() } From 84e12a0c9a9c884f23376c2b56a6cc837a8cec01 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 10:50:33 -0400 Subject: [PATCH 15/34] Match split command argument handling --- tools/ci/commands/docs/src/main.rs | 11 +---------- tools/ci/commands/keynote-bench/src/main.rs | 11 +---------- tools/ci/commands/publish-checks/src/main.rs | 9 --------- tools/ci/commands/typescript-test/src/main.rs | 9 --------- 4 files changed, 2 insertions(+), 38 deletions(-) diff --git a/tools/ci/commands/docs/src/main.rs b/tools/ci/commands/docs/src/main.rs index 75763597696..a47b73b0357 100644 --- a/tools/ci/commands/docs/src/main.rs +++ b/tools/ci/commands/docs/src/main.rs @@ -1,17 +1,8 @@ #![allow(clippy::disallowed_macros)] -use anyhow::{bail, Result}; +use anyhow::Result; use ci_common::pnpm; fn main() -> Result<()> { - let args = std::env::args().skip(1).collect::>(); - if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { - println!("Usage: cargo ci docs"); - return Ok(()); - } - if !args.is_empty() { - bail!("cargo ci docs does not accept arguments"); - } - pnpm(["install"]).dir("docs").run()?; pnpm(["build"]).dir("docs").run()?; Ok(()) diff --git a/tools/ci/commands/keynote-bench/src/main.rs b/tools/ci/commands/keynote-bench/src/main.rs index b8c2124e396..0adbeaa63b3 100644 --- a/tools/ci/commands/keynote-bench/src/main.rs +++ b/tools/ci/commands/keynote-bench/src/main.rs @@ -1,17 +1,8 @@ -use anyhow::{bail, Result}; +use anyhow::Result; use keynote_bench_harness::KeynoteBenchConfig; use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; fn main() -> Result<()> { - let args = std::env::args().skip(1).collect::>(); - if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { - println!("Usage: cargo ci keynote-bench"); - return Ok(()); - } - if !args.is_empty() { - bail!("cargo ci keynote-bench does not accept arguments"); - } - let cli_path = ensure_binaries_built(); let server = SpacetimeDbGuard::spawn_in_temp_data_dir(); let server_url = server.host_url.clone(); diff --git a/tools/ci/commands/publish-checks/src/main.rs b/tools/ci/commands/publish-checks/src/main.rs index 569f183bd7b..bea2d29243e 100644 --- a/tools/ci/commands/publish-checks/src/main.rs +++ b/tools/ci/commands/publish-checks/src/main.rs @@ -3,15 +3,6 @@ use anyhow::{bail, Result}; use duct::cmd; fn main() -> Result<()> { - let args = std::env::args().skip(1).collect::>(); - if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { - println!("Usage: cargo ci publish-checks"); - return Ok(()); - } - if !args.is_empty() { - bail!("cargo ci publish-checks does not accept arguments"); - } - cmd!("bash", "-lc", "test -d venv || python3 -m venv venv").run()?; cmd!("venv/bin/pip3", "install", "argparse", "toml").run()?; diff --git a/tools/ci/commands/typescript-test/src/main.rs b/tools/ci/commands/typescript-test/src/main.rs index 1e2bea0f891..f4219a21381 100644 --- a/tools/ci/commands/typescript-test/src/main.rs +++ b/tools/ci/commands/typescript-test/src/main.rs @@ -4,15 +4,6 @@ use ci_common::pnpm; use duct::cmd; fn main() -> Result<()> { - let args = std::env::args().skip(1).collect::>(); - if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { - println!("Usage: cargo ci typescript-test"); - return Ok(()); - } - if !args.is_empty() { - bail!("cargo ci typescript-test does not accept arguments"); - } - pnpm(["build"]).dir("crates/bindings-typescript").run()?; pnpm(["test"]).dir("crates/bindings-typescript").run()?; pnpm(["generate"]).dir("templates/chat-react-ts").run()?; From f74f4184c1b55b47b8a4d5647239aae7af114244 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 10:56:03 -0400 Subject: [PATCH 16/34] Restore clap CI command dispatch --- Cargo.lock | 1 + tools/ci/Cargo.toml | 1 + tools/ci/src/main.rs | 304 +++++++++++++++++-------------------------- 3 files changed, 119 insertions(+), 187 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d0a01e9f3a5..18004020f46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -899,6 +899,7 @@ name = "ci" version = "0.1.0" dependencies = [ "anyhow", + "clap 4.5.50", "duct", ] diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml index 61bd0a0e422..ba066db1498 100644 --- a/tools/ci/Cargo.toml +++ b/tools/ci/Cargo.toml @@ -5,4 +5,5 @@ edition.workspace = true [dependencies] anyhow.workspace = true +clap.workspace = true duct.workspace = true diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index d36e784d58e..161d7fcdd61 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,214 +1,104 @@ #![allow(clippy::disallowed_macros)] + use anyhow::{bail, Result}; +use clap::{Args, CommandFactory, Parser, Subcommand}; use duct::cmd; use std::fs; use std::path::Path; -struct Command { - name: &'static str, - package: &'static str, -} -struct CommandGroup { - name: &'static str, - commands: &'static [Command], +const README_PATH: &str = "tools/ci/README.md"; + +/// SpacetimeDB CI tasks +/// +/// This tool provides several subcommands for automating CI workflows in SpacetimeDB. +/// +/// It may be invoked via `cargo ci `, or simply `cargo ci` to run all subcommands in +/// sequence. It is mostly designed to be run in CI environments via the github workflows, but can +/// also be run locally. +#[derive(Parser)] +#[command(name = "cargo ci", subcommand_required = false, arg_required_else_help = false)] +struct Cli { + #[command(subcommand)] + cmd: Option, + + /// Skip specified subcommands when running all. + /// + /// When no subcommand is specified, all subcommands are run in sequence. This option allows + /// specifying subcommands to skip when running all. For example, to skip the `unreal-tests` + /// subcommand, use `--skip unreal-tests`. + #[arg(long, default_value = "other-workflows")] + skip: Vec, } -const OTHER_WORKFLOWS_COMMANDS: &[Command] = &[ - Command { - name: "coordinate-internal-tests", - package: "ci-coordinate-internal-tests", - }, - Command { - name: "codeowners-check", - package: "ci-codeowners-check", - }, - Command { - name: "cla-assistant", - package: "ci-cla-assistant", - }, -]; -const COMMANDS: &[Command] = &[ - Command { - name: "test", - package: "ci-test", - }, - Command { - name: "lint", - package: "ci-lint", - }, - Command { - name: "wasm-bindings", - package: "ci-wasm-bindings", - }, - Command { - name: "dlls", - package: "", - }, - Command { - name: "smoketests", - package: "ci-smoketests", - }, - Command { - name: "keynote-bench", - package: "ci-keynote-bench", - }, - Command { - name: "update-flow", - package: "ci-update-flow", - }, - Command { - name: "cli-docs", - package: "ci-cli-docs", - }, - Command { - name: "self-docs", - package: "", - }, - Command { - name: "global-json-policy", - package: "ci-global-json-policy", - }, - Command { - name: "publish-checks", - package: "ci-publish-checks", - }, - Command { - name: "typescript-test", - package: "ci-typescript-test", - }, - Command { - name: "version-upgrade-check", - package: "ci-version-upgrade-check", - }, - Command { - name: "docs", - package: "ci-docs-build", - }, -]; -const COMMAND_GROUPS: &[CommandGroup] = &[CommandGroup { - name: "other-workflows", - commands: OTHER_WORKFLOWS_COMMANDS, -}]; -const DEFAULT_SKIP: &[&str] = &["other-workflows"]; -fn print_help() { - println!("Usage: cargo ci [--skip ] [COMMAND] [ARGS]..."); - println!(); - println!("Commands:"); - for command in COMMANDS { - println!(" {}", command.name); - } - for group in COMMAND_GROUPS { - println!(" {}", group.name); - } +#[derive(Args)] +struct ForwardedArgs { + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + args: Vec, } -fn command_for(name: &str) -> Option<&'static Command> { - COMMANDS.iter().find(|candidate| candidate.name == name) + +#[derive(Subcommand)] +enum CiCmd { + /// Runs tests. + Test(ForwardedArgs), + /// Lints the codebase. + Lint(ForwardedArgs), + /// Tests Wasm bindings. + WasmBindings(ForwardedArgs), + /// Deprecated; use `cargo regen csharp dlls`. + Dlls(ForwardedArgs), + /// Runs smoketests. + Smoketests(ForwardedArgs), + /// Runs the keynote benchmark as a CI performance regression gate. + KeynoteBench(ForwardedArgs), + /// Tests the update flow. + UpdateFlow(ForwardedArgs), + CliDocs(ForwardedArgs), + SelfDocs { + /// Only check for changes, do not generate the docs. + #[arg(long)] + check: bool, + }, + GlobalJsonPolicy(ForwardedArgs), + PublishChecks(ForwardedArgs), + TypescriptTest(ForwardedArgs), + VersionUpgradeCheck(ForwardedArgs), + Docs(ForwardedArgs), + OtherWorkflows { + #[command(subcommand)] + cmd: OtherWorkflowsCmd, + }, } -fn command_group_for(name: &str) -> Option<&'static CommandGroup> { - COMMAND_GROUPS.iter().find(|candidate| candidate.name == name) + +#[derive(Subcommand)] +enum OtherWorkflowsCmd { + CoordinateInternalTests(ForwardedArgs), + CodeownersCheck(ForwardedArgs), + ClaAssistant(ForwardedArgs), } -fn run_command(command: &Command, forwarded: &[String]) -> Result<()> { - if command.name == "self-docs" { - return run_self_docs(forwarded); - } - if command.name == "dlls" { - return run_dlls(forwarded); - } - let mut cargo_args = vec!["run", "--package", command.package, "--"]; - cargo_args.extend(forwarded.iter().map(String::as_str)); + +fn run_package(package: &str, args: &[String]) -> Result<()> { + let mut cargo_args = vec!["run", "--package", package, "--"]; + cargo_args.extend(args.iter().map(String::as_str)); cmd("cargo", cargo_args).run()?; Ok(()) } -fn main() -> Result<()> { - let mut raw = std::env::args().skip(1).collect::>(); - if raw.first().is_some_and(|arg| arg == "-h" || arg == "--help") { - print_help(); - return Ok(()); - } - let mut skips = DEFAULT_SKIP.iter().map(|s| s.to_string()).collect::>(); - let mut i = 0; - while i < raw.len() { - if raw[i] == "--skip" { - if i + 1 >= raw.len() { - bail!("--skip requires a command name"); - } - skips.push(raw.remove(i + 1)); - raw.remove(i); - } else { - i += 1; - } - } - if raw.is_empty() { - for command in COMMANDS { - if !skips.iter().any(|skip| skip == command.name) { - run_command(command, &[])?; - } - } - return Ok(()); - } - let command_name = raw.remove(0); - if let Some(group) = command_group_for(&command_name) { - return run_command_group(group, raw); - } - let Some(command) = command_for(&command_name) else { - bail!("unknown cargo ci command `{command_name}`"); - }; - run_command(command, &raw) -} -fn run_command_group(group: &CommandGroup, mut forwarded: Vec) -> Result<()> { - if forwarded.first().is_some_and(|arg| arg == "-h" || arg == "--help") { - println!("Usage: cargo ci {} ", group.name); - println!(); - println!("Commands:"); - for command in group.commands { - println!(" {}", command.name); - } - return Ok(()); - } - if forwarded.is_empty() { - bail!("cargo ci {} requires a command name", group.name); - } - let command_name = forwarded.remove(0); - let Some(command) = group.commands.iter().find(|candidate| candidate.name == command_name) else { - bail!("unknown cargo ci {} command `{command_name}`", group.name); - }; - run_command(command, &forwarded) -} -fn run_self_docs(args: &[String]) -> Result<()> { - if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { - println!("Usage: cargo ci self-docs [--check]"); - println!(); - println!("Options:"); - println!(" --check Only check for changes, do not generate the docs"); - println!(" -h, --help Print help"); - return Ok(()); - } +fn run_self_docs(check: bool) -> Result<()> { + let readme_content = include_str!("../README.md"); + let path = Path::new(README_PATH); - let check = match args { - [] => false, - [arg] if arg == "--check" => true, - _ => bail!("cargo ci self-docs accepts only --check"), - }; - - let readme = generate_cli_docs(); - let path = Path::new("tools/ci/README.md"); if check { let existing = fs::read_to_string(path).unwrap_or_default(); - if existing != readme { + if existing != readme_content { bail!("README.md is out of date. Please run `cargo ci self-docs` to update it."); } } else { - fs::write(path, readme)?; + fs::write(path, readme_content)?; } Ok(()) } fn run_dlls(args: &[String]) -> Result<()> { - if args.first().is_some_and(|arg| arg == "-h" || arg == "--help") { - println!("Usage: cargo ci dlls"); - return Ok(()); - } if !args.is_empty() { bail!("cargo ci dlls does not accept arguments"); } @@ -217,6 +107,46 @@ fn run_dlls(args: &[String]) -> Result<()> { Ok(()) } -fn generate_cli_docs() -> String { - include_str!("../README.md").to_owned() +fn run_command(cmd: CiCmd) -> Result<()> { + match cmd { + CiCmd::Test(args) => run_package("ci-test", &args.args), + CiCmd::Lint(args) => run_package("ci-lint", &args.args), + CiCmd::WasmBindings(args) => run_package("ci-wasm-bindings", &args.args), + CiCmd::Dlls(args) => run_dlls(&args.args), + CiCmd::Smoketests(args) => run_package("ci-smoketests", &args.args), + CiCmd::KeynoteBench(args) => run_package("ci-keynote-bench", &args.args), + CiCmd::UpdateFlow(args) => run_package("ci-update-flow", &args.args), + CiCmd::CliDocs(args) => run_package("ci-cli-docs", &args.args), + CiCmd::SelfDocs { check } => run_self_docs(check), + CiCmd::GlobalJsonPolicy(args) => run_package("ci-global-json-policy", &args.args), + CiCmd::PublishChecks(args) => run_package("ci-publish-checks", &args.args), + CiCmd::TypescriptTest(args) => run_package("ci-typescript-test", &args.args), + CiCmd::VersionUpgradeCheck(args) => run_package("ci-version-upgrade-check", &args.args), + CiCmd::Docs(args) => run_package("ci-docs-build", &args.args), + CiCmd::OtherWorkflows { cmd } => match cmd { + OtherWorkflowsCmd::CoordinateInternalTests(args) => run_package("ci-coordinate-internal-tests", &args.args), + OtherWorkflowsCmd::CodeownersCheck(args) => run_package("ci-codeowners-check", &args.args), + OtherWorkflowsCmd::ClaAssistant(args) => run_package("ci-cla-assistant", &args.args), + }, + } +} + +fn run_all_clap_subcommands(skip: &[String]) -> Result<()> { + for subcommand in Cli::command().get_subcommands() { + let name = subcommand.get_name(); + if skip.iter().any(|skip| skip == name) { + continue; + } + cmd!("cargo", "ci", name).run()?; + } + Ok(()) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + + match cli.cmd { + Some(cmd) => run_command(cmd), + None => run_all_clap_subcommands(&cli.skip), + } } From 539edb5293946b3b6ca959473a9141d6cea0980f Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 11:04:58 -0400 Subject: [PATCH 17/34] Address cargo ci review comments --- tools/ci/README.md | 187 ++++++----------------------- tools/ci/commands/lint/src/main.rs | 8 +- tools/ci/src/ci_docs.rs | 85 +++++++++++++ tools/ci/src/main.rs | 14 +-- 4 files changed, 133 insertions(+), 161 deletions(-) create mode 100644 tools/ci/src/ci_docs.rs diff --git a/tools/ci/README.md b/tools/ci/README.md index f0563733ee2..df8ad0980bb 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -10,7 +10,7 @@ SpacetimeDB CI tasks This tool provides several subcommands for automating CI workflows in SpacetimeDB. -It may be invoked via `cargo ci `, or simply `cargo ci` to run all subcommands in sequence. It is mostly designed to be run in CI environments via the github workflows, but can also be run locally +It may be invoked via `cargo ci `, or simply `cargo ci` to run all subcommands in sequence. It is mostly designed to be run in CI environments via the github workflows, but can also be run locally. **Usage:** ```bash @@ -19,7 +19,7 @@ Usage: cargo ci [OPTIONS] [COMMAND] **Options:** -- `--skip `: Skip specified subcommands when running all +- `--skip `: Skip specified subcommands when running all. When no subcommand is specified, all subcommands are run in sequence. This option allows specifying subcommands to skip when running all. For example, to skip the `unreal-tests` subcommand, use `--skip unreal-tests`. @@ -27,55 +27,42 @@ When no subcommand is specified, all subcommands are run in sequence. This optio ### `test` -Runs tests - -Runs rust tests, codegens csharp sdk and runs csharp tests. This does not include Unreal tests. This expects to run in a clean git state. - **Usage:** ```bash -Usage: test +Usage: test [ARGS]... ``` **Options:** -- `--help`: Print help (see a summary with '-h') +- `args `: Arguments forwarded to the split CI command package +- `--help`: Print help ### `lint` -Lints the codebase - -Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there are no warnings. - **Usage:** ```bash -Usage: lint +Usage: lint [ARGS]... ``` **Options:** -- `--help`: Print help (see a summary with '-h') +- `args `: Arguments forwarded to the split CI command package +- `--help`: Print help ### `wasm-bindings` -Tests Wasm bindings - -Runs tests for the codegen crate and builds a test module with the wasm bindings. - **Usage:** ```bash -Usage: wasm-bindings +Usage: wasm-bindings [ARGS]... ``` **Options:** -- `--help`: Print help (see a summary with '-h') +- `args `: Arguments forwarded to the split CI command package +- `--help`: Print help ### `dlls` -Deprecated; use `cargo regen csharp dlls`. - -Builds and packs C# DLLs and NuGet packages for local Unity workflows. - **Usage:** ```bash Usage: dlls @@ -83,115 +70,55 @@ Usage: dlls **Options:** -- `--help`: Print help (see a summary with '-h') +- `--help`: Print help ### `smoketests` -Runs smoketests - -Executes the smoketests suite with some default exclusions. - **Usage:** ```bash -Usage: smoketests [OPTIONS] [ARGS]... [COMMAND] -``` - -**Options:** - -- `--server `: Run tests against a remote server instead of spawning local servers. - -When specified, tests will connect to the given URL instead of starting local server instances. Tests that require local server control (like restart tests) will be skipped. - -- `--auth-host `: Use a SpacetimeAuth-issued login for remote-server tests. - -This is required for servers that reject direct server-issued logins for privileged operations. - -Optionally accepts an auth host to pass through to `spacetime login`, for example `--auth-host=https://spacetimedb.com`. - -- `--dotnet `: -- `args `: Additional arguments to pass to the test runner -- `--help`: Print help (see a summary with '-h') - -#### `prepare` - -Only build binaries without running tests - -Use this before running `cargo test --all` to ensure binaries are built. - -**Usage:** -```bash -Usage: prepare -``` - -**Options:** - -- `--help`: Print help (see a summary with '-h') - -#### `check-mod-list` - -**Usage:** -```bash -Usage: check-mod-list +Usage: smoketests [ARGS]... ``` **Options:** +- `args `: Arguments forwarded to the split CI command package - `--help`: Print help -#### `help` - -**Usage:** -```bash -Usage: help [COMMAND]... -``` - -**Options:** - -- `subcommand `: Print help for the subcommand(s) - ### `keynote-bench` -Runs the keynote benchmark as a CI performance regression gate. - -Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. - **Usage:** ```bash -Usage: keynote-bench +Usage: keynote-bench [ARGS]... ``` **Options:** -- `--help`: Print help (see a summary with '-h') +- `args `: Arguments forwarded to the split CI command package +- `--help`: Print help ### `update-flow` -Tests the update flow - -Tests the self-update flow by building the spacetimedb-update binary for the specified target, by default the current target, and performing a self-install into a temporary directory. - **Usage:** ```bash -Usage: update-flow [OPTIONS] +Usage: update-flow [ARGS]... ``` **Options:** -- `--target `: Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms. -- `--github-token-auth`: Whether to enable github token authentication feature when building the update binary. By default this is disabled. -- `--help`: Print help (see a summary with '-h') +- `args `: Arguments forwarded to the split CI command package +- `--help`: Print help ### `cli-docs` **Usage:** ```bash -Usage: cli-docs [OPTIONS] +Usage: cli-docs [ARGS]... ``` **Options:** -- `--spacetime-path `: specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located) -- `--help`: Print help (see a summary with '-h') +- `args `: Arguments forwarded to the split CI command package +- `--help`: Print help ### `self-docs` @@ -203,61 +130,66 @@ Usage: self-docs [OPTIONS] **Options:** - `--check`: Only check for changes, do not generate the docs -- `--help`: Print help (see a summary with '-h') +- `--help`: Print help ### `global-json-policy` **Usage:** ```bash -Usage: global-json-policy +Usage: global-json-policy [ARGS]... ``` **Options:** +- `args `: Arguments forwarded to the split CI command package - `--help`: Print help ### `publish-checks` **Usage:** ```bash -Usage: publish-checks +Usage: publish-checks [ARGS]... ``` **Options:** +- `args `: Arguments forwarded to the split CI command package - `--help`: Print help ### `typescript-test` **Usage:** ```bash -Usage: typescript-test +Usage: typescript-test [ARGS]... ``` **Options:** +- `args `: Arguments forwarded to the split CI command package - `--help`: Print help ### `version-upgrade-check` **Usage:** ```bash -Usage: version-upgrade-check +Usage: version-upgrade-check [ARGS]... ``` **Options:** +- `args `: Arguments forwarded to the split CI command package - `--help`: Print help ### `docs` **Usage:** ```bash -Usage: docs +Usage: docs [ARGS]... ``` **Options:** +- `args `: Arguments forwarded to the split CI command package - `--help`: Print help ### `other-workflows` @@ -275,77 +207,38 @@ Usage: other-workflows **Usage:** ```bash -Usage: coordinate-internal-tests [OPTIONS] --public-sha +Usage: coordinate-internal-tests [ARGS]... ``` **Options:** -- `--public-sha `: Immutable public commit to test -- `--public-pr-number `: Public pull request number, when coordinating a pull request run +- `args `: Arguments forwarded to the split CI command package - `--help`: Print help #### `codeowners-check` **Usage:** ```bash -Usage: codeowners-check --base-ref --pr-number +Usage: codeowners-check [ARGS]... ``` **Options:** -- `--base-ref `: Git ref to compare against, usually origin/ -- `--pr-number `: Pull request number to inspect for approval state +- `args `: Arguments forwarded to the split CI command package - `--help`: Print help #### `cla-assistant` **Usage:** ```bash -Usage: cla-assistant +Usage: cla-assistant [ARGS]... ``` **Options:** +- `args `: Arguments forwarded to the split CI command package - `--help`: Print help -##### `retry` - -**Usage:** -```bash -Usage: retry [OPTIONS] --pr-number -``` - -**Options:** - -- `--pr-number `: Pull request number to check -- `--repo `: Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY -- `--help`: Print help - -##### `status` - -**Usage:** -```bash -Usage: status [OPTIONS] <--pr |--sha > -``` - -**Options:** - -- `--pr `: Pull request number whose head commit should be checked -- `--sha `: Commit SHA to check -- `--repo `: Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY -- `--help`: Print help - -##### `help` - -**Usage:** -```bash -Usage: help [COMMAND]... -``` - -**Options:** - -- `subcommand `: Print help for the subcommand(s) - #### `help` **Usage:** diff --git a/tools/ci/commands/lint/src/main.rs b/tools/ci/commands/lint/src/main.rs index ba977131f0b..d8ada4e7817 100644 --- a/tools/ci/commands/lint/src/main.rs +++ b/tools/ci/commands/lint/src/main.rs @@ -1,7 +1,7 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Context, Result}; -use ci_common::pnpm; +use ci_common::{ensure_repo_root, pnpm}; use duct::cmd; use serde_json::Value; use std::collections::BTreeSet; @@ -270,12 +270,6 @@ fn check_codex_plugin_skills_sync() -> Result<()> { Ok(()) } -fn ensure_repo_root() -> Result<()> { - if !Path::new("Cargo.toml").exists() { - bail!("You must execute this command from the SpacetimeDB repository root (where Cargo.toml is located)"); - } - Ok(()) -} fn tracked_rs_files_under(path: &str) -> Result> { let output = cmd!("git", "ls-files", "--", path).read()?; Ok(output diff --git a/tools/ci/src/ci_docs.rs b/tools/ci/src/ci_docs.rs new file mode 100644 index 00000000000..3fadcdaf0cc --- /dev/null +++ b/tools/ci/src/ci_docs.rs @@ -0,0 +1,85 @@ +use clap::{Command, CommandFactory}; + +use crate::Cli; + +// TODO: use clap_markdown instead of this custom implementation in the future +pub fn generate_cli_docs() -> String { + let mut cli = Cli::command(); + let usage = generate_markdown(&mut cli, 2); + + format!( + "\ +# SpacetimeDB's cargo ci + +## Overview + +This document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options. + +{} +--- + +This document is auto-generated by running: + +```bash +cargo ci self-docs +```", + usage + ) +} + +fn generate_markdown(cmd: &mut Command, heading_level: usize) -> String { + let mut out = String::new(); + + let heading = "#".repeat(heading_level); + out.push_str(&format!("{} `{}`\n\n", heading, cmd.get_name())); + + if let Some(long_about) = cmd.get_long_about() { + out.push_str(&format!("{}\n\n", long_about)); + } + + out.push_str(&format!("**Usage:**\n```bash\n{}\n```\n\n", cmd.render_usage())); + + let mut options = String::new(); + for arg in cmd.get_arguments() { + let mut names = arg + .get_long() + .map(|l| format!("--{}", l)) + .or_else(|| arg.get_short().map(|s| format!("-{}", s))) + .unwrap_or_else(|| arg.get_id().to_string()); + if let Some(value_names) = arg.get_value_names().filter(|_| arg.get_action().takes_values()) { + for value_name in value_names { + names.push_str(&format!(" <{value_name}>")); + } + } + let help = arg + .get_long_help() + .or_else(|| arg.get_help()) + .map(|help| help.to_string()) + .unwrap_or_else(|| { + eprintln!("Warning: argument `{}` is missing help text", arg.get_id()); + "".to_string() + }); + if help.is_empty() { + options.push_str(&format!("- `{names}`:\n")); + } else { + options.push_str(&format!( + "- `{}`: {}\n{}", + names, + help, + if help.lines().count() > 1 { "\n" } else { "" } + )); + } + } + + if !options.is_empty() { + out.push_str("**Options:**\n\n"); + out.push_str(&options); + out.push('\n'); + } + + for sub in cmd.get_subcommands_mut() { + out.push_str(&generate_markdown(sub, heading_level + 1)); + } + + out +} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 161d7fcdd61..d472db14e51 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -8,6 +8,8 @@ use std::path::Path; const README_PATH: &str = "tools/ci/README.md"; +mod ci_docs; + /// SpacetimeDB CI tasks /// /// This tool provides several subcommands for automating CI workflows in SpacetimeDB. @@ -32,6 +34,7 @@ struct Cli { #[derive(Args)] struct ForwardedArgs { + /// Arguments forwarded to the split CI command package. #[arg(trailing_var_arg = true, allow_hyphen_values = true)] args: Vec, } @@ -45,7 +48,7 @@ enum CiCmd { /// Tests Wasm bindings. WasmBindings(ForwardedArgs), /// Deprecated; use `cargo regen csharp dlls`. - Dlls(ForwardedArgs), + Dlls, /// Runs smoketests. Smoketests(ForwardedArgs), /// Runs the keynote benchmark as a CI performance regression gate. @@ -84,7 +87,7 @@ fn run_package(package: &str, args: &[String]) -> Result<()> { } fn run_self_docs(check: bool) -> Result<()> { - let readme_content = include_str!("../README.md"); + let readme_content = ci_docs::generate_cli_docs(); let path = Path::new(README_PATH); if check { @@ -98,10 +101,7 @@ fn run_self_docs(check: bool) -> Result<()> { Ok(()) } -fn run_dlls(args: &[String]) -> Result<()> { - if !args.is_empty() { - bail!("cargo ci dlls does not accept arguments"); - } +fn run_dlls() -> Result<()> { eprintln!("warning: `cargo ci dlls` is deprecated; use `cargo regen csharp dlls` instead"); cmd!("cargo", "regen", "csharp", "dlls").run()?; Ok(()) @@ -112,7 +112,7 @@ fn run_command(cmd: CiCmd) -> Result<()> { CiCmd::Test(args) => run_package("ci-test", &args.args), CiCmd::Lint(args) => run_package("ci-lint", &args.args), CiCmd::WasmBindings(args) => run_package("ci-wasm-bindings", &args.args), - CiCmd::Dlls(args) => run_dlls(&args.args), + CiCmd::Dlls => run_dlls(), CiCmd::Smoketests(args) => run_package("ci-smoketests", &args.args), CiCmd::KeynoteBench(args) => run_package("ci-keynote-bench", &args.args), CiCmd::UpdateFlow(args) => run_package("ci-update-flow", &args.args), From 430a67874c7ebe4eb75d078cdfd888f92fdef024 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 11:08:50 -0400 Subject: [PATCH 18/34] Allow CLI output in split CI commands --- tools/ci/commands/cla-assistant/src/main.rs | 2 ++ tools/ci/commands/coordinate-internal-tests/src/main.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tools/ci/commands/cla-assistant/src/main.rs b/tools/ci/commands/cla-assistant/src/main.rs index a7aa580734b..ac27b6b423b 100644 --- a/tools/ci/commands/cla-assistant/src/main.rs +++ b/tools/ci/commands/cla-assistant/src/main.rs @@ -1,3 +1,5 @@ +#![allow(clippy::disallowed_macros)] + use std::collections::BTreeMap; use std::env; diff --git a/tools/ci/commands/coordinate-internal-tests/src/main.rs b/tools/ci/commands/coordinate-internal-tests/src/main.rs index e46858a32c3..234a9f107cf 100644 --- a/tools/ci/commands/coordinate-internal-tests/src/main.rs +++ b/tools/ci/commands/coordinate-internal-tests/src/main.rs @@ -1,3 +1,5 @@ +#![allow(clippy::disallowed_macros)] + use anyhow::{bail, ensure, Context, Result}; use clap::Parser; use duct::{cmd, Expression}; From 07cf1ed2959b91fd946a6b0b0c7f8665bd224c84 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 11:17:40 -0400 Subject: [PATCH 19/34] Preserve smoketest runner args --- tools/ci/src/main.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index d472db14e51..dba152f8e02 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -86,6 +86,14 @@ fn run_package(package: &str, args: &[String]) -> Result<()> { Ok(()) } +fn run_smoketests(args: &[String]) -> Result<()> { + let mut args = args.to_vec(); + if args.first().is_some_and(|arg| arg.starts_with("--test-")) { + args.insert(0, "--".to_string()); + } + run_package("ci-smoketests", &args) +} + fn run_self_docs(check: bool) -> Result<()> { let readme_content = ci_docs::generate_cli_docs(); let path = Path::new(README_PATH); @@ -113,7 +121,7 @@ fn run_command(cmd: CiCmd) -> Result<()> { CiCmd::Lint(args) => run_package("ci-lint", &args.args), CiCmd::WasmBindings(args) => run_package("ci-wasm-bindings", &args.args), CiCmd::Dlls => run_dlls(), - CiCmd::Smoketests(args) => run_package("ci-smoketests", &args.args), + CiCmd::Smoketests(args) => run_smoketests(&args.args), CiCmd::KeynoteBench(args) => run_package("ci-keynote-bench", &args.args), CiCmd::UpdateFlow(args) => run_package("ci-update-flow", &args.args), CiCmd::CliDocs(args) => run_package("ci-cli-docs", &args.args), From c2ba67729d35db76c5bd2f91c1e77759208280c6 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 13:59:47 -0400 Subject: [PATCH 20/34] Address cargo ci review followups --- tools/ci/README.md | 4 ++-- tools/ci/commands/cla-assistant/src/main.rs | 8 ++------ tools/ci/src/main.rs | 4 ++-- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/tools/ci/README.md b/tools/ci/README.md index df8ad0980bb..108cc61642b 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -10,7 +10,7 @@ SpacetimeDB CI tasks This tool provides several subcommands for automating CI workflows in SpacetimeDB. -It may be invoked via `cargo ci `, or simply `cargo ci` to run all subcommands in sequence. It is mostly designed to be run in CI environments via the github workflows, but can also be run locally. +It may be invoked via `cargo ci `, or simply `cargo ci` to run all subcommands in sequence. It is mostly designed to be run in CI environments via the github workflows, but can also be run locally **Usage:** ```bash @@ -19,7 +19,7 @@ Usage: cargo ci [OPTIONS] [COMMAND] **Options:** -- `--skip `: Skip specified subcommands when running all. +- `--skip `: Skip specified subcommands when running all When no subcommand is specified, all subcommands are run in sequence. This option allows specifying subcommands to skip when running all. For example, to skip the `unreal-tests` subcommand, use `--skip unreal-tests`. diff --git a/tools/ci/commands/cla-assistant/src/main.rs b/tools/ci/commands/cla-assistant/src/main.rs index ac27b6b423b..037111a7663 100644 --- a/tools/ci/commands/cla-assistant/src/main.rs +++ b/tools/ci/commands/cla-assistant/src/main.rs @@ -59,17 +59,13 @@ struct Cli { cmd: ClaAssistantCmd, } -fn run(cmd: ClaAssistantCmd) -> Result<()> { - match cmd { +fn main() -> Result<()> { + match Cli::parse().cmd { ClaAssistantCmd::Retry(args) => retry(args), ClaAssistantCmd::Status(args) => status(args), } } -fn main() -> Result<()> { - run(Cli::parse().cmd) -} - fn retry(args: RetryArgs) -> Result<()> { let repo = Repo::from_arg_or_env(args.repo)?; let token = env::var("GITHUB_TOKEN").context("GITHUB_TOKEN is required")?; diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index dba152f8e02..9176d192a1c 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -16,14 +16,14 @@ mod ci_docs; /// /// It may be invoked via `cargo ci `, or simply `cargo ci` to run all subcommands in /// sequence. It is mostly designed to be run in CI environments via the github workflows, but can -/// also be run locally. +/// also be run locally #[derive(Parser)] #[command(name = "cargo ci", subcommand_required = false, arg_required_else_help = false)] struct Cli { #[command(subcommand)] cmd: Option, - /// Skip specified subcommands when running all. + /// Skip specified subcommands when running all /// /// When no subcommand is specified, all subcommands are run in sequence. This option allows /// specifying subcommands to skip when running all. For example, to skip the `unreal-tests` From d02e783742c9cc9227ecc950dc0684bfcb988ea7 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 14:01:43 -0400 Subject: [PATCH 21/34] Inline coordinate internal tests entrypoint --- tools/ci/commands/coordinate-internal-tests/src/main.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tools/ci/commands/coordinate-internal-tests/src/main.rs b/tools/ci/commands/coordinate-internal-tests/src/main.rs index 234a9f107cf..ec2f4b37307 100644 --- a/tools/ci/commands/coordinate-internal-tests/src/main.rs +++ b/tools/ci/commands/coordinate-internal-tests/src/main.rs @@ -452,8 +452,8 @@ fn write_github_output(name: &str, value: impl std::fmt::Display) -> Result<()> writeln!(output, "{name}={value}").context("failed to write GITHUB_OUTPUT") } -/// Coordinates the public Internal Tests run without checking out or executing private code. -fn coordinate(args: Args) -> Result<()> { +fn main() -> Result<()> { + let args = Args::parse(); let private_source = resolve_private_source(args.public_pr_number)?; let coordinated = match private_source { @@ -468,10 +468,6 @@ fn coordinate(args: Args) -> Result<()> { Ok(()) } -fn main() -> Result<()> { - coordinate(Args::parse()) -} - #[cfg(test)] mod tests { use super::*; From 3d5d8fa65708711f8f7ace7fd0d3f7da38dab10c Mon Sep 17 00:00:00 2001 From: Zeke Foppa <196249+bfops@users.noreply.github.com> Date: Sun, 9 Aug 2026 11:02:31 -0700 Subject: [PATCH 22/34] Apply suggestion from @bfops Signed-off-by: Zeke Foppa <196249+bfops@users.noreply.github.com> --- tools/ci/commands/coordinate-internal-tests/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci/commands/coordinate-internal-tests/src/main.rs b/tools/ci/commands/coordinate-internal-tests/src/main.rs index ec2f4b37307..ba6252e3484 100644 --- a/tools/ci/commands/coordinate-internal-tests/src/main.rs +++ b/tools/ci/commands/coordinate-internal-tests/src/main.rs @@ -452,6 +452,7 @@ fn write_github_output(name: &str, value: impl std::fmt::Display) -> Result<()> writeln!(output, "{name}={value}").context("failed to write GITHUB_OUTPUT") } +/// Coordinates the public Internal Tests run without checking out or executing private code. fn main() -> Result<()> { let args = Args::parse(); let private_source = resolve_private_source(args.public_pr_number)?; From c1e9dbd324c35d71d7ff757e29732b5d5a78b605 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 14:15:02 -0400 Subject: [PATCH 23/34] Render split cargo ci help from child commands --- tools/ci/README.md | 135 +++++++++++++++++++++++++++------------- tools/ci/src/ci_docs.rs | 47 +++++++++++--- tools/ci/src/main.rs | 2 +- 3 files changed, 133 insertions(+), 51 deletions(-) diff --git a/tools/ci/README.md b/tools/ci/README.md index 108cc61642b..aaf5b09477b 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -74,15 +74,46 @@ Usage: dlls ### `smoketests` -**Usage:** -```bash -Usage: smoketests [ARGS]... -``` +**Help:** +```text +This command builds the binaries needed by the smoketests, then runs them. This prevents race +conditions when running tests in parallel with nextest, where multiple test processes might try to +build the same binaries simultaneously -**Options:** +Usage: ci-smoketests [OPTIONS] [ARGS]... [COMMAND] -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help +Commands: + prepare Only build binaries without running tests + check-mod-list + help Print this message or the help of the given subcommand(s) + +Arguments: + [ARGS]... + Additional arguments to pass to the test runner + +Options: + --server + Run tests against a remote server instead of spawning local servers. + + When specified, tests will connect to the given URL instead of starting local server + instances. Tests that require local server control (like restart tests) will be skipped. + + --auth-host[=] + Use a SpacetimeAuth-issued login for remote-server tests. + + This is required for servers that reject direct server-issued logins for privileged + operations. + + Optionally accepts an auth host to pass through to `spacetime login`, for example + `--auth-host=https://spacetimedb.com`. + + --dotnet + [default: true] + [possible values: true, false] + + -h, --help + Print help (see a summary with '-h') +``` ### `keynote-bench` @@ -98,27 +129,37 @@ Usage: keynote-bench [ARGS]... ### `update-flow` -**Usage:** -```bash -Usage: update-flow [ARGS]... -``` +**Help:** +```text +Usage: ci-update-flow [OPTIONS] -**Options:** +Options: + --target + Target triple to build for, by default the current target. Used by github workflows to + check the update flow on multiple platforms. -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help + --github-token-auth + Whether to enable github token authentication feature when building the update binary. By + default this is disabled. + + -h, --help + Print help (see a summary with '-h') +``` ### `cli-docs` -**Usage:** -```bash -Usage: cli-docs [ARGS]... -``` +**Help:** +```text +Usage: ci-cli-docs [OPTIONS] -**Options:** +Options: + --spacetime-path + specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is + located) -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help + -h, --help + Print help (see a summary with '-h') +``` ### `self-docs` @@ -205,39 +246,47 @@ Usage: other-workflows #### `coordinate-internal-tests` -**Usage:** -```bash -Usage: coordinate-internal-tests [ARGS]... -``` +**Help:** +```text +Selects or starts the private workflow for a public Internal Tests run -**Options:** +Usage: ci-coordinate-internal-tests [OPTIONS] --public-sha -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help +Options: + --public-sha + Immutable public commit to test + --public-pr-number + Public pull request number, when coordinating a pull request run + -h, --help + Print help +``` #### `codeowners-check` -**Usage:** -```bash -Usage: codeowners-check [ARGS]... -``` - -**Options:** +**Help:** +```text +Usage: ci-codeowners-check --base-ref --pr-number -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help +Options: + --base-ref Git ref to compare against, usually origin/ + --pr-number Pull request number to inspect for approval state + -h, --help Print help +``` #### `cla-assistant` -**Usage:** -```bash -Usage: cla-assistant [ARGS]... -``` +**Help:** +```text +Usage: ci-cla-assistant -**Options:** +Commands: + retry Retries CLA Assistant if `license/cla` is the only remaining PR blocker + status Returns the `license/cla` status for a pull request or commit SHA + help Print this message or the help of the given subcommand(s) -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help +Options: + -h, --help Print help +``` #### `help` diff --git a/tools/ci/src/ci_docs.rs b/tools/ci/src/ci_docs.rs index 3fadcdaf0cc..8396342aad2 100644 --- a/tools/ci/src/ci_docs.rs +++ b/tools/ci/src/ci_docs.rs @@ -1,13 +1,15 @@ +use anyhow::{Context, Result}; use clap::{Command, CommandFactory}; +use duct::cmd; use crate::Cli; // TODO: use clap_markdown instead of this custom implementation in the future -pub fn generate_cli_docs() -> String { +pub fn generate_cli_docs() -> Result { let mut cli = Cli::command(); - let usage = generate_markdown(&mut cli, 2); + let usage = generate_markdown(&mut cli, 2, &[])?; - format!( + Ok(format!( "\ # SpacetimeDB's cargo ci @@ -24,15 +26,44 @@ This document is auto-generated by running: cargo ci self-docs ```", usage - ) + )) } -fn generate_markdown(cmd: &mut Command, heading_level: usize) -> String { +fn split_command_package(path: &[String]) -> Option<&'static str> { + match path { + [cmd] if cmd == "smoketests" => Some("ci-smoketests"), + [cmd] if cmd == "update-flow" => Some("ci-update-flow"), + [cmd] if cmd == "cli-docs" => Some("ci-cli-docs"), + [group, cmd] if group == "other-workflows" && cmd == "coordinate-internal-tests" => { + Some("ci-coordinate-internal-tests") + } + [group, cmd] if group == "other-workflows" && cmd == "codeowners-check" => Some("ci-codeowners-check"), + [group, cmd] if group == "other-workflows" && cmd == "cla-assistant" => Some("ci-cla-assistant"), + _ => None, + } +} + +fn package_help(package: &str) -> Result { + let help = cmd!("cargo", "run", "--quiet", "--package", package, "--", "--help") + .read() + .with_context(|| format!("failed to render help for {package}"))?; + Ok(help.lines().map(str::trim_end).collect::>().join("\n")) +} + +fn generate_markdown(cmd: &mut Command, heading_level: usize, path: &[String]) -> Result { let mut out = String::new(); let heading = "#".repeat(heading_level); out.push_str(&format!("{} `{}`\n\n", heading, cmd.get_name())); + if let Some(package) = split_command_package(path) { + let help = package_help(package)?; + out.push_str("**Help:**\n```text\n"); + out.push_str(help.trim_end()); + out.push_str("\n```\n\n"); + return Ok(out); + } + if let Some(long_about) = cmd.get_long_about() { out.push_str(&format!("{}\n\n", long_about)); } @@ -78,8 +109,10 @@ fn generate_markdown(cmd: &mut Command, heading_level: usize) -> String { } for sub in cmd.get_subcommands_mut() { - out.push_str(&generate_markdown(sub, heading_level + 1)); + let mut sub_path = path.to_vec(); + sub_path.push(sub.get_name().to_owned()); + out.push_str(&generate_markdown(sub, heading_level + 1, &sub_path)?); } - out + Ok(out) } diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 9176d192a1c..74728e3f972 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -95,7 +95,7 @@ fn run_smoketests(args: &[String]) -> Result<()> { } fn run_self_docs(check: bool) -> Result<()> { - let readme_content = ci_docs::generate_cli_docs(); + let readme_content = ci_docs::generate_cli_docs()?; let path = Path::new(README_PATH); if check { From 323cd2a2010aac1ac86e124dde4bf6e1ef9565d0 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 14:29:43 -0400 Subject: [PATCH 24/34] Delegate split cargo ci help --- tools/ci/src/ci_docs.rs | 27 +++++++++---------------- tools/ci/src/main.rs | 45 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/tools/ci/src/ci_docs.rs b/tools/ci/src/ci_docs.rs index 8396342aad2..bd8c7db00f5 100644 --- a/tools/ci/src/ci_docs.rs +++ b/tools/ci/src/ci_docs.rs @@ -29,24 +29,14 @@ cargo ci self-docs )) } -fn split_command_package(path: &[String]) -> Option<&'static str> { - match path { - [cmd] if cmd == "smoketests" => Some("ci-smoketests"), - [cmd] if cmd == "update-flow" => Some("ci-update-flow"), - [cmd] if cmd == "cli-docs" => Some("ci-cli-docs"), - [group, cmd] if group == "other-workflows" && cmd == "coordinate-internal-tests" => { - Some("ci-coordinate-internal-tests") - } - [group, cmd] if group == "other-workflows" && cmd == "codeowners-check" => Some("ci-codeowners-check"), - [group, cmd] if group == "other-workflows" && cmd == "cla-assistant" => Some("ci-cla-assistant"), - _ => None, - } -} +fn command_help(path: &[String]) -> Result { + let mut args = vec!["run", "--quiet", "--package", "ci", "--"]; + args.extend(path.iter().map(String::as_str)); + args.push("--help"); -fn package_help(package: &str) -> Result { - let help = cmd!("cargo", "run", "--quiet", "--package", package, "--", "--help") + let help = cmd("cargo", args) .read() - .with_context(|| format!("failed to render help for {package}"))?; + .with_context(|| format!("failed to render help for `cargo ci {}`", path.join(" ")))?; Ok(help.lines().map(str::trim_end).collect::>().join("\n")) } @@ -56,8 +46,9 @@ fn generate_markdown(cmd: &mut Command, heading_level: usize, path: &[String]) - let heading = "#".repeat(heading_level); out.push_str(&format!("{} `{}`\n\n", heading, cmd.get_name())); - if let Some(package) = split_command_package(path) { - let help = package_help(package)?; + let path_parts = path.iter().map(String::as_str).collect::>(); + if crate::split_command_help_package(&path_parts).is_some() { + let help = command_help(path)?; out.push_str("**Help:**\n```text\n"); out.push_str(help.trim_end()); out.push_str("\n```\n\n"); diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 74728e3f972..3908621996e 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -86,6 +86,46 @@ fn run_package(package: &str, args: &[String]) -> Result<()> { Ok(()) } +pub(crate) fn split_command_help_package(path: &[&str]) -> Option<&'static str> { + match path { + ["smoketests"] => Some("ci-smoketests"), + ["update-flow"] => Some("ci-update-flow"), + ["cli-docs"] => Some("ci-cli-docs"), + ["other-workflows", "coordinate-internal-tests"] => Some("ci-coordinate-internal-tests"), + ["other-workflows", "codeowners-check"] => Some("ci-codeowners-check"), + ["other-workflows", "cla-assistant"] => Some("ci-cla-assistant"), + _ => None, + } +} + +fn split_help_command(args: &[String]) -> Option<(&'static str, Vec)> { + if let Some(help_path) = args.strip_prefix(&["help".to_string()]) { + for path_len in [2, 1] { + if help_path.len() >= path_len { + let path = help_path[..path_len].iter().map(String::as_str).collect::>(); + if let Some(package) = split_command_help_package(&path) { + return Some((package, vec!["--help".to_string()])); + } + } + } + } + + if !args.iter().any(|arg| arg == "--help" || arg == "-h") { + return None; + } + + for path_len in [2, 1] { + if args.len() >= path_len { + let path = args[..path_len].iter().map(String::as_str).collect::>(); + if let Some(package) = split_command_help_package(&path) { + return Some((package, args[path_len..].to_vec())); + } + } + } + + None +} + fn run_smoketests(args: &[String]) -> Result<()> { let mut args = args.to_vec(); if args.first().is_some_and(|arg| arg.starts_with("--test-")) { @@ -151,6 +191,11 @@ fn run_all_clap_subcommands(skip: &[String]) -> Result<()> { } fn main() -> Result<()> { + let args = std::env::args().skip(1).collect::>(); + if let Some((package, args)) = split_help_command(&args) { + return run_package(package, &args); + } + let cli = Cli::parse(); match cli.cmd { From a7dc141a17f4426062870e80fd9b8e4aecff0750 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 14:48:44 -0400 Subject: [PATCH 25/34] Use split binary help for cargo ci subcommands --- Cargo.lock | 9 + tools/ci/README.md | 161 ++++++++++-------- tools/ci/commands/cla-assistant/src/main.rs | 1 + tools/ci/commands/cli-docs/src/main.rs | 1 + .../ci/commands/codeowners-check/src/main.rs | 1 + .../coordinate-internal-tests/src/main.rs | 1 + tools/ci/commands/docs/Cargo.toml | 1 + tools/ci/commands/docs/src/main.rs | 7 + .../ci/commands/global-json-policy/Cargo.toml | 1 + .../commands/global-json-policy/src/main.rs | 7 + tools/ci/commands/keynote-bench/Cargo.toml | 1 + tools/ci/commands/keynote-bench/src/main.rs | 11 ++ tools/ci/commands/lint/Cargo.toml | 1 + tools/ci/commands/lint/src/main.rs | 10 ++ tools/ci/commands/publish-checks/Cargo.toml | 1 + tools/ci/commands/publish-checks/src/main.rs | 7 + tools/ci/commands/smoketests/src/main.rs | 4 + tools/ci/commands/test/Cargo.toml | 1 + tools/ci/commands/test/src/main.rs | 12 ++ tools/ci/commands/typescript-test/Cargo.toml | 1 + tools/ci/commands/typescript-test/src/main.rs | 7 + tools/ci/commands/update-flow/src/main.rs | 4 + .../commands/version-upgrade-check/Cargo.toml | 1 + .../version-upgrade-check/src/main.rs | 8 + tools/ci/commands/wasm-bindings/Cargo.toml | 1 + tools/ci/commands/wasm-bindings/src/main.rs | 10 ++ tools/ci/src/ci_docs.rs | 22 +-- tools/ci/src/main.rs | 152 +++++++++++++---- 28 files changed, 325 insertions(+), 119 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 18004020f46..8537415c508 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -961,6 +961,7 @@ version = "0.1.0" dependencies = [ "anyhow", "ci-common", + "clap 4.5.50", ] [[package]] @@ -968,6 +969,7 @@ name = "ci-global-json-policy" version = "0.1.0" dependencies = [ "anyhow", + "clap 4.5.50", "duct", ] @@ -976,6 +978,7 @@ name = "ci-keynote-bench" version = "0.1.0" dependencies = [ "anyhow", + "clap 4.5.50", "keynote-bench-harness", "spacetimedb-guard", ] @@ -986,6 +989,7 @@ version = "0.1.0" dependencies = [ "anyhow", "ci-common", + "clap 4.5.50", "duct", "serde_json", ] @@ -995,6 +999,7 @@ name = "ci-publish-checks" version = "0.1.0" dependencies = [ "anyhow", + "clap 4.5.50", "duct", ] @@ -1016,6 +1021,7 @@ version = "0.1.0" dependencies = [ "anyhow", "ci-common", + "clap 4.5.50", "duct", ] @@ -1025,6 +1031,7 @@ version = "0.1.0" dependencies = [ "anyhow", "ci-common", + "clap 4.5.50", "duct", ] @@ -1043,6 +1050,7 @@ name = "ci-version-upgrade-check" version = "0.1.0" dependencies = [ "anyhow", + "clap 4.5.50", "duct", ] @@ -1052,6 +1060,7 @@ version = "0.1.0" dependencies = [ "anyhow", "ci-common", + "clap 4.5.50", "duct", ] diff --git a/tools/ci/README.md b/tools/ci/README.md index aaf5b09477b..f23663b82dc 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -27,39 +27,50 @@ When no subcommand is specified, all subcommands are run in sequence. This optio ### `test` -**Usage:** -```bash -Usage: test [ARGS]... -``` +**Help:** +```text +Runs tests -**Options:** +Runs rust tests, codegens csharp sdk and runs csharp tests. This does not include Unreal tests. This +expects to run in a clean git state. -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help +Usage: ci-test + +Options: + -h, --help + Print help (see a summary with '-h') +``` ### `lint` -**Usage:** -```bash -Usage: lint [ARGS]... -``` +**Help:** +```text +Lints the codebase -**Options:** +Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there are no +warnings. -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help +Usage: ci-lint + +Options: + -h, --help + Print help (see a summary with '-h') +``` ### `wasm-bindings` -**Usage:** -```bash -Usage: wasm-bindings [ARGS]... -``` +**Help:** +```text +Tests Wasm bindings -**Options:** +Runs tests for the codegen crate and builds a test module with the wasm bindings. -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help +Usage: ci-wasm-bindings + +Options: + -h, --help + Print help (see a summary with '-h') +``` ### `dlls` @@ -76,9 +87,9 @@ Usage: dlls **Help:** ```text -This command builds the binaries needed by the smoketests, then runs them. This prevents race -conditions when running tests in parallel with nextest, where multiple test processes might try to -build the same binaries simultaneously +Runs smoketests + +Executes the smoketests suite with some default exclusions. Usage: ci-smoketests [OPTIONS] [ARGS]... [COMMAND] @@ -117,20 +128,30 @@ Options: ### `keynote-bench` -**Usage:** -```bash -Usage: keynote-bench [ARGS]... -``` +**Help:** +```text +Runs the keynote benchmark as a CI performance regression gate. -**Options:** +Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the keynote +SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and fails if +throughput is below 275K TPS for TypeScript or 300K TPS for Rust. -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help +Usage: ci-keynote-bench + +Options: + -h, --help + Print help (see a summary with '-h') +``` ### `update-flow` **Help:** ```text +Tests the update flow + +Tests the self-update flow by building the spacetimedb-update binary for the specified target, by +default the current target, and performing a self-install into a temporary directory. + Usage: ci-update-flow [OPTIONS] Options: @@ -150,6 +171,8 @@ Options: **Help:** ```text +Generates CLI documentation and checks for changes + Usage: ci-cli-docs [OPTIONS] Options: @@ -175,63 +198,63 @@ Usage: self-docs [OPTIONS] ### `global-json-policy` -**Usage:** -```bash -Usage: global-json-policy [ARGS]... -``` +**Help:** +```text +Verify that any non-root global.json files are symlinks to the root global.json -**Options:** +Usage: ci-global-json-policy -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help +Options: + -h, --help Print help +``` ### `publish-checks` -**Usage:** -```bash -Usage: publish-checks [ARGS]... -``` +**Help:** +```text +Checks that publishable crates satisfy publish constraints -**Options:** +Usage: ci-publish-checks -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help +Options: + -h, --help Print help +``` ### `typescript-test` -**Usage:** -```bash -Usage: typescript-test [ARGS]... -``` +**Help:** +```text +Runs TypeScript workspace tests and template build checks -**Options:** +Usage: ci-typescript-test -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help +Options: + -h, --help Print help +``` ### `version-upgrade-check` -**Usage:** -```bash -Usage: version-upgrade-check [ARGS]... -``` +**Help:** +```text +Verifies that the repository version upgrade tool still works -**Options:** +Usage: ci-version-upgrade-check -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help +Options: + -h, --help Print help +``` ### `docs` -**Usage:** -```bash -Usage: docs [ARGS]... -``` +**Help:** +```text +Builds the docs site -**Options:** +Usage: ci-docs-build -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help +Options: + -h, --help Print help +``` ### `other-workflows` @@ -248,7 +271,7 @@ Usage: other-workflows **Help:** ```text -Selects or starts the private workflow for a public Internal Tests run +Selects or starts the private workflow for a public Internal Tests run. Usage: ci-coordinate-internal-tests [OPTIONS] --public-sha @@ -265,6 +288,8 @@ Options: **Help:** ```text +Checks that sensitive CODEOWNERS-controlled files have the required approvals. + Usage: ci-codeowners-check --base-ref --pr-number Options: @@ -277,6 +302,8 @@ Options: **Help:** ```text +Interacts with CLA Assistant. + Usage: ci-cla-assistant Commands: diff --git a/tools/ci/commands/cla-assistant/src/main.rs b/tools/ci/commands/cla-assistant/src/main.rs index 037111a7663..779edbbe842 100644 --- a/tools/ci/commands/cla-assistant/src/main.rs +++ b/tools/ci/commands/cla-assistant/src/main.rs @@ -54,6 +54,7 @@ struct StatusArgs { } #[derive(Parser)] +#[command(about = "Interacts with CLA Assistant.")] struct Cli { #[command(subcommand)] cmd: ClaAssistantCmd, diff --git a/tools/ci/commands/cli-docs/src/main.rs b/tools/ci/commands/cli-docs/src/main.rs index 9c5d29121e7..67870c6b98d 100644 --- a/tools/ci/commands/cli-docs/src/main.rs +++ b/tools/ci/commands/cli-docs/src/main.rs @@ -6,6 +6,7 @@ use duct::cmd; use std::env; #[derive(Parser)] +#[command(about = "Generates CLI documentation and checks for changes")] struct Args { #[arg( long, diff --git a/tools/ci/commands/codeowners-check/src/main.rs b/tools/ci/commands/codeowners-check/src/main.rs index fe0370c1590..5d32e7ead06 100644 --- a/tools/ci/commands/codeowners-check/src/main.rs +++ b/tools/ci/commands/codeowners-check/src/main.rs @@ -9,6 +9,7 @@ use std::path::{Path, PathBuf}; const REPO: &str = "clockworklabs/SpacetimeDB"; #[derive(Parser)] +#[command(about = "Checks that sensitive CODEOWNERS-controlled files have the required approvals.")] struct Args { /// Git ref to compare against, usually origin/. #[arg(long)] diff --git a/tools/ci/commands/coordinate-internal-tests/src/main.rs b/tools/ci/commands/coordinate-internal-tests/src/main.rs index ba6252e3484..850b8152e4d 100644 --- a/tools/ci/commands/coordinate-internal-tests/src/main.rs +++ b/tools/ci/commands/coordinate-internal-tests/src/main.rs @@ -18,6 +18,7 @@ const PRIVATE_DEFAULT_BRANCH: &str = "master"; /// Selects or starts the private workflow for a public Internal Tests run. #[derive(Parser)] +#[command(about = "Selects or starts the private workflow for a public Internal Tests run.")] struct Args { /// Immutable public commit to test. #[arg(long)] diff --git a/tools/ci/commands/docs/Cargo.toml b/tools/ci/commands/docs/Cargo.toml index bcd15b81605..156da133e6b 100644 --- a/tools/ci/commands/docs/Cargo.toml +++ b/tools/ci/commands/docs/Cargo.toml @@ -5,4 +5,5 @@ edition.workspace = true [dependencies] anyhow.workspace = true +clap.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/docs/src/main.rs b/tools/ci/commands/docs/src/main.rs index a47b73b0357..ef3d4f3cfc1 100644 --- a/tools/ci/commands/docs/src/main.rs +++ b/tools/ci/commands/docs/src/main.rs @@ -1,8 +1,15 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; use ci_common::pnpm; +use clap::Parser; + +/// Builds the docs site. +#[derive(Parser)] +struct Args {} fn main() -> Result<()> { + Args::parse(); + pnpm(["install"]).dir("docs").run()?; pnpm(["build"]).dir("docs").run()?; Ok(()) diff --git a/tools/ci/commands/global-json-policy/Cargo.toml b/tools/ci/commands/global-json-policy/Cargo.toml index 0903a7c3478..72b73691f91 100644 --- a/tools/ci/commands/global-json-policy/Cargo.toml +++ b/tools/ci/commands/global-json-policy/Cargo.toml @@ -5,4 +5,5 @@ edition.workspace = true [dependencies] anyhow.workspace = true +clap.workspace = true duct.workspace = true diff --git a/tools/ci/commands/global-json-policy/src/main.rs b/tools/ci/commands/global-json-policy/src/main.rs index 647be569f64..e0a6982196e 100644 --- a/tools/ci/commands/global-json-policy/src/main.rs +++ b/tools/ci/commands/global-json-policy/src/main.rs @@ -1,10 +1,15 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Result}; +use clap::Parser; use duct::cmd; use std::fs; use std::path::{Path, PathBuf}; +/// Verify that any non-root global.json files are symlinks to the root global.json. +#[derive(Parser)] +struct Args {} + fn ensure_repo_root() -> Result<()> { if !Path::new("Cargo.toml").exists() { bail!("You must execute this command from the SpacetimeDB repository root (where Cargo.toml is located)"); @@ -55,5 +60,7 @@ fn check_global_json_policy() -> Result<()> { } fn main() -> Result<()> { + Args::parse(); + check_global_json_policy() } diff --git a/tools/ci/commands/keynote-bench/Cargo.toml b/tools/ci/commands/keynote-bench/Cargo.toml index 74888015d06..c171abbcb2e 100644 --- a/tools/ci/commands/keynote-bench/Cargo.toml +++ b/tools/ci/commands/keynote-bench/Cargo.toml @@ -5,5 +5,6 @@ edition.workspace = true [dependencies] anyhow.workspace = true +clap.workspace = true keynote-bench-harness = { path = "../../../keynote-bench-harness" } spacetimedb-guard.workspace = true diff --git a/tools/ci/commands/keynote-bench/src/main.rs b/tools/ci/commands/keynote-bench/src/main.rs index 0adbeaa63b3..6ed92f6cce8 100644 --- a/tools/ci/commands/keynote-bench/src/main.rs +++ b/tools/ci/commands/keynote-bench/src/main.rs @@ -1,8 +1,19 @@ use anyhow::Result; +use clap::Parser; use keynote_bench_harness::KeynoteBenchConfig; use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; +/// Runs the keynote benchmark as a CI performance regression gate. +/// +/// Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the +/// keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and +/// fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. +#[derive(Parser)] +struct Args {} + fn main() -> Result<()> { + Args::parse(); + let cli_path = ensure_binaries_built(); let server = SpacetimeDbGuard::spawn_in_temp_data_dir(); let server_url = server.host_url.clone(); diff --git a/tools/ci/commands/lint/Cargo.toml b/tools/ci/commands/lint/Cargo.toml index 533b6580542..44bfa5776bd 100644 --- a/tools/ci/commands/lint/Cargo.toml +++ b/tools/ci/commands/lint/Cargo.toml @@ -5,6 +5,7 @@ edition.workspace = true [dependencies] anyhow.workspace = true +clap.workspace = true duct.workspace = true serde_json.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/lint/src/main.rs b/tools/ci/commands/lint/src/main.rs index d8ada4e7817..66450d3c95c 100644 --- a/tools/ci/commands/lint/src/main.rs +++ b/tools/ci/commands/lint/src/main.rs @@ -2,6 +2,7 @@ use anyhow::{bail, Context, Result}; use ci_common::{ensure_repo_root, pnpm}; +use clap::Parser; use duct::cmd; use serde_json::Value; use std::collections::BTreeSet; @@ -10,6 +11,13 @@ use std::fs; use std::path::Path; use std::path::PathBuf; +/// Lints the codebase +/// +/// Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there +/// are no warnings. +#[derive(Parser)] +struct Args {} + fn package_json_pnpm_version(package_manager: &str) -> Option<&str> { package_manager.strip_prefix("pnpm@") } @@ -280,6 +288,8 @@ fn tracked_rs_files_under(path: &str) -> Result> { } fn main() -> Result<()> { + Args::parse(); + ensure_repo_root()?; check_pnpm_release_age_policy()?; check_codex_plugin_skills_sync()?; diff --git a/tools/ci/commands/publish-checks/Cargo.toml b/tools/ci/commands/publish-checks/Cargo.toml index 5c701a42b45..4f2cda1ede6 100644 --- a/tools/ci/commands/publish-checks/Cargo.toml +++ b/tools/ci/commands/publish-checks/Cargo.toml @@ -5,4 +5,5 @@ edition.workspace = true [dependencies] anyhow.workspace = true +clap.workspace = true duct.workspace = true diff --git a/tools/ci/commands/publish-checks/src/main.rs b/tools/ci/commands/publish-checks/src/main.rs index bea2d29243e..e10d89adc0f 100644 --- a/tools/ci/commands/publish-checks/src/main.rs +++ b/tools/ci/commands/publish-checks/src/main.rs @@ -1,8 +1,15 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Result}; +use clap::Parser; use duct::cmd; +/// Checks that publishable crates satisfy publish constraints. +#[derive(Parser)] +struct Args {} + fn main() -> Result<()> { + Args::parse(); + cmd!("bash", "-lc", "test -d venv || python3 -m venv venv").run()?; cmd!("venv/bin/pip3", "install", "argparse", "toml").run()?; diff --git a/tools/ci/commands/smoketests/src/main.rs b/tools/ci/commands/smoketests/src/main.rs index 7e271a0c95d..00ca60319ae 100644 --- a/tools/ci/commands/smoketests/src/main.rs +++ b/tools/ci/commands/smoketests/src/main.rs @@ -17,6 +17,10 @@ fn ensure_repo_root() -> Result<()> { use tempfile::TempDir; #[derive(Parser)] +#[command( + about = "Runs smoketests", + long_about = "Runs smoketests\n\nExecutes the smoketests suite with some default exclusions." +)] /// This command builds the binaries needed by the smoketests, then runs them. This prevents /// race conditions when running tests in parallel with nextest, where multiple test processes /// might try to build the same binaries simultaneously. diff --git a/tools/ci/commands/test/Cargo.toml b/tools/ci/commands/test/Cargo.toml index a81c71dff76..91d98a7f490 100644 --- a/tools/ci/commands/test/Cargo.toml +++ b/tools/ci/commands/test/Cargo.toml @@ -5,5 +5,6 @@ edition.workspace = true [dependencies] anyhow.workspace = true +clap.workspace = true duct.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/test/src/main.rs b/tools/ci/commands/test/src/main.rs index 73cd6d145e7..3124b4b5be1 100644 --- a/tools/ci/commands/test/src/main.rs +++ b/tools/ci/commands/test/src/main.rs @@ -1,8 +1,20 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; use ci_common::pnpm; +use clap::Parser; use duct::cmd; + +/// Runs tests +/// +/// Runs rust tests, codegens csharp sdk and runs csharp tests. +/// This does not include Unreal tests. +/// This expects to run in a clean git state. +#[derive(Parser)] +struct Args {} + fn main() -> Result<()> { + Args::parse(); + pnpm(["build"]).dir("crates/bindings-typescript").run()?; // TODO: This doesn't work on at least user Linux machines, because something here apparently uses `sudo`? diff --git a/tools/ci/commands/typescript-test/Cargo.toml b/tools/ci/commands/typescript-test/Cargo.toml index 91d8f05fe17..170f3618076 100644 --- a/tools/ci/commands/typescript-test/Cargo.toml +++ b/tools/ci/commands/typescript-test/Cargo.toml @@ -5,5 +5,6 @@ edition.workspace = true [dependencies] anyhow.workspace = true +clap.workspace = true duct.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/typescript-test/src/main.rs b/tools/ci/commands/typescript-test/src/main.rs index f4219a21381..e03e97aeb1e 100644 --- a/tools/ci/commands/typescript-test/src/main.rs +++ b/tools/ci/commands/typescript-test/src/main.rs @@ -1,9 +1,16 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Result}; use ci_common::pnpm; +use clap::Parser; use duct::cmd; +/// Runs TypeScript workspace tests and template build checks. +#[derive(Parser)] +struct Args {} + fn main() -> Result<()> { + Args::parse(); + pnpm(["build"]).dir("crates/bindings-typescript").run()?; pnpm(["test"]).dir("crates/bindings-typescript").run()?; pnpm(["generate"]).dir("templates/chat-react-ts").run()?; diff --git a/tools/ci/commands/update-flow/src/main.rs b/tools/ci/commands/update-flow/src/main.rs index f27c8fafe0b..31875593a8e 100644 --- a/tools/ci/commands/update-flow/src/main.rs +++ b/tools/ci/commands/update-flow/src/main.rs @@ -4,6 +4,10 @@ use clap::Parser; use duct::cmd; #[derive(Parser)] +#[command( + about = "Tests the update flow", + long_about = "Tests the update flow\n\nTests the self-update flow by building the spacetimedb-update binary for the specified target, by default the current target, and performing a self-install into a temporary directory." +)] struct Args { #[arg( long, diff --git a/tools/ci/commands/version-upgrade-check/Cargo.toml b/tools/ci/commands/version-upgrade-check/Cargo.toml index 03332a62469..4ab568a5789 100644 --- a/tools/ci/commands/version-upgrade-check/Cargo.toml +++ b/tools/ci/commands/version-upgrade-check/Cargo.toml @@ -5,4 +5,5 @@ edition.workspace = true [dependencies] anyhow.workspace = true +clap.workspace = true duct.workspace = true diff --git a/tools/ci/commands/version-upgrade-check/src/main.rs b/tools/ci/commands/version-upgrade-check/src/main.rs index c4bbd3c4905..97febd5ce60 100644 --- a/tools/ci/commands/version-upgrade-check/src/main.rs +++ b/tools/ci/commands/version-upgrade-check/src/main.rs @@ -1,6 +1,12 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; +use clap::Parser; use duct::cmd; + +/// Verifies that the repository version upgrade tool still works. +#[derive(Parser)] +struct Args {} + fn run_version_upgrade_check() -> Result<()> { cmd!( "cargo", @@ -17,5 +23,7 @@ fn run_version_upgrade_check() -> Result<()> { } fn main() -> Result<()> { + Args::parse(); + run_version_upgrade_check() } diff --git a/tools/ci/commands/wasm-bindings/Cargo.toml b/tools/ci/commands/wasm-bindings/Cargo.toml index 4f6d571f1f9..e807f25ea7e 100644 --- a/tools/ci/commands/wasm-bindings/Cargo.toml +++ b/tools/ci/commands/wasm-bindings/Cargo.toml @@ -5,5 +5,6 @@ edition.workspace = true [dependencies] anyhow.workspace = true +clap.workspace = true duct.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/wasm-bindings/src/main.rs b/tools/ci/commands/wasm-bindings/src/main.rs index 121fd5d7225..a83bef2b1a7 100644 --- a/tools/ci/commands/wasm-bindings/src/main.rs +++ b/tools/ci/commands/wasm-bindings/src/main.rs @@ -1,8 +1,18 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; use ci_common::pnpm; +use clap::Parser; use duct::cmd; + +/// Tests Wasm bindings +/// +/// Runs tests for the codegen crate and builds a test module with the wasm bindings. +#[derive(Parser)] +struct Args {} + fn main() -> Result<()> { + Args::parse(); + pnpm([ "install", "--filter", diff --git a/tools/ci/src/ci_docs.rs b/tools/ci/src/ci_docs.rs index bd8c7db00f5..6ddb0517604 100644 --- a/tools/ci/src/ci_docs.rs +++ b/tools/ci/src/ci_docs.rs @@ -1,12 +1,9 @@ -use anyhow::{Context, Result}; -use clap::{Command, CommandFactory}; -use duct::cmd; - -use crate::Cli; +use anyhow::Result; +use clap::Command; // TODO: use clap_markdown instead of this custom implementation in the future pub fn generate_cli_docs() -> Result { - let mut cli = Cli::command(); + let mut cli = crate::command_with_split_help_overrides()?; let usage = generate_markdown(&mut cli, 2, &[])?; Ok(format!( @@ -29,17 +26,6 @@ cargo ci self-docs )) } -fn command_help(path: &[String]) -> Result { - let mut args = vec!["run", "--quiet", "--package", "ci", "--"]; - args.extend(path.iter().map(String::as_str)); - args.push("--help"); - - let help = cmd("cargo", args) - .read() - .with_context(|| format!("failed to render help for `cargo ci {}`", path.join(" ")))?; - Ok(help.lines().map(str::trim_end).collect::>().join("\n")) -} - fn generate_markdown(cmd: &mut Command, heading_level: usize, path: &[String]) -> Result { let mut out = String::new(); @@ -48,7 +34,7 @@ fn generate_markdown(cmd: &mut Command, heading_level: usize, path: &[String]) - let path_parts = path.iter().map(String::as_str).collect::>(); if crate::split_command_help_package(&path_parts).is_some() { - let help = command_help(path)?; + let help = cmd.render_long_help().to_string(); out.push_str("**Help:**\n```text\n"); out.push_str(help.trim_end()); out.push_str("\n```\n\n"); diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 3908621996e..d148f323723 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,7 +1,7 @@ #![allow(clippy::disallowed_macros)] -use anyhow::{bail, Result}; -use clap::{Args, CommandFactory, Parser, Subcommand}; +use anyhow::{bail, Context, Result}; +use clap::{Args, Command, CommandFactory, FromArgMatches, Parser, Subcommand}; use duct::cmd; use std::fs; use std::path::Path; @@ -41,30 +41,67 @@ struct ForwardedArgs { #[derive(Subcommand)] enum CiCmd { - /// Runs tests. + /// Runs tests + /// + /// Runs rust tests, codegens csharp sdk and runs csharp tests. + /// This does not include Unreal tests. + /// This expects to run in a clean git state. + #[command(override_help = "")] Test(ForwardedArgs), - /// Lints the codebase. + /// Lints the codebase + /// + /// Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there + /// are no warnings. + #[command(override_help = "")] Lint(ForwardedArgs), - /// Tests Wasm bindings. + /// Tests Wasm bindings + /// + /// Runs tests for the codegen crate and builds a test module with the wasm bindings. + #[command(override_help = "")] WasmBindings(ForwardedArgs), /// Deprecated; use `cargo regen csharp dlls`. Dlls, - /// Runs smoketests. + /// Runs smoketests + /// + /// Executes the smoketests suite with some default exclusions. + #[command(override_help = "")] Smoketests(ForwardedArgs), /// Runs the keynote benchmark as a CI performance regression gate. + /// + /// Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the + /// keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and + /// fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. + #[command(override_help = "")] KeynoteBench(ForwardedArgs), - /// Tests the update flow. + /// Tests the update flow + /// + /// Tests the self-update flow by building the spacetimedb-update binary for the specified + /// target, by default the current target, and performing a self-install into a temporary + /// directory. + #[command(override_help = "")] UpdateFlow(ForwardedArgs), + /// Generates CLI documentation and checks for changes + #[command(override_help = "")] CliDocs(ForwardedArgs), SelfDocs { /// Only check for changes, do not generate the docs. #[arg(long)] check: bool, }, + /// Verify that any non-root global.json files are symlinks to the root global.json. + #[command(override_help = "")] GlobalJsonPolicy(ForwardedArgs), + /// Checks that publishable crates satisfy publish constraints. + #[command(override_help = "")] PublishChecks(ForwardedArgs), + /// Runs TypeScript workspace tests and template build checks. + #[command(override_help = "")] TypescriptTest(ForwardedArgs), + /// Verifies that the repository version upgrade tool still works. + #[command(override_help = "")] VersionUpgradeCheck(ForwardedArgs), + /// Builds the docs site. + #[command(override_help = "")] Docs(ForwardedArgs), OtherWorkflows { #[command(subcommand)] @@ -74,8 +111,14 @@ enum CiCmd { #[derive(Subcommand)] enum OtherWorkflowsCmd { + /// Selects or starts the private workflow for a public Internal Tests run. + #[command(override_help = "")] CoordinateInternalTests(ForwardedArgs), + /// Checks that sensitive CODEOWNERS-controlled files have the required approvals. + #[command(override_help = "")] CodeownersCheck(ForwardedArgs), + /// Interacts with CLA Assistant. + #[command(override_help = "")] ClaAssistant(ForwardedArgs), } @@ -88,9 +131,18 @@ fn run_package(package: &str, args: &[String]) -> Result<()> { pub(crate) fn split_command_help_package(path: &[&str]) -> Option<&'static str> { match path { + ["test"] => Some("ci-test"), + ["lint"] => Some("ci-lint"), + ["wasm-bindings"] => Some("ci-wasm-bindings"), ["smoketests"] => Some("ci-smoketests"), + ["keynote-bench"] => Some("ci-keynote-bench"), ["update-flow"] => Some("ci-update-flow"), ["cli-docs"] => Some("ci-cli-docs"), + ["global-json-policy"] => Some("ci-global-json-policy"), + ["publish-checks"] => Some("ci-publish-checks"), + ["typescript-test"] => Some("ci-typescript-test"), + ["version-upgrade-check"] => Some("ci-version-upgrade-check"), + ["docs"] => Some("ci-docs-build"), ["other-workflows", "coordinate-internal-tests"] => Some("ci-coordinate-internal-tests"), ["other-workflows", "codeowners-check"] => Some("ci-codeowners-check"), ["other-workflows", "cla-assistant"] => Some("ci-cla-assistant"), @@ -98,32 +150,69 @@ pub(crate) fn split_command_help_package(path: &[&str]) -> Option<&'static str> } } -fn split_help_command(args: &[String]) -> Option<(&'static str, Vec)> { - if let Some(help_path) = args.strip_prefix(&["help".to_string()]) { - for path_len in [2, 1] { - if help_path.len() >= path_len { - let path = help_path[..path_len].iter().map(String::as_str).collect::>(); - if let Some(package) = split_command_help_package(&path) { - return Some((package, vec!["--help".to_string()])); - } - } - } - } +fn split_command_help(package: &str) -> Result { + let help = cmd!("cargo", "run", "--quiet", "--package", package, "--", "--help") + .read() + .with_context(|| format!("failed to render help for `{package}`"))?; + Ok(help.lines().map(str::trim_end).collect::>().join("\n")) +} - if !args.iter().any(|arg| arg == "--help" || arg == "-h") { - return None; +fn set_split_help_override(command: &mut Command, path: &[&str], help: String) { + let (name, rest) = path.split_first().expect("split command path is never empty"); + let subcommand = command + .find_subcommand_mut(name) + .unwrap_or_else(|| panic!("missing split subcommand `{}`", path.join(" "))); + + if rest.is_empty() { + *subcommand = subcommand.clone().override_help(help); + } else { + set_split_help_override(subcommand, rest, help); } +} - for path_len in [2, 1] { - if args.len() >= path_len { - let path = args[..path_len].iter().map(String::as_str).collect::>(); - if let Some(package) = split_command_help_package(&path) { - return Some((package, args[path_len..].to_vec())); - } - } +pub(crate) fn command_with_split_help_overrides() -> Result { + let mut command = Cli::command(); + + for (path, package) in [ + (&["test"][..], "ci-test"), + (&["lint"], "ci-lint"), + (&["wasm-bindings"], "ci-wasm-bindings"), + (&["smoketests"], "ci-smoketests"), + (&["keynote-bench"], "ci-keynote-bench"), + (&["update-flow"], "ci-update-flow"), + (&["cli-docs"], "ci-cli-docs"), + (&["global-json-policy"], "ci-global-json-policy"), + (&["publish-checks"], "ci-publish-checks"), + (&["typescript-test"], "ci-typescript-test"), + (&["version-upgrade-check"], "ci-version-upgrade-check"), + (&["docs"], "ci-docs-build"), + ( + &["other-workflows", "coordinate-internal-tests"], + "ci-coordinate-internal-tests", + ), + (&["other-workflows", "codeowners-check"], "ci-codeowners-check"), + (&["other-workflows", "cla-assistant"], "ci-cla-assistant"), + ] { + let help = split_command_help(package)?; + set_split_help_override(&mut command, path, help); } - None + Ok(command) +} + +fn should_load_split_help(args: &[String]) -> bool { + args.iter().any(|arg| arg == "--help" || arg == "-h" || arg == "help") + || args.get(1).is_some_and(|arg| arg == "self-docs") +} + +fn parse_cli() -> Result { + let args = std::env::args().collect::>(); + if should_load_split_help(&args) { + let matches = command_with_split_help_overrides()?.get_matches_from(args); + Ok(Cli::from_arg_matches(&matches)?) + } else { + Ok(Cli::parse_from(args)) + } } fn run_smoketests(args: &[String]) -> Result<()> { @@ -191,12 +280,7 @@ fn run_all_clap_subcommands(skip: &[String]) -> Result<()> { } fn main() -> Result<()> { - let args = std::env::args().skip(1).collect::>(); - if let Some((package, args)) = split_help_command(&args) { - return run_package(package, &args); - } - - let cli = Cli::parse(); + let cli = parse_cli()?; match cli.cmd { Some(cmd) => run_command(cmd), From cb1cdf593161448847610d9ee01341bea2a296a8 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 14:57:56 -0400 Subject: [PATCH 26/34] Trim split help docs output --- tools/ci/README.md | 91 +++++++++-------------------------------- tools/ci/src/ci_docs.rs | 7 ++-- tools/ci/src/main.rs | 1 + 3 files changed, 23 insertions(+), 76 deletions(-) diff --git a/tools/ci/README.md b/tools/ci/README.md index f23663b82dc..f49a3dc503c 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -27,40 +27,30 @@ When no subcommand is specified, all subcommands are run in sequence. This optio ### `test` -**Help:** -```text Runs tests -Runs rust tests, codegens csharp sdk and runs csharp tests. This does not include Unreal tests. This -expects to run in a clean git state. +Runs rust tests, codegens csharp sdk and runs csharp tests. This does not include Unreal tests. This expects to run in a clean git state. Usage: ci-test Options: -h, --help Print help (see a summary with '-h') -``` ### `lint` -**Help:** -```text Lints the codebase -Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there are no -warnings. +Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there are no warnings. Usage: ci-lint Options: -h, --help Print help (see a summary with '-h') -``` ### `wasm-bindings` -**Help:** -```text Tests Wasm bindings Runs tests for the codegen crate and builds a test module with the wasm bindings. @@ -70,10 +60,11 @@ Usage: ci-wasm-bindings Options: -h, --help Print help (see a summary with '-h') -``` ### `dlls` +Deprecated; use `cargo regen csharp dlls` + **Usage:** ```bash Usage: dlls @@ -85,8 +76,6 @@ Usage: dlls ### `smoketests` -**Help:** -```text Runs smoketests Executes the smoketests suite with some default exclusions. @@ -106,17 +95,14 @@ Options: --server Run tests against a remote server instead of spawning local servers. - When specified, tests will connect to the given URL instead of starting local server - instances. Tests that require local server control (like restart tests) will be skipped. + When specified, tests will connect to the given URL instead of starting local server instances. Tests that require local server control (like restart tests) will be skipped. --auth-host[=] Use a SpacetimeAuth-issued login for remote-server tests. - This is required for servers that reject direct server-issued logins for privileged - operations. + This is required for servers that reject direct server-issued logins for privileged operations. - Optionally accepts an auth host to pass through to `spacetime login`, for example - `--auth-host=https://spacetimedb.com`. + Optionally accepts an auth host to pass through to `spacetime login`, for example `--auth-host=https://spacetimedb.com`. --dotnet [default: true] @@ -124,65 +110,49 @@ Options: -h, --help Print help (see a summary with '-h') -``` ### `keynote-bench` -**Help:** -```text Runs the keynote benchmark as a CI performance regression gate. -Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the keynote -SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and fails if -throughput is below 275K TPS for TypeScript or 300K TPS for Rust. +Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. Usage: ci-keynote-bench Options: -h, --help Print help (see a summary with '-h') -``` ### `update-flow` -**Help:** -```text Tests the update flow -Tests the self-update flow by building the spacetimedb-update binary for the specified target, by -default the current target, and performing a self-install into a temporary directory. +Tests the self-update flow by building the spacetimedb-update binary for the specified target, by default the current target, and performing a self-install into a temporary directory. Usage: ci-update-flow [OPTIONS] Options: --target - Target triple to build for, by default the current target. Used by github workflows to - check the update flow on multiple platforms. + Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms. --github-token-auth - Whether to enable github token authentication feature when building the update binary. By - default this is disabled. + Whether to enable github token authentication feature when building the update binary. By default this is disabled. -h, --help Print help (see a summary with '-h') -``` ### `cli-docs` -**Help:** -```text Generates CLI documentation and checks for changes Usage: ci-cli-docs [OPTIONS] Options: --spacetime-path - specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is - located) + specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located) -h, --help Print help (see a summary with '-h') -``` ### `self-docs` @@ -198,63 +168,48 @@ Usage: self-docs [OPTIONS] ### `global-json-policy` -**Help:** -```text Verify that any non-root global.json files are symlinks to the root global.json Usage: ci-global-json-policy Options: -h, --help Print help -``` ### `publish-checks` -**Help:** -```text Checks that publishable crates satisfy publish constraints Usage: ci-publish-checks Options: -h, --help Print help -``` ### `typescript-test` -**Help:** -```text Runs TypeScript workspace tests and template build checks Usage: ci-typescript-test Options: -h, --help Print help -``` ### `version-upgrade-check` -**Help:** -```text Verifies that the repository version upgrade tool still works Usage: ci-version-upgrade-check Options: -h, --help Print help -``` ### `docs` -**Help:** -```text Builds the docs site Usage: ci-docs-build Options: -h, --help Print help -``` ### `other-workflows` @@ -269,25 +224,17 @@ Usage: other-workflows #### `coordinate-internal-tests` -**Help:** -```text Selects or starts the private workflow for a public Internal Tests run. Usage: ci-coordinate-internal-tests [OPTIONS] --public-sha Options: - --public-sha - Immutable public commit to test - --public-pr-number - Public pull request number, when coordinating a pull request run - -h, --help - Print help -``` + --public-sha Immutable public commit to test + --public-pr-number Public pull request number, when coordinating a pull request run + -h, --help Print help #### `codeowners-check` -**Help:** -```text Checks that sensitive CODEOWNERS-controlled files have the required approvals. Usage: ci-codeowners-check --base-ref --pr-number @@ -296,12 +243,9 @@ Options: --base-ref Git ref to compare against, usually origin/ --pr-number Pull request number to inspect for approval state -h, --help Print help -``` #### `cla-assistant` -**Help:** -```text Interacts with CLA Assistant. Usage: ci-cla-assistant @@ -313,10 +257,11 @@ Commands: Options: -h, --help Print help -``` #### `help` +Print this message or the help of the given subcommand(s) + **Usage:** ```bash Usage: help [COMMAND]... @@ -328,6 +273,8 @@ Usage: help [COMMAND]... ### `help` +Print this message or the help of the given subcommand(s) + **Usage:** ```bash Usage: help [COMMAND]... diff --git a/tools/ci/src/ci_docs.rs b/tools/ci/src/ci_docs.rs index 6ddb0517604..d861c5f7dca 100644 --- a/tools/ci/src/ci_docs.rs +++ b/tools/ci/src/ci_docs.rs @@ -35,14 +35,13 @@ fn generate_markdown(cmd: &mut Command, heading_level: usize, path: &[String]) - let path_parts = path.iter().map(String::as_str).collect::>(); if crate::split_command_help_package(&path_parts).is_some() { let help = cmd.render_long_help().to_string(); - out.push_str("**Help:**\n```text\n"); out.push_str(help.trim_end()); - out.push_str("\n```\n\n"); + out.push_str("\n\n"); return Ok(out); } - if let Some(long_about) = cmd.get_long_about() { - out.push_str(&format!("{}\n\n", long_about)); + if let Some(about) = cmd.get_long_about().or_else(|| cmd.get_about()) { + out.push_str(&format!("{}\n\n", about)); } out.push_str(&format!("**Usage:**\n```bash\n{}\n```\n\n", cmd.render_usage())); diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index d148f323723..6c23586e901 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -152,6 +152,7 @@ pub(crate) fn split_command_help_package(path: &[&str]) -> Option<&'static str> fn split_command_help(package: &str) -> Result { let help = cmd!("cargo", "run", "--quiet", "--package", package, "--", "--help") + .env("COLUMNS", "1000") .read() .with_context(|| format!("failed to render help for `{package}`"))?; Ok(help.lines().map(str::trim_end).collect::>().join("\n")) From 40bac67580880f773f82c0b3f23acd0d08402a7b Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 15:00:27 -0400 Subject: [PATCH 27/34] Keep simple CI docs compact --- tools/ci/README.md | 107 +++++++++++++++++++++++++--------------- tools/ci/src/ci_docs.rs | 23 ++++++++- tools/ci/src/main.rs | 21 -------- 3 files changed, 88 insertions(+), 63 deletions(-) diff --git a/tools/ci/README.md b/tools/ci/README.md index f49a3dc503c..8284d65328b 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -31,11 +31,15 @@ Runs tests Runs rust tests, codegens csharp sdk and runs csharp tests. This does not include Unreal tests. This expects to run in a clean git state. -Usage: ci-test +**Usage:** +```bash +Usage: test [ARGS]... +``` -Options: - -h, --help - Print help (see a summary with '-h') +**Options:** + +- `args `: Arguments forwarded to the split CI command package +- `--help`: Print help (see a summary with '-h') ### `lint` @@ -43,11 +47,15 @@ Lints the codebase Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there are no warnings. -Usage: ci-lint +**Usage:** +```bash +Usage: lint [ARGS]... +``` -Options: - -h, --help - Print help (see a summary with '-h') +**Options:** + +- `args `: Arguments forwarded to the split CI command package +- `--help`: Print help (see a summary with '-h') ### `wasm-bindings` @@ -55,11 +63,15 @@ Tests Wasm bindings Runs tests for the codegen crate and builds a test module with the wasm bindings. -Usage: ci-wasm-bindings +**Usage:** +```bash +Usage: wasm-bindings [ARGS]... +``` -Options: - -h, --help - Print help (see a summary with '-h') +**Options:** + +- `args `: Arguments forwarded to the split CI command package +- `--help`: Print help (see a summary with '-h') ### `dlls` @@ -117,11 +129,15 @@ Runs the keynote benchmark as a CI performance regression gate. Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. -Usage: ci-keynote-bench +**Usage:** +```bash +Usage: keynote-bench [ARGS]... +``` + +**Options:** -Options: - -h, --help - Print help (see a summary with '-h') +- `args `: Arguments forwarded to the split CI command package +- `--help`: Print help (see a summary with '-h') ### `update-flow` @@ -168,48 +184,63 @@ Usage: self-docs [OPTIONS] ### `global-json-policy` -Verify that any non-root global.json files are symlinks to the root global.json +**Usage:** +```bash +Usage: global-json-policy [ARGS]... +``` -Usage: ci-global-json-policy +**Options:** -Options: - -h, --help Print help +- `args `: Arguments forwarded to the split CI command package +- `--help`: Print help ### `publish-checks` -Checks that publishable crates satisfy publish constraints +**Usage:** +```bash +Usage: publish-checks [ARGS]... +``` -Usage: ci-publish-checks +**Options:** -Options: - -h, --help Print help +- `args `: Arguments forwarded to the split CI command package +- `--help`: Print help ### `typescript-test` -Runs TypeScript workspace tests and template build checks +**Usage:** +```bash +Usage: typescript-test [ARGS]... +``` -Usage: ci-typescript-test +**Options:** -Options: - -h, --help Print help +- `args `: Arguments forwarded to the split CI command package +- `--help`: Print help ### `version-upgrade-check` -Verifies that the repository version upgrade tool still works +**Usage:** +```bash +Usage: version-upgrade-check [ARGS]... +``` -Usage: ci-version-upgrade-check +**Options:** -Options: - -h, --help Print help +- `args `: Arguments forwarded to the split CI command package +- `--help`: Print help ### `docs` -Builds the docs site +**Usage:** +```bash +Usage: docs [ARGS]... +``` -Usage: ci-docs-build +**Options:** -Options: - -h, --help Print help +- `args `: Arguments forwarded to the split CI command package +- `--help`: Print help ### `other-workflows` @@ -260,8 +291,6 @@ Options: #### `help` -Print this message or the help of the given subcommand(s) - **Usage:** ```bash Usage: help [COMMAND]... @@ -273,8 +302,6 @@ Usage: help [COMMAND]... ### `help` -Print this message or the help of the given subcommand(s) - **Usage:** ```bash Usage: help [COMMAND]... diff --git a/tools/ci/src/ci_docs.rs b/tools/ci/src/ci_docs.rs index d861c5f7dca..646161a5132 100644 --- a/tools/ci/src/ci_docs.rs +++ b/tools/ci/src/ci_docs.rs @@ -33,14 +33,21 @@ fn generate_markdown(cmd: &mut Command, heading_level: usize, path: &[String]) - out.push_str(&format!("{} `{}`\n\n", heading, cmd.get_name())); let path_parts = path.iter().map(String::as_str).collect::>(); - if crate::split_command_help_package(&path_parts).is_some() { + if should_render_split_help(&path_parts) { let help = cmd.render_long_help().to_string(); out.push_str(help.trim_end()); out.push_str("\n\n"); return Ok(out); } - if let Some(about) = cmd.get_long_about().or_else(|| cmd.get_about()) { + let about = cmd.get_long_about().or_else(|| { + if path_parts.as_slice() == ["dlls"] { + cmd.get_about() + } else { + None + } + }); + if let Some(about) = about { out.push_str(&format!("{}\n\n", about)); } @@ -92,3 +99,15 @@ fn generate_markdown(cmd: &mut Command, heading_level: usize, path: &[String]) - Ok(out) } + +fn should_render_split_help(path: &[&str]) -> bool { + matches!( + path, + ["smoketests"] + | ["update-flow"] + | ["cli-docs"] + | ["other-workflows", "coordinate-internal-tests"] + | ["other-workflows", "codeowners-check"] + | ["other-workflows", "cla-assistant"] + ) +} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 6c23586e901..e9dbdc21624 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -129,27 +129,6 @@ fn run_package(package: &str, args: &[String]) -> Result<()> { Ok(()) } -pub(crate) fn split_command_help_package(path: &[&str]) -> Option<&'static str> { - match path { - ["test"] => Some("ci-test"), - ["lint"] => Some("ci-lint"), - ["wasm-bindings"] => Some("ci-wasm-bindings"), - ["smoketests"] => Some("ci-smoketests"), - ["keynote-bench"] => Some("ci-keynote-bench"), - ["update-flow"] => Some("ci-update-flow"), - ["cli-docs"] => Some("ci-cli-docs"), - ["global-json-policy"] => Some("ci-global-json-policy"), - ["publish-checks"] => Some("ci-publish-checks"), - ["typescript-test"] => Some("ci-typescript-test"), - ["version-upgrade-check"] => Some("ci-version-upgrade-check"), - ["docs"] => Some("ci-docs-build"), - ["other-workflows", "coordinate-internal-tests"] => Some("ci-coordinate-internal-tests"), - ["other-workflows", "codeowners-check"] => Some("ci-codeowners-check"), - ["other-workflows", "cla-assistant"] => Some("ci-cla-assistant"), - _ => None, - } -} - fn split_command_help(package: &str) -> Result { let help = cmd!("cargo", "run", "--quiet", "--package", package, "--", "--help") .env("COLUMNS", "1000") From a841ee0a89835350fb1938ff7efcd0e43683ca1e Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 15:02:27 -0400 Subject: [PATCH 28/34] Remove cargo ci self-docs --- .github/workflows/ci.yml | 33 ---- tools/ci/README.md | 321 --------------------------------------- tools/ci/src/ci_docs.rs | 113 -------------- tools/ci/src/main.rs | 30 +--- 4 files changed, 1 insertion(+), 496 deletions(-) delete mode 100644 tools/ci/README.md delete mode 100644 tools/ci/src/ci_docs.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c39b4a2458e..a01744937dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -681,39 +681,6 @@ jobs: cargo test -- --test-threads=1 ' - ci_command_docs: - needs: [merge_queue_noop] - if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} - name: Check CI command docs - runs-on: spacetimedb-new-runner-2 - env: - RUST_BACKTRACE: full - steps: - - name: Find Git ref - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - run: | - PR_NUMBER="${{ github.event.inputs.pr_number || null }}" - if test -n "${PR_NUMBER}"; then - GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" - else - GIT_REF="${{ github.ref }}" - fi - echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - - - name: Checkout sources - uses: actions/checkout@v4 - with: - ref: ${{ env.GIT_REF }} - - - uses: dsherret/rust-toolchain-file@v1 - - name: Set default rust toolchain - run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - - name: Check for docs change - run: cargo ci self-docs --check - cli_docs: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} diff --git a/tools/ci/README.md b/tools/ci/README.md deleted file mode 100644 index 8284d65328b..00000000000 --- a/tools/ci/README.md +++ /dev/null @@ -1,321 +0,0 @@ -# SpacetimeDB's cargo ci - -## Overview - -This document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options. - -## `cargo ci` - -SpacetimeDB CI tasks - -This tool provides several subcommands for automating CI workflows in SpacetimeDB. - -It may be invoked via `cargo ci `, or simply `cargo ci` to run all subcommands in sequence. It is mostly designed to be run in CI environments via the github workflows, but can also be run locally - -**Usage:** -```bash -Usage: cargo ci [OPTIONS] [COMMAND] -``` - -**Options:** - -- `--skip `: Skip specified subcommands when running all - -When no subcommand is specified, all subcommands are run in sequence. This option allows specifying subcommands to skip when running all. For example, to skip the `unreal-tests` subcommand, use `--skip unreal-tests`. - -- `--help`: Print help (see a summary with '-h') - -### `test` - -Runs tests - -Runs rust tests, codegens csharp sdk and runs csharp tests. This does not include Unreal tests. This expects to run in a clean git state. - -**Usage:** -```bash -Usage: test [ARGS]... -``` - -**Options:** - -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help (see a summary with '-h') - -### `lint` - -Lints the codebase - -Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there are no warnings. - -**Usage:** -```bash -Usage: lint [ARGS]... -``` - -**Options:** - -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help (see a summary with '-h') - -### `wasm-bindings` - -Tests Wasm bindings - -Runs tests for the codegen crate and builds a test module with the wasm bindings. - -**Usage:** -```bash -Usage: wasm-bindings [ARGS]... -``` - -**Options:** - -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help (see a summary with '-h') - -### `dlls` - -Deprecated; use `cargo regen csharp dlls` - -**Usage:** -```bash -Usage: dlls -``` - -**Options:** - -- `--help`: Print help - -### `smoketests` - -Runs smoketests - -Executes the smoketests suite with some default exclusions. - -Usage: ci-smoketests [OPTIONS] [ARGS]... [COMMAND] - -Commands: - prepare Only build binaries without running tests - check-mod-list - help Print this message or the help of the given subcommand(s) - -Arguments: - [ARGS]... - Additional arguments to pass to the test runner - -Options: - --server - Run tests against a remote server instead of spawning local servers. - - When specified, tests will connect to the given URL instead of starting local server instances. Tests that require local server control (like restart tests) will be skipped. - - --auth-host[=] - Use a SpacetimeAuth-issued login for remote-server tests. - - This is required for servers that reject direct server-issued logins for privileged operations. - - Optionally accepts an auth host to pass through to `spacetime login`, for example `--auth-host=https://spacetimedb.com`. - - --dotnet - [default: true] - [possible values: true, false] - - -h, --help - Print help (see a summary with '-h') - -### `keynote-bench` - -Runs the keynote benchmark as a CI performance regression gate. - -Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. - -**Usage:** -```bash -Usage: keynote-bench [ARGS]... -``` - -**Options:** - -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help (see a summary with '-h') - -### `update-flow` - -Tests the update flow - -Tests the self-update flow by building the spacetimedb-update binary for the specified target, by default the current target, and performing a self-install into a temporary directory. - -Usage: ci-update-flow [OPTIONS] - -Options: - --target - Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms. - - --github-token-auth - Whether to enable github token authentication feature when building the update binary. By default this is disabled. - - -h, --help - Print help (see a summary with '-h') - -### `cli-docs` - -Generates CLI documentation and checks for changes - -Usage: ci-cli-docs [OPTIONS] - -Options: - --spacetime-path - specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located) - - -h, --help - Print help (see a summary with '-h') - -### `self-docs` - -**Usage:** -```bash -Usage: self-docs [OPTIONS] -``` - -**Options:** - -- `--check`: Only check for changes, do not generate the docs -- `--help`: Print help - -### `global-json-policy` - -**Usage:** -```bash -Usage: global-json-policy [ARGS]... -``` - -**Options:** - -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help - -### `publish-checks` - -**Usage:** -```bash -Usage: publish-checks [ARGS]... -``` - -**Options:** - -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help - -### `typescript-test` - -**Usage:** -```bash -Usage: typescript-test [ARGS]... -``` - -**Options:** - -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help - -### `version-upgrade-check` - -**Usage:** -```bash -Usage: version-upgrade-check [ARGS]... -``` - -**Options:** - -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help - -### `docs` - -**Usage:** -```bash -Usage: docs [ARGS]... -``` - -**Options:** - -- `args `: Arguments forwarded to the split CI command package -- `--help`: Print help - -### `other-workflows` - -**Usage:** -```bash -Usage: other-workflows -``` - -**Options:** - -- `--help`: Print help - -#### `coordinate-internal-tests` - -Selects or starts the private workflow for a public Internal Tests run. - -Usage: ci-coordinate-internal-tests [OPTIONS] --public-sha - -Options: - --public-sha Immutable public commit to test - --public-pr-number Public pull request number, when coordinating a pull request run - -h, --help Print help - -#### `codeowners-check` - -Checks that sensitive CODEOWNERS-controlled files have the required approvals. - -Usage: ci-codeowners-check --base-ref --pr-number - -Options: - --base-ref Git ref to compare against, usually origin/ - --pr-number Pull request number to inspect for approval state - -h, --help Print help - -#### `cla-assistant` - -Interacts with CLA Assistant. - -Usage: ci-cla-assistant - -Commands: - retry Retries CLA Assistant if `license/cla` is the only remaining PR blocker - status Returns the `license/cla` status for a pull request or commit SHA - help Print this message or the help of the given subcommand(s) - -Options: - -h, --help Print help - -#### `help` - -**Usage:** -```bash -Usage: help [COMMAND]... -``` - -**Options:** - -- `subcommand `: Print help for the subcommand(s) - -### `help` - -**Usage:** -```bash -Usage: help [COMMAND]... -``` - -**Options:** - -- `subcommand `: Print help for the subcommand(s) - - ---- - -This document is auto-generated by running: - -```bash -cargo ci self-docs -``` \ No newline at end of file diff --git a/tools/ci/src/ci_docs.rs b/tools/ci/src/ci_docs.rs deleted file mode 100644 index 646161a5132..00000000000 --- a/tools/ci/src/ci_docs.rs +++ /dev/null @@ -1,113 +0,0 @@ -use anyhow::Result; -use clap::Command; - -// TODO: use clap_markdown instead of this custom implementation in the future -pub fn generate_cli_docs() -> Result { - let mut cli = crate::command_with_split_help_overrides()?; - let usage = generate_markdown(&mut cli, 2, &[])?; - - Ok(format!( - "\ -# SpacetimeDB's cargo ci - -## Overview - -This document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options. - -{} ---- - -This document is auto-generated by running: - -```bash -cargo ci self-docs -```", - usage - )) -} - -fn generate_markdown(cmd: &mut Command, heading_level: usize, path: &[String]) -> Result { - let mut out = String::new(); - - let heading = "#".repeat(heading_level); - out.push_str(&format!("{} `{}`\n\n", heading, cmd.get_name())); - - let path_parts = path.iter().map(String::as_str).collect::>(); - if should_render_split_help(&path_parts) { - let help = cmd.render_long_help().to_string(); - out.push_str(help.trim_end()); - out.push_str("\n\n"); - return Ok(out); - } - - let about = cmd.get_long_about().or_else(|| { - if path_parts.as_slice() == ["dlls"] { - cmd.get_about() - } else { - None - } - }); - if let Some(about) = about { - out.push_str(&format!("{}\n\n", about)); - } - - out.push_str(&format!("**Usage:**\n```bash\n{}\n```\n\n", cmd.render_usage())); - - let mut options = String::new(); - for arg in cmd.get_arguments() { - let mut names = arg - .get_long() - .map(|l| format!("--{}", l)) - .or_else(|| arg.get_short().map(|s| format!("-{}", s))) - .unwrap_or_else(|| arg.get_id().to_string()); - if let Some(value_names) = arg.get_value_names().filter(|_| arg.get_action().takes_values()) { - for value_name in value_names { - names.push_str(&format!(" <{value_name}>")); - } - } - let help = arg - .get_long_help() - .or_else(|| arg.get_help()) - .map(|help| help.to_string()) - .unwrap_or_else(|| { - eprintln!("Warning: argument `{}` is missing help text", arg.get_id()); - "".to_string() - }); - if help.is_empty() { - options.push_str(&format!("- `{names}`:\n")); - } else { - options.push_str(&format!( - "- `{}`: {}\n{}", - names, - help, - if help.lines().count() > 1 { "\n" } else { "" } - )); - } - } - - if !options.is_empty() { - out.push_str("**Options:**\n\n"); - out.push_str(&options); - out.push('\n'); - } - - for sub in cmd.get_subcommands_mut() { - let mut sub_path = path.to_vec(); - sub_path.push(sub.get_name().to_owned()); - out.push_str(&generate_markdown(sub, heading_level + 1, &sub_path)?); - } - - Ok(out) -} - -fn should_render_split_help(path: &[&str]) -> bool { - matches!( - path, - ["smoketests"] - | ["update-flow"] - | ["cli-docs"] - | ["other-workflows", "coordinate-internal-tests"] - | ["other-workflows", "codeowners-check"] - | ["other-workflows", "cla-assistant"] - ) -} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index e9dbdc21624..b0bc7d9182a 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,14 +1,8 @@ #![allow(clippy::disallowed_macros)] -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result}; use clap::{Args, Command, CommandFactory, FromArgMatches, Parser, Subcommand}; use duct::cmd; -use std::fs; -use std::path::Path; - -const README_PATH: &str = "tools/ci/README.md"; - -mod ci_docs; /// SpacetimeDB CI tasks /// @@ -83,11 +77,6 @@ enum CiCmd { /// Generates CLI documentation and checks for changes #[command(override_help = "")] CliDocs(ForwardedArgs), - SelfDocs { - /// Only check for changes, do not generate the docs. - #[arg(long)] - check: bool, - }, /// Verify that any non-root global.json files are symlinks to the root global.json. #[command(override_help = "")] GlobalJsonPolicy(ForwardedArgs), @@ -182,7 +171,6 @@ pub(crate) fn command_with_split_help_overrides() -> Result { fn should_load_split_help(args: &[String]) -> bool { args.iter().any(|arg| arg == "--help" || arg == "-h" || arg == "help") - || args.get(1).is_some_and(|arg| arg == "self-docs") } fn parse_cli() -> Result { @@ -203,21 +191,6 @@ fn run_smoketests(args: &[String]) -> Result<()> { run_package("ci-smoketests", &args) } -fn run_self_docs(check: bool) -> Result<()> { - let readme_content = ci_docs::generate_cli_docs()?; - let path = Path::new(README_PATH); - - if check { - let existing = fs::read_to_string(path).unwrap_or_default(); - if existing != readme_content { - bail!("README.md is out of date. Please run `cargo ci self-docs` to update it."); - } - } else { - fs::write(path, readme_content)?; - } - Ok(()) -} - fn run_dlls() -> Result<()> { eprintln!("warning: `cargo ci dlls` is deprecated; use `cargo regen csharp dlls` instead"); cmd!("cargo", "regen", "csharp", "dlls").run()?; @@ -234,7 +207,6 @@ fn run_command(cmd: CiCmd) -> Result<()> { CiCmd::KeynoteBench(args) => run_package("ci-keynote-bench", &args.args), CiCmd::UpdateFlow(args) => run_package("ci-update-flow", &args.args), CiCmd::CliDocs(args) => run_package("ci-cli-docs", &args.args), - CiCmd::SelfDocs { check } => run_self_docs(check), CiCmd::GlobalJsonPolicy(args) => run_package("ci-global-json-policy", &args.args), CiCmd::PublishChecks(args) => run_package("ci-publish-checks", &args.args), CiCmd::TypescriptTest(args) => run_package("ci-typescript-test", &args.args), From 2a76f40c6b512081f5160b5a4db623258c745060 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 15:19:01 -0400 Subject: [PATCH 29/34] Address cargo ci split review --- Cargo.lock | 2 ++ .../ci/commands/global-json-policy/Cargo.toml | 1 + .../ci/commands/global-json-policy/src/main.rs | 18 ++++-------------- tools/ci/commands/smoketests/Cargo.toml | 1 + tools/ci/commands/smoketests/src/main.rs | 18 +++++------------- 5 files changed, 13 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8537415c508..6f31f8fe017 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -969,6 +969,7 @@ name = "ci-global-json-policy" version = "0.1.0" dependencies = [ "anyhow", + "ci-common", "clap 4.5.50", "duct", ] @@ -1008,6 +1009,7 @@ name = "ci-smoketests" version = "0.1.0" dependencies = [ "anyhow", + "ci-common", "clap 4.5.50", "duct", "env_logger 0.10.2", diff --git a/tools/ci/commands/global-json-policy/Cargo.toml b/tools/ci/commands/global-json-policy/Cargo.toml index 72b73691f91..b45b8ff0cab 100644 --- a/tools/ci/commands/global-json-policy/Cargo.toml +++ b/tools/ci/commands/global-json-policy/Cargo.toml @@ -7,3 +7,4 @@ edition.workspace = true anyhow.workspace = true clap.workspace = true duct.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/global-json-policy/src/main.rs b/tools/ci/commands/global-json-policy/src/main.rs index e0a6982196e..d9c0a8a1920 100644 --- a/tools/ci/commands/global-json-policy/src/main.rs +++ b/tools/ci/commands/global-json-policy/src/main.rs @@ -1,6 +1,7 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Result}; +use ci_common::ensure_repo_root; use clap::Parser; use duct::cmd; use std::fs; @@ -10,19 +11,14 @@ use std::path::{Path, PathBuf}; #[derive(Parser)] struct Args {} -fn ensure_repo_root() -> Result<()> { - if !Path::new("Cargo.toml").exists() { - bail!("You must execute this command from the SpacetimeDB repository root (where Cargo.toml is located)"); - } - Ok(()) -} - fn git_tracked_files(pathspec: &str) -> Result> { let output = cmd!("git", "ls-files", pathspec).read()?; Ok(output.lines().map(PathBuf::from).collect()) } -fn check_global_json_policy() -> Result<()> { +fn main() -> Result<()> { + Args::parse(); + ensure_repo_root()?; let root_json = Path::new("global.json"); @@ -58,9 +54,3 @@ fn check_global_json_policy() -> Result<()> { Ok(()) } - -fn main() -> Result<()> { - Args::parse(); - - check_global_json_policy() -} diff --git a/tools/ci/commands/smoketests/Cargo.toml b/tools/ci/commands/smoketests/Cargo.toml index 3fe57943e37..7e289951c14 100644 --- a/tools/ci/commands/smoketests/Cargo.toml +++ b/tools/ci/commands/smoketests/Cargo.toml @@ -10,3 +10,4 @@ clap.workspace = true tempfile.workspace = true spacetimedb-guard.workspace = true env_logger.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/smoketests/src/main.rs b/tools/ci/commands/smoketests/src/main.rs index 00ca60319ae..ba7f2d82209 100644 --- a/tools/ci/commands/smoketests/src/main.rs +++ b/tools/ci/commands/smoketests/src/main.rs @@ -1,5 +1,6 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, ensure, Context, Result}; +use ci_common::ensure_repo_root; use clap::{Parser, Subcommand}; use duct::cmd; use spacetimedb_guard::ensure_binaries_built; @@ -7,13 +8,6 @@ use std::ffi::OsStr; use std::path::Path; use std::process::{Command, Stdio}; use std::{env, fs}; - -fn ensure_repo_root() -> Result<()> { - if !Path::new("Cargo.toml").exists() { - bail!("You must execute this command from the SpacetimeDB repository root (where Cargo.toml is located)"); - } - Ok(()) -} use tempfile::TempDir; #[derive(Parser)] @@ -62,7 +56,10 @@ enum SmoketestCmd { CheckModList, } -fn run(args: SmoketestsArgs) -> Result<()> { +fn main() -> Result<()> { + env_logger::init(); + + let args = SmoketestsArgs::parse(); match args.cmd { Some(SmoketestCmd::Prepare) => { build_cli()?; @@ -361,8 +358,3 @@ fn check_smoketests_mod_rs_complete() -> Result<()> { Ok(()) } - -fn main() -> Result<()> { - env_logger::init(); - run(SmoketestsArgs::parse()) -} From 577f1f3115b8a5cc44c5bca3127319c232157b21 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Sun, 9 Aug 2026 12:20:04 -0700 Subject: [PATCH 30/34] [bot/split-public-cargo-ci]: remove unneeded --- tools/ci/src/main.rs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index b0bc7d9182a..a431b16f1b4 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -40,57 +40,45 @@ enum CiCmd { /// Runs rust tests, codegens csharp sdk and runs csharp tests. /// This does not include Unreal tests. /// This expects to run in a clean git state. - #[command(override_help = "")] Test(ForwardedArgs), /// Lints the codebase /// /// Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there /// are no warnings. - #[command(override_help = "")] Lint(ForwardedArgs), /// Tests Wasm bindings /// /// Runs tests for the codegen crate and builds a test module with the wasm bindings. - #[command(override_help = "")] WasmBindings(ForwardedArgs), /// Deprecated; use `cargo regen csharp dlls`. Dlls, /// Runs smoketests /// /// Executes the smoketests suite with some default exclusions. - #[command(override_help = "")] Smoketests(ForwardedArgs), /// Runs the keynote benchmark as a CI performance regression gate. /// /// Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the /// keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and /// fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. - #[command(override_help = "")] KeynoteBench(ForwardedArgs), /// Tests the update flow /// /// Tests the self-update flow by building the spacetimedb-update binary for the specified /// target, by default the current target, and performing a self-install into a temporary /// directory. - #[command(override_help = "")] UpdateFlow(ForwardedArgs), /// Generates CLI documentation and checks for changes - #[command(override_help = "")] CliDocs(ForwardedArgs), /// Verify that any non-root global.json files are symlinks to the root global.json. - #[command(override_help = "")] GlobalJsonPolicy(ForwardedArgs), /// Checks that publishable crates satisfy publish constraints. - #[command(override_help = "")] PublishChecks(ForwardedArgs), /// Runs TypeScript workspace tests and template build checks. - #[command(override_help = "")] TypescriptTest(ForwardedArgs), /// Verifies that the repository version upgrade tool still works. - #[command(override_help = "")] VersionUpgradeCheck(ForwardedArgs), /// Builds the docs site. - #[command(override_help = "")] Docs(ForwardedArgs), OtherWorkflows { #[command(subcommand)] @@ -101,13 +89,10 @@ enum CiCmd { #[derive(Subcommand)] enum OtherWorkflowsCmd { /// Selects or starts the private workflow for a public Internal Tests run. - #[command(override_help = "")] CoordinateInternalTests(ForwardedArgs), /// Checks that sensitive CODEOWNERS-controlled files have the required approvals. - #[command(override_help = "")] CodeownersCheck(ForwardedArgs), /// Interacts with CLA Assistant. - #[command(override_help = "")] ClaAssistant(ForwardedArgs), } From accb6d43f5a9609a4dc2cb19b2f63663b512b4ae Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Sun, 9 Aug 2026 12:21:59 -0700 Subject: [PATCH 31/34] [bot/split-public-cargo-ci]: remove --- tools/ci/commands/smoketests/Cargo.toml | 1 - tools/ci/commands/smoketests/src/main.rs | 2 -- 2 files changed, 3 deletions(-) diff --git a/tools/ci/commands/smoketests/Cargo.toml b/tools/ci/commands/smoketests/Cargo.toml index 7e289951c14..f161fe64586 100644 --- a/tools/ci/commands/smoketests/Cargo.toml +++ b/tools/ci/commands/smoketests/Cargo.toml @@ -9,5 +9,4 @@ duct.workspace = true clap.workspace = true tempfile.workspace = true spacetimedb-guard.workspace = true -env_logger.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/smoketests/src/main.rs b/tools/ci/commands/smoketests/src/main.rs index ba7f2d82209..9397d7dc3cb 100644 --- a/tools/ci/commands/smoketests/src/main.rs +++ b/tools/ci/commands/smoketests/src/main.rs @@ -57,8 +57,6 @@ enum SmoketestCmd { } fn main() -> Result<()> { - env_logger::init(); - let args = SmoketestsArgs::parse(); match args.cmd { Some(SmoketestCmd::Prepare) => { From 0115f5f5401efce979f3c1ad11cfabb660172783 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 15:35:40 -0400 Subject: [PATCH 32/34] Use split command args for cargo ci help --- Cargo.lock | 15 ++ tools/ci/Cargo.toml | 15 ++ tools/ci/commands/cla-assistant/src/lib.rs | 48 +++++ tools/ci/commands/cla-assistant/src/main.rs | 50 +---- tools/ci/commands/cli-docs/src/lib.rs | 10 + tools/ci/commands/cli-docs/src/main.rs | 12 +- tools/ci/commands/codeowners-check/src/lib.rs | 12 ++ .../ci/commands/codeowners-check/src/main.rs | 16 +- .../coordinate-internal-tests/src/lib.rs | 13 ++ .../coordinate-internal-tests/src/main.rs | 15 +- tools/ci/commands/docs/src/lib.rs | 4 + tools/ci/commands/docs/src/main.rs | 8 +- .../ci/commands/global-json-policy/src/lib.rs | 4 + .../commands/global-json-policy/src/main.rs | 8 +- tools/ci/commands/keynote-bench/src/lib.rs | 4 + tools/ci/commands/keynote-bench/src/main.rs | 8 +- tools/ci/commands/lint/src/lib.rs | 4 + tools/ci/commands/lint/src/main.rs | 8 +- tools/ci/commands/publish-checks/src/lib.rs | 4 + tools/ci/commands/publish-checks/src/main.rs | 8 +- tools/ci/commands/smoketests/src/lib.rs | 40 ++++ tools/ci/commands/smoketests/src/main.rs | 44 +---- tools/ci/commands/test/src/lib.rs | 4 + tools/ci/commands/test/src/main.rs | 8 +- tools/ci/commands/typescript-test/src/lib.rs | 4 + tools/ci/commands/typescript-test/src/main.rs | 8 +- tools/ci/commands/update-flow/src/lib.rs | 16 ++ tools/ci/commands/update-flow/src/main.rs | 18 +- .../commands/version-upgrade-check/src/lib.rs | 4 + .../version-upgrade-check/src/main.rs | 8 +- tools/ci/commands/wasm-bindings/src/lib.rs | 4 + tools/ci/commands/wasm-bindings/src/main.rs | 8 +- tools/ci/src/main.rs | 184 +++++++----------- 33 files changed, 364 insertions(+), 252 deletions(-) create mode 100644 tools/ci/commands/cla-assistant/src/lib.rs create mode 100644 tools/ci/commands/cli-docs/src/lib.rs create mode 100644 tools/ci/commands/codeowners-check/src/lib.rs create mode 100644 tools/ci/commands/coordinate-internal-tests/src/lib.rs create mode 100644 tools/ci/commands/docs/src/lib.rs create mode 100644 tools/ci/commands/global-json-policy/src/lib.rs create mode 100644 tools/ci/commands/keynote-bench/src/lib.rs create mode 100644 tools/ci/commands/lint/src/lib.rs create mode 100644 tools/ci/commands/publish-checks/src/lib.rs create mode 100644 tools/ci/commands/smoketests/src/lib.rs create mode 100644 tools/ci/commands/test/src/lib.rs create mode 100644 tools/ci/commands/typescript-test/src/lib.rs create mode 100644 tools/ci/commands/update-flow/src/lib.rs create mode 100644 tools/ci/commands/version-upgrade-check/src/lib.rs create mode 100644 tools/ci/commands/wasm-bindings/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 6f31f8fe017..3ce16e36591 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -899,6 +899,21 @@ name = "ci" version = "0.1.0" dependencies = [ "anyhow", + "ci-cla-assistant", + "ci-cli-docs", + "ci-codeowners-check", + "ci-coordinate-internal-tests", + "ci-docs-build", + "ci-global-json-policy", + "ci-keynote-bench", + "ci-lint", + "ci-publish-checks", + "ci-smoketests", + "ci-test", + "ci-typescript-test", + "ci-update-flow", + "ci-version-upgrade-check", + "ci-wasm-bindings", "clap 4.5.50", "duct", ] diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml index ba066db1498..21f9807674f 100644 --- a/tools/ci/Cargo.toml +++ b/tools/ci/Cargo.toml @@ -7,3 +7,18 @@ edition.workspace = true anyhow.workspace = true clap.workspace = true duct.workspace = true +ci-cla-assistant = { path = "commands/cla-assistant" } +ci-cli-docs = { path = "commands/cli-docs" } +ci-codeowners-check = { path = "commands/codeowners-check" } +ci-coordinate-internal-tests = { path = "commands/coordinate-internal-tests" } +ci-docs-build = { path = "commands/docs" } +ci-global-json-policy = { path = "commands/global-json-policy" } +ci-keynote-bench = { path = "commands/keynote-bench" } +ci-lint = { path = "commands/lint" } +ci-publish-checks = { path = "commands/publish-checks" } +ci-smoketests = { path = "commands/smoketests" } +ci-test = { path = "commands/test" } +ci-typescript-test = { path = "commands/typescript-test" } +ci-update-flow = { path = "commands/update-flow" } +ci-version-upgrade-check = { path = "commands/version-upgrade-check" } +ci-wasm-bindings = { path = "commands/wasm-bindings" } diff --git a/tools/ci/commands/cla-assistant/src/lib.rs b/tools/ci/commands/cla-assistant/src/lib.rs new file mode 100644 index 00000000000..d20c55961e7 --- /dev/null +++ b/tools/ci/commands/cla-assistant/src/lib.rs @@ -0,0 +1,48 @@ +use clap::{ArgGroup, Args as ClapArgs, Subcommand}; + +#[derive(Subcommand)] +pub enum ClaAssistantCmd { + /// Retries CLA Assistant if `license/cla` is the only remaining PR blocker. + Retry(RetryArgs), + + /// Returns the `license/cla` status for a pull request or commit SHA. + Status(StatusArgs), +} + +#[derive(ClapArgs)] +pub struct RetryArgs { + /// Pull request number to check. + #[arg(long)] + pub pr_number: u64, + + /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. + #[arg(long)] + pub repo: Option, +} + +#[derive(ClapArgs)] +#[command(group( + ArgGroup::new("target") + .required(true) + .multiple(false) + .args(["pr", "sha"]), +))] +pub struct StatusArgs { + /// Pull request number whose head commit should be checked. + #[arg(long)] + pub pr: Option, + + /// Commit SHA to check. + #[arg(long)] + pub sha: Option, + + /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. + #[arg(long)] + pub repo: Option, +} + +#[derive(ClapArgs)] +pub struct Args { + #[command(subcommand)] + pub cmd: ClaAssistantCmd, +} diff --git a/tools/ci/commands/cla-assistant/src/main.rs b/tools/ci/commands/cla-assistant/src/main.rs index 779edbbe842..434eedf667c 100644 --- a/tools/ci/commands/cla-assistant/src/main.rs +++ b/tools/ci/commands/cla-assistant/src/main.rs @@ -4,7 +4,8 @@ use std::collections::BTreeMap; use std::env; use anyhow::{anyhow, bail, Context, Result}; -use clap::{ArgGroup, Args, Parser, Subcommand}; +use ci_cla_assistant::{Args, ClaAssistantCmd, RetryArgs, StatusArgs}; +use clap::Parser; use reqwest::blocking::Client; use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, AUTHORIZATION, USER_AGENT}; use serde::de::DeserializeOwned; @@ -12,56 +13,15 @@ use serde::{Deserialize, Serialize}; const CLA_CONTEXT: &str = "license/cla"; -#[derive(Subcommand)] -enum ClaAssistantCmd { - /// Retries CLA Assistant if `license/cla` is the only remaining PR blocker. - Retry(RetryArgs), - - /// Returns the `license/cla` status for a pull request or commit SHA. - Status(StatusArgs), -} - -#[derive(Args)] -struct RetryArgs { - /// Pull request number to check. - #[arg(long)] - pr_number: u64, - - /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. - #[arg(long)] - repo: Option, -} - -#[derive(Args)] -#[command(group( - ArgGroup::new("target") - .required(true) - .multiple(false) - .args(["pr", "sha"]), -))] -struct StatusArgs { - /// Pull request number whose head commit should be checked. - #[arg(long)] - pr: Option, - - /// Commit SHA to check. - #[arg(long)] - sha: Option, - - /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. - #[arg(long)] - repo: Option, -} - #[derive(Parser)] #[command(about = "Interacts with CLA Assistant.")] struct Cli { - #[command(subcommand)] - cmd: ClaAssistantCmd, + #[command(flatten)] + args: Args, } fn main() -> Result<()> { - match Cli::parse().cmd { + match Cli::parse().args.cmd { ClaAssistantCmd::Retry(args) => retry(args), ClaAssistantCmd::Status(args) => status(args), } diff --git a/tools/ci/commands/cli-docs/src/lib.rs b/tools/ci/commands/cli-docs/src/lib.rs new file mode 100644 index 00000000000..35c2dfed8ca --- /dev/null +++ b/tools/ci/commands/cli-docs/src/lib.rs @@ -0,0 +1,10 @@ +use clap::Args as ClapArgs; + +#[derive(ClapArgs)] +pub struct Args { + #[arg( + long, + long_help = "specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located)" + )] + pub spacetime_path: Option, +} diff --git a/tools/ci/commands/cli-docs/src/main.rs b/tools/ci/commands/cli-docs/src/main.rs index 67870c6b98d..53f5b1b5ff7 100644 --- a/tools/ci/commands/cli-docs/src/main.rs +++ b/tools/ci/commands/cli-docs/src/main.rs @@ -1,5 +1,6 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; +use ci_cli_docs::Args; use ci_common::pnpm; use clap::Parser; use duct::cmd; @@ -7,16 +8,13 @@ use std::env; #[derive(Parser)] #[command(about = "Generates CLI documentation and checks for changes")] -struct Args { - #[arg( - long, - long_help = "specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located)" - )] - spacetime_path: Option, +struct Cli { + #[command(flatten)] + args: Args, } fn main() -> Result<()> { - let Args { spacetime_path } = Args::parse(); + let Args { spacetime_path } = Cli::parse().args; if let Some(path) = spacetime_path { env::set_current_dir(path).ok(); } diff --git a/tools/ci/commands/codeowners-check/src/lib.rs b/tools/ci/commands/codeowners-check/src/lib.rs new file mode 100644 index 00000000000..ed27d6fda38 --- /dev/null +++ b/tools/ci/commands/codeowners-check/src/lib.rs @@ -0,0 +1,12 @@ +use clap::Args as ClapArgs; + +#[derive(ClapArgs)] +pub struct Args { + /// Git ref to compare against, usually origin/. + #[arg(long)] + pub base_ref: String, + + /// Pull request number to inspect for approval state. + #[arg(long)] + pub pr_number: u64, +} diff --git a/tools/ci/commands/codeowners-check/src/main.rs b/tools/ci/commands/codeowners-check/src/main.rs index 5d32e7ead06..6476648b483 100644 --- a/tools/ci/commands/codeowners-check/src/main.rs +++ b/tools/ci/commands/codeowners-check/src/main.rs @@ -1,4 +1,5 @@ use anyhow::{anyhow, bail, Context, Result}; +use ci_codeowners_check::Args; use ci_common::ensure_repo_root; use clap::Parser; use duct::cmd; @@ -8,16 +9,11 @@ use std::path::{Path, PathBuf}; const REPO: &str = "clockworklabs/SpacetimeDB"; -#[derive(Parser)] #[command(about = "Checks that sensitive CODEOWNERS-controlled files have the required approvals.")] -struct Args { - /// Git ref to compare against, usually origin/. - #[arg(long)] - base_ref: String, - - /// Pull request number to inspect for approval state. - #[arg(long)] - pr_number: u64, +#[derive(Parser)] +struct Cli { + #[command(flatten)] + args: Args, } fn run(base_ref: &str, pr_number: u64) -> Result<()> { @@ -35,7 +31,7 @@ fn run(base_ref: &str, pr_number: u64) -> Result<()> { } fn main() -> Result<()> { - let args = Args::parse(); + let args = Cli::parse().args; run(&args.base_ref, args.pr_number) } diff --git a/tools/ci/commands/coordinate-internal-tests/src/lib.rs b/tools/ci/commands/coordinate-internal-tests/src/lib.rs new file mode 100644 index 00000000000..a0e066b8cf2 --- /dev/null +++ b/tools/ci/commands/coordinate-internal-tests/src/lib.rs @@ -0,0 +1,13 @@ +use clap::Args as ClapArgs; + +/// Selects or starts the private workflow for a public Internal Tests run. +#[derive(ClapArgs)] +pub struct Args { + /// Immutable public commit to test. + #[arg(long)] + pub public_sha: String, + + /// Public pull request number, when coordinating a pull request run. + #[arg(long)] + pub public_pr_number: Option, +} diff --git a/tools/ci/commands/coordinate-internal-tests/src/main.rs b/tools/ci/commands/coordinate-internal-tests/src/main.rs index 850b8152e4d..0fde3826a76 100644 --- a/tools/ci/commands/coordinate-internal-tests/src/main.rs +++ b/tools/ci/commands/coordinate-internal-tests/src/main.rs @@ -1,6 +1,7 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, ensure, Context, Result}; +use ci_coordinate_internal_tests::Args; use clap::Parser; use duct::{cmd, Expression}; use serde::de::DeserializeOwned; @@ -16,17 +17,11 @@ const PRIVATE_REPO: &str = "clockworklabs/SpacetimeDBPrivate"; const PRIVATE_WORKFLOW: &str = "ci.yml"; const PRIVATE_DEFAULT_BRANCH: &str = "master"; -/// Selects or starts the private workflow for a public Internal Tests run. #[derive(Parser)] #[command(about = "Selects or starts the private workflow for a public Internal Tests run.")] -struct Args { - /// Immutable public commit to test. - #[arg(long)] - public_sha: String, - - /// Public pull request number, when coordinating a pull request run. - #[arg(long)] - public_pr_number: Option, +struct Cli { + #[command(flatten)] + args: Args, } #[derive(Debug)] @@ -455,7 +450,7 @@ fn write_github_output(name: &str, value: impl std::fmt::Display) -> Result<()> /// Coordinates the public Internal Tests run without checking out or executing private code. fn main() -> Result<()> { - let args = Args::parse(); + let args = Cli::parse().args; let private_source = resolve_private_source(args.public_pr_number)?; let coordinated = match private_source { diff --git a/tools/ci/commands/docs/src/lib.rs b/tools/ci/commands/docs/src/lib.rs new file mode 100644 index 00000000000..d129311db6c --- /dev/null +++ b/tools/ci/commands/docs/src/lib.rs @@ -0,0 +1,4 @@ +use clap::Args as ClapArgs; + +#[derive(ClapArgs)] +pub struct Args {} diff --git a/tools/ci/commands/docs/src/main.rs b/tools/ci/commands/docs/src/main.rs index ef3d4f3cfc1..cd3fb33c6bb 100644 --- a/tools/ci/commands/docs/src/main.rs +++ b/tools/ci/commands/docs/src/main.rs @@ -1,14 +1,18 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; use ci_common::pnpm; +use ci_docs_build::Args; use clap::Parser; /// Builds the docs site. #[derive(Parser)] -struct Args {} +struct Cli { + #[command(flatten)] + args: Args, +} fn main() -> Result<()> { - Args::parse(); + Cli::parse(); pnpm(["install"]).dir("docs").run()?; pnpm(["build"]).dir("docs").run()?; diff --git a/tools/ci/commands/global-json-policy/src/lib.rs b/tools/ci/commands/global-json-policy/src/lib.rs new file mode 100644 index 00000000000..d129311db6c --- /dev/null +++ b/tools/ci/commands/global-json-policy/src/lib.rs @@ -0,0 +1,4 @@ +use clap::Args as ClapArgs; + +#[derive(ClapArgs)] +pub struct Args {} diff --git a/tools/ci/commands/global-json-policy/src/main.rs b/tools/ci/commands/global-json-policy/src/main.rs index d9c0a8a1920..32aedacc1f0 100644 --- a/tools/ci/commands/global-json-policy/src/main.rs +++ b/tools/ci/commands/global-json-policy/src/main.rs @@ -2,6 +2,7 @@ use anyhow::{bail, Result}; use ci_common::ensure_repo_root; +use ci_global_json_policy::Args; use clap::Parser; use duct::cmd; use std::fs; @@ -9,7 +10,10 @@ use std::path::{Path, PathBuf}; /// Verify that any non-root global.json files are symlinks to the root global.json. #[derive(Parser)] -struct Args {} +struct Cli { + #[command(flatten)] + args: Args, +} fn git_tracked_files(pathspec: &str) -> Result> { let output = cmd!("git", "ls-files", pathspec).read()?; @@ -17,7 +21,7 @@ fn git_tracked_files(pathspec: &str) -> Result> { } fn main() -> Result<()> { - Args::parse(); + Cli::parse(); ensure_repo_root()?; diff --git a/tools/ci/commands/keynote-bench/src/lib.rs b/tools/ci/commands/keynote-bench/src/lib.rs new file mode 100644 index 00000000000..d129311db6c --- /dev/null +++ b/tools/ci/commands/keynote-bench/src/lib.rs @@ -0,0 +1,4 @@ +use clap::Args as ClapArgs; + +#[derive(ClapArgs)] +pub struct Args {} diff --git a/tools/ci/commands/keynote-bench/src/main.rs b/tools/ci/commands/keynote-bench/src/main.rs index 6ed92f6cce8..ede810009e0 100644 --- a/tools/ci/commands/keynote-bench/src/main.rs +++ b/tools/ci/commands/keynote-bench/src/main.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use ci_keynote_bench::Args; use clap::Parser; use keynote_bench_harness::KeynoteBenchConfig; use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; @@ -9,10 +10,13 @@ use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; /// keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and /// fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. #[derive(Parser)] -struct Args {} +struct Cli { + #[command(flatten)] + args: Args, +} fn main() -> Result<()> { - Args::parse(); + Cli::parse(); let cli_path = ensure_binaries_built(); let server = SpacetimeDbGuard::spawn_in_temp_data_dir(); diff --git a/tools/ci/commands/lint/src/lib.rs b/tools/ci/commands/lint/src/lib.rs new file mode 100644 index 00000000000..d129311db6c --- /dev/null +++ b/tools/ci/commands/lint/src/lib.rs @@ -0,0 +1,4 @@ +use clap::Args as ClapArgs; + +#[derive(ClapArgs)] +pub struct Args {} diff --git a/tools/ci/commands/lint/src/main.rs b/tools/ci/commands/lint/src/main.rs index 66450d3c95c..52a42f86d93 100644 --- a/tools/ci/commands/lint/src/main.rs +++ b/tools/ci/commands/lint/src/main.rs @@ -2,6 +2,7 @@ use anyhow::{bail, Context, Result}; use ci_common::{ensure_repo_root, pnpm}; +use ci_lint::Args; use clap::Parser; use duct::cmd; use serde_json::Value; @@ -16,7 +17,10 @@ use std::path::PathBuf; /// Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there /// are no warnings. #[derive(Parser)] -struct Args {} +struct Cli { + #[command(flatten)] + args: Args, +} fn package_json_pnpm_version(package_manager: &str) -> Option<&str> { package_manager.strip_prefix("pnpm@") @@ -288,7 +292,7 @@ fn tracked_rs_files_under(path: &str) -> Result> { } fn main() -> Result<()> { - Args::parse(); + Cli::parse(); ensure_repo_root()?; check_pnpm_release_age_policy()?; diff --git a/tools/ci/commands/publish-checks/src/lib.rs b/tools/ci/commands/publish-checks/src/lib.rs new file mode 100644 index 00000000000..d129311db6c --- /dev/null +++ b/tools/ci/commands/publish-checks/src/lib.rs @@ -0,0 +1,4 @@ +use clap::Args as ClapArgs; + +#[derive(ClapArgs)] +pub struct Args {} diff --git a/tools/ci/commands/publish-checks/src/main.rs b/tools/ci/commands/publish-checks/src/main.rs index e10d89adc0f..9a966565b36 100644 --- a/tools/ci/commands/publish-checks/src/main.rs +++ b/tools/ci/commands/publish-checks/src/main.rs @@ -1,14 +1,18 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Result}; +use ci_publish_checks::Args; use clap::Parser; use duct::cmd; /// Checks that publishable crates satisfy publish constraints. #[derive(Parser)] -struct Args {} +struct Cli { + #[command(flatten)] + args: Args, +} fn main() -> Result<()> { - Args::parse(); + Cli::parse(); cmd!("bash", "-lc", "test -d venv || python3 -m venv venv").run()?; cmd!("venv/bin/pip3", "install", "argparse", "toml").run()?; diff --git a/tools/ci/commands/smoketests/src/lib.rs b/tools/ci/commands/smoketests/src/lib.rs new file mode 100644 index 00000000000..80a4c02f35e --- /dev/null +++ b/tools/ci/commands/smoketests/src/lib.rs @@ -0,0 +1,40 @@ +use clap::{Args as ClapArgs, Subcommand}; + +#[derive(ClapArgs)] +pub struct SmoketestsArgs { + #[command(subcommand)] + pub cmd: Option, + + /// Run tests against a remote server instead of spawning local servers. + /// + /// When specified, tests will connect to the given URL instead of starting + /// local server instances. Tests that require local server control (like + /// restart tests) will be skipped. + #[arg(long)] + pub server: Option, + + /// Use a SpacetimeAuth-issued login for remote-server tests. + /// + /// This is required for servers that reject direct server-issued logins for privileged operations. + /// + /// Optionally accepts an auth host to pass through to `spacetime login`, + /// for example `--auth-host=https://spacetimedb.com`. + #[arg(long, num_args = 0..=1, require_equals = true, default_missing_value = "")] + pub auth_host: Option, + + #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] + pub dotnet: bool, + + /// Additional arguments to pass to the test runner + #[arg(trailing_var_arg = true)] + pub args: Vec, +} + +#[derive(Subcommand)] +pub enum SmoketestCmd { + /// Only build binaries without running tests + /// + /// Use this before running `cargo test --all` to ensure binaries are built. + Prepare, + CheckModList, +} diff --git a/tools/ci/commands/smoketests/src/main.rs b/tools/ci/commands/smoketests/src/main.rs index 9397d7dc3cb..feb7d3e78d0 100644 --- a/tools/ci/commands/smoketests/src/main.rs +++ b/tools/ci/commands/smoketests/src/main.rs @@ -1,7 +1,8 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, ensure, Context, Result}; use ci_common::ensure_repo_root; -use clap::{Parser, Subcommand}; +use ci_smoketests::{SmoketestCmd, SmoketestsArgs}; +use clap::Parser; use duct::cmd; use spacetimedb_guard::ensure_binaries_built; use std::ffi::OsStr; @@ -18,46 +19,13 @@ use tempfile::TempDir; /// This command builds the binaries needed by the smoketests, then runs them. This prevents /// race conditions when running tests in parallel with nextest, where multiple test processes /// might try to build the same binaries simultaneously. -struct SmoketestsArgs { - #[command(subcommand)] - cmd: Option, - - /// Run tests against a remote server instead of spawning local servers. - /// - /// When specified, tests will connect to the given URL instead of starting - /// local server instances. Tests that require local server control (like - /// restart tests) will be skipped. - #[arg(long)] - server: Option, - - /// Use a SpacetimeAuth-issued login for remote-server tests. - /// - /// This is required for servers that reject direct server-issued logins for privileged operations. - /// - /// Optionally accepts an auth host to pass through to `spacetime login`, - /// for example `--auth-host=https://spacetimedb.com`. - #[arg(long, num_args = 0..=1, require_equals = true, default_missing_value = "")] - auth_host: Option, - - #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] - dotnet: bool, - - /// Additional arguments to pass to the test runner - #[arg(trailing_var_arg = true)] - args: Vec, -} - -#[derive(Subcommand)] -enum SmoketestCmd { - /// Only build binaries without running tests - /// - /// Use this before running `cargo test --all` to ensure binaries are built. - Prepare, - CheckModList, +struct Cli { + #[command(flatten)] + args: SmoketestsArgs, } fn main() -> Result<()> { - let args = SmoketestsArgs::parse(); + let args = Cli::parse().args; match args.cmd { Some(SmoketestCmd::Prepare) => { build_cli()?; diff --git a/tools/ci/commands/test/src/lib.rs b/tools/ci/commands/test/src/lib.rs new file mode 100644 index 00000000000..d129311db6c --- /dev/null +++ b/tools/ci/commands/test/src/lib.rs @@ -0,0 +1,4 @@ +use clap::Args as ClapArgs; + +#[derive(ClapArgs)] +pub struct Args {} diff --git a/tools/ci/commands/test/src/main.rs b/tools/ci/commands/test/src/main.rs index 3124b4b5be1..6d3fb13dc91 100644 --- a/tools/ci/commands/test/src/main.rs +++ b/tools/ci/commands/test/src/main.rs @@ -1,6 +1,7 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; use ci_common::pnpm; +use ci_test::Args; use clap::Parser; use duct::cmd; @@ -10,10 +11,13 @@ use duct::cmd; /// This does not include Unreal tests. /// This expects to run in a clean git state. #[derive(Parser)] -struct Args {} +struct Cli { + #[command(flatten)] + args: Args, +} fn main() -> Result<()> { - Args::parse(); + Cli::parse(); pnpm(["build"]).dir("crates/bindings-typescript").run()?; diff --git a/tools/ci/commands/typescript-test/src/lib.rs b/tools/ci/commands/typescript-test/src/lib.rs new file mode 100644 index 00000000000..d129311db6c --- /dev/null +++ b/tools/ci/commands/typescript-test/src/lib.rs @@ -0,0 +1,4 @@ +use clap::Args as ClapArgs; + +#[derive(ClapArgs)] +pub struct Args {} diff --git a/tools/ci/commands/typescript-test/src/main.rs b/tools/ci/commands/typescript-test/src/main.rs index e03e97aeb1e..77e01fc9e63 100644 --- a/tools/ci/commands/typescript-test/src/main.rs +++ b/tools/ci/commands/typescript-test/src/main.rs @@ -1,15 +1,19 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Result}; use ci_common::pnpm; +use ci_typescript_test::Args; use clap::Parser; use duct::cmd; /// Runs TypeScript workspace tests and template build checks. #[derive(Parser)] -struct Args {} +struct Cli { + #[command(flatten)] + args: Args, +} fn main() -> Result<()> { - Args::parse(); + Cli::parse(); pnpm(["build"]).dir("crates/bindings-typescript").run()?; pnpm(["test"]).dir("crates/bindings-typescript").run()?; diff --git a/tools/ci/commands/update-flow/src/lib.rs b/tools/ci/commands/update-flow/src/lib.rs new file mode 100644 index 00000000000..5543d9cc3d8 --- /dev/null +++ b/tools/ci/commands/update-flow/src/lib.rs @@ -0,0 +1,16 @@ +use clap::Args as ClapArgs; + +#[derive(ClapArgs)] +pub struct Args { + #[arg( + long, + long_help = "Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms." + )] + pub target: Option, + #[arg( + long, + default_value = "false", + long_help = "Whether to enable github token authentication feature when building the update binary. By default this is disabled." + )] + pub github_token_auth: bool, +} diff --git a/tools/ci/commands/update-flow/src/main.rs b/tools/ci/commands/update-flow/src/main.rs index 31875593a8e..ba70b6e3275 100644 --- a/tools/ci/commands/update-flow/src/main.rs +++ b/tools/ci/commands/update-flow/src/main.rs @@ -1,5 +1,6 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; +use ci_update_flow::Args; use clap::Parser; use duct::cmd; @@ -8,25 +9,16 @@ use duct::cmd; about = "Tests the update flow", long_about = "Tests the update flow\n\nTests the self-update flow by building the spacetimedb-update binary for the specified target, by default the current target, and performing a self-install into a temporary directory." )] -struct Args { - #[arg( - long, - long_help = "Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms." - )] - target: Option, - #[arg( - long, - default_value = "false", - long_help = "Whether to enable github token authentication feature when building the update binary. By default this is disabled." - )] - github_token_auth: bool, +struct Cli { + #[command(flatten)] + args: Args, } fn main() -> Result<()> { let Args { target, github_token_auth, - } = Args::parse(); + } = Cli::parse().args; let mut common_args = vec![]; if let Some(target) = target.as_ref() { common_args.push("--target"); diff --git a/tools/ci/commands/version-upgrade-check/src/lib.rs b/tools/ci/commands/version-upgrade-check/src/lib.rs new file mode 100644 index 00000000000..d129311db6c --- /dev/null +++ b/tools/ci/commands/version-upgrade-check/src/lib.rs @@ -0,0 +1,4 @@ +use clap::Args as ClapArgs; + +#[derive(ClapArgs)] +pub struct Args {} diff --git a/tools/ci/commands/version-upgrade-check/src/main.rs b/tools/ci/commands/version-upgrade-check/src/main.rs index 97febd5ce60..05b533db0ac 100644 --- a/tools/ci/commands/version-upgrade-check/src/main.rs +++ b/tools/ci/commands/version-upgrade-check/src/main.rs @@ -1,11 +1,15 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; +use ci_version_upgrade_check::Args; use clap::Parser; use duct::cmd; /// Verifies that the repository version upgrade tool still works. #[derive(Parser)] -struct Args {} +struct Cli { + #[command(flatten)] + args: Args, +} fn run_version_upgrade_check() -> Result<()> { cmd!( @@ -23,7 +27,7 @@ fn run_version_upgrade_check() -> Result<()> { } fn main() -> Result<()> { - Args::parse(); + Cli::parse(); run_version_upgrade_check() } diff --git a/tools/ci/commands/wasm-bindings/src/lib.rs b/tools/ci/commands/wasm-bindings/src/lib.rs new file mode 100644 index 00000000000..d129311db6c --- /dev/null +++ b/tools/ci/commands/wasm-bindings/src/lib.rs @@ -0,0 +1,4 @@ +use clap::Args as ClapArgs; + +#[derive(ClapArgs)] +pub struct Args {} diff --git a/tools/ci/commands/wasm-bindings/src/main.rs b/tools/ci/commands/wasm-bindings/src/main.rs index a83bef2b1a7..ed4aae9ef95 100644 --- a/tools/ci/commands/wasm-bindings/src/main.rs +++ b/tools/ci/commands/wasm-bindings/src/main.rs @@ -1,6 +1,7 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; use ci_common::pnpm; +use ci_wasm_bindings::Args; use clap::Parser; use duct::cmd; @@ -8,10 +9,13 @@ use duct::cmd; /// /// Runs tests for the codegen crate and builds a test module with the wasm bindings. #[derive(Parser)] -struct Args {} +struct Cli { + #[command(flatten)] + args: Args, +} fn main() -> Result<()> { - Args::parse(); + Cli::parse(); pnpm([ "install", diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index a431b16f1b4..ac31a5b3c47 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,7 +1,7 @@ #![allow(clippy::disallowed_macros)] -use anyhow::{Context, Result}; -use clap::{Args, Command, CommandFactory, FromArgMatches, Parser, Subcommand}; +use anyhow::Result; +use clap::{CommandFactory, Parser, Subcommand}; use duct::cmd; /// SpacetimeDB CI tasks @@ -26,13 +26,6 @@ struct Cli { skip: Vec, } -#[derive(Args)] -struct ForwardedArgs { - /// Arguments forwarded to the split CI command package. - #[arg(trailing_var_arg = true, allow_hyphen_values = true)] - args: Vec, -} - #[derive(Subcommand)] enum CiCmd { /// Runs tests @@ -40,46 +33,46 @@ enum CiCmd { /// Runs rust tests, codegens csharp sdk and runs csharp tests. /// This does not include Unreal tests. /// This expects to run in a clean git state. - Test(ForwardedArgs), + Test(ci_test::Args), /// Lints the codebase /// /// Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there /// are no warnings. - Lint(ForwardedArgs), + Lint(ci_lint::Args), /// Tests Wasm bindings /// /// Runs tests for the codegen crate and builds a test module with the wasm bindings. - WasmBindings(ForwardedArgs), + WasmBindings(ci_wasm_bindings::Args), /// Deprecated; use `cargo regen csharp dlls`. Dlls, /// Runs smoketests /// /// Executes the smoketests suite with some default exclusions. - Smoketests(ForwardedArgs), + Smoketests(ci_smoketests::SmoketestsArgs), /// Runs the keynote benchmark as a CI performance regression gate. /// /// Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the /// keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and /// fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. - KeynoteBench(ForwardedArgs), + KeynoteBench(ci_keynote_bench::Args), /// Tests the update flow /// /// Tests the self-update flow by building the spacetimedb-update binary for the specified /// target, by default the current target, and performing a self-install into a temporary /// directory. - UpdateFlow(ForwardedArgs), + UpdateFlow(ci_update_flow::Args), /// Generates CLI documentation and checks for changes - CliDocs(ForwardedArgs), + CliDocs(ci_cli_docs::Args), /// Verify that any non-root global.json files are symlinks to the root global.json. - GlobalJsonPolicy(ForwardedArgs), + GlobalJsonPolicy(ci_global_json_policy::Args), /// Checks that publishable crates satisfy publish constraints. - PublishChecks(ForwardedArgs), + PublishChecks(ci_publish_checks::Args), /// Runs TypeScript workspace tests and template build checks. - TypescriptTest(ForwardedArgs), + TypescriptTest(ci_typescript_test::Args), /// Verifies that the repository version upgrade tool still works. - VersionUpgradeCheck(ForwardedArgs), + VersionUpgradeCheck(ci_version_upgrade_check::Args), /// Builds the docs site. - Docs(ForwardedArgs), + Docs(ci_docs_build::Args), OtherWorkflows { #[command(subcommand)] cmd: OtherWorkflowsCmd, @@ -89,11 +82,11 @@ enum CiCmd { #[derive(Subcommand)] enum OtherWorkflowsCmd { /// Selects or starts the private workflow for a public Internal Tests run. - CoordinateInternalTests(ForwardedArgs), + CoordinateInternalTests(ci_coordinate_internal_tests::Args), /// Checks that sensitive CODEOWNERS-controlled files have the required approvals. - CodeownersCheck(ForwardedArgs), + CodeownersCheck(ci_codeowners_check::Args), /// Interacts with CLA Assistant. - ClaAssistant(ForwardedArgs), + ClaAssistant(ci_cla_assistant::Args), } fn run_package(package: &str, args: &[String]) -> Result<()> { @@ -103,77 +96,30 @@ fn run_package(package: &str, args: &[String]) -> Result<()> { Ok(()) } -fn split_command_help(package: &str) -> Result { - let help = cmd!("cargo", "run", "--quiet", "--package", package, "--", "--help") - .env("COLUMNS", "1000") - .read() - .with_context(|| format!("failed to render help for `{package}`"))?; - Ok(help.lines().map(str::trim_end).collect::>().join("\n")) -} - -fn set_split_help_override(command: &mut Command, path: &[&str], help: String) { - let (name, rest) = path.split_first().expect("split command path is never empty"); - let subcommand = command - .find_subcommand_mut(name) - .unwrap_or_else(|| panic!("missing split subcommand `{}`", path.join(" "))); - - if rest.is_empty() { - *subcommand = subcommand.clone().override_help(help); - } else { - set_split_help_override(subcommand, rest, help); - } -} - -pub(crate) fn command_with_split_help_overrides() -> Result { - let mut command = Cli::command(); - - for (path, package) in [ - (&["test"][..], "ci-test"), - (&["lint"], "ci-lint"), - (&["wasm-bindings"], "ci-wasm-bindings"), - (&["smoketests"], "ci-smoketests"), - (&["keynote-bench"], "ci-keynote-bench"), - (&["update-flow"], "ci-update-flow"), - (&["cli-docs"], "ci-cli-docs"), - (&["global-json-policy"], "ci-global-json-policy"), - (&["publish-checks"], "ci-publish-checks"), - (&["typescript-test"], "ci-typescript-test"), - (&["version-upgrade-check"], "ci-version-upgrade-check"), - (&["docs"], "ci-docs-build"), - ( - &["other-workflows", "coordinate-internal-tests"], - "ci-coordinate-internal-tests", - ), - (&["other-workflows", "codeowners-check"], "ci-codeowners-check"), - (&["other-workflows", "cla-assistant"], "ci-cla-assistant"), - ] { - let help = split_command_help(package)?; - set_split_help_override(&mut command, path, help); - } - - Ok(command) -} - -fn should_load_split_help(args: &[String]) -> bool { - args.iter().any(|arg| arg == "--help" || arg == "-h" || arg == "help") -} - -fn parse_cli() -> Result { - let args = std::env::args().collect::>(); - if should_load_split_help(&args) { - let matches = command_with_split_help_overrides()?.get_matches_from(args); - Ok(Cli::from_arg_matches(&matches)?) - } else { - Ok(Cli::parse_from(args)) +fn forwarded_args(raw_args: &[String], path: &[&str]) -> Vec { + let args = &raw_args[1..]; + let mut start = None; + let mut idx = 0; + while idx < args.len() { + match args[idx].as_str() { + "--skip" => idx += 2, + arg if arg.starts_with("--skip=") => idx += 1, + _ => { + if args[idx] == path[0] + && args + .get(idx..idx + path.len()) + .is_some_and(|window| window.iter().map(String::as_str).eq(path.iter().copied())) + { + start = Some(idx); + break; + } + idx += 1; + } + } } -} -fn run_smoketests(args: &[String]) -> Result<()> { - let mut args = args.to_vec(); - if args.first().is_some_and(|arg| arg.starts_with("--test-")) { - args.insert(0, "--".to_string()); - } - run_package("ci-smoketests", &args) + let start = start.unwrap_or_else(|| panic!("missing command path `{}` in raw argv", path.join(" "))); + args[start + path.len()..].to_vec() } fn run_dlls() -> Result<()> { @@ -182,25 +128,40 @@ fn run_dlls() -> Result<()> { Ok(()) } -fn run_command(cmd: CiCmd) -> Result<()> { +fn run_command(cmd: CiCmd, raw_args: &[String]) -> Result<()> { match cmd { - CiCmd::Test(args) => run_package("ci-test", &args.args), - CiCmd::Lint(args) => run_package("ci-lint", &args.args), - CiCmd::WasmBindings(args) => run_package("ci-wasm-bindings", &args.args), + CiCmd::Test(_) => run_package("ci-test", &forwarded_args(raw_args, &["test"])), + CiCmd::Lint(_) => run_package("ci-lint", &forwarded_args(raw_args, &["lint"])), + CiCmd::WasmBindings(_) => run_package("ci-wasm-bindings", &forwarded_args(raw_args, &["wasm-bindings"])), CiCmd::Dlls => run_dlls(), - CiCmd::Smoketests(args) => run_smoketests(&args.args), - CiCmd::KeynoteBench(args) => run_package("ci-keynote-bench", &args.args), - CiCmd::UpdateFlow(args) => run_package("ci-update-flow", &args.args), - CiCmd::CliDocs(args) => run_package("ci-cli-docs", &args.args), - CiCmd::GlobalJsonPolicy(args) => run_package("ci-global-json-policy", &args.args), - CiCmd::PublishChecks(args) => run_package("ci-publish-checks", &args.args), - CiCmd::TypescriptTest(args) => run_package("ci-typescript-test", &args.args), - CiCmd::VersionUpgradeCheck(args) => run_package("ci-version-upgrade-check", &args.args), - CiCmd::Docs(args) => run_package("ci-docs-build", &args.args), + CiCmd::Smoketests(_) => run_package("ci-smoketests", &forwarded_args(raw_args, &["smoketests"])), + CiCmd::KeynoteBench(_) => run_package("ci-keynote-bench", &forwarded_args(raw_args, &["keynote-bench"])), + CiCmd::UpdateFlow(_) => run_package("ci-update-flow", &forwarded_args(raw_args, &["update-flow"])), + CiCmd::CliDocs(_) => run_package("ci-cli-docs", &forwarded_args(raw_args, &["cli-docs"])), + CiCmd::GlobalJsonPolicy(_) => run_package( + "ci-global-json-policy", + &forwarded_args(raw_args, &["global-json-policy"]), + ), + CiCmd::PublishChecks(_) => run_package("ci-publish-checks", &forwarded_args(raw_args, &["publish-checks"])), + CiCmd::TypescriptTest(_) => run_package("ci-typescript-test", &forwarded_args(raw_args, &["typescript-test"])), + CiCmd::VersionUpgradeCheck(_) => run_package( + "ci-version-upgrade-check", + &forwarded_args(raw_args, &["version-upgrade-check"]), + ), + CiCmd::Docs(_) => run_package("ci-docs-build", &forwarded_args(raw_args, &["docs"])), CiCmd::OtherWorkflows { cmd } => match cmd { - OtherWorkflowsCmd::CoordinateInternalTests(args) => run_package("ci-coordinate-internal-tests", &args.args), - OtherWorkflowsCmd::CodeownersCheck(args) => run_package("ci-codeowners-check", &args.args), - OtherWorkflowsCmd::ClaAssistant(args) => run_package("ci-cla-assistant", &args.args), + OtherWorkflowsCmd::CoordinateInternalTests(_) => run_package( + "ci-coordinate-internal-tests", + &forwarded_args(raw_args, &["other-workflows", "coordinate-internal-tests"]), + ), + OtherWorkflowsCmd::CodeownersCheck(_) => run_package( + "ci-codeowners-check", + &forwarded_args(raw_args, &["other-workflows", "codeowners-check"]), + ), + OtherWorkflowsCmd::ClaAssistant(_) => run_package( + "ci-cla-assistant", + &forwarded_args(raw_args, &["other-workflows", "cla-assistant"]), + ), }, } } @@ -217,10 +178,11 @@ fn run_all_clap_subcommands(skip: &[String]) -> Result<()> { } fn main() -> Result<()> { - let cli = parse_cli()?; + let raw_args = std::env::args().collect::>(); + let cli = Cli::parse_from(&raw_args); match cli.cmd { - Some(cmd) => run_command(cmd), + Some(cmd) => run_command(cmd, &raw_args), None => run_all_clap_subcommands(&cli.skip), } } From 14fa25ed7393428c386349abcd8f7eee23009c3f Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 15:37:22 -0400 Subject: [PATCH 33/34] Update cargo ci lockfile --- Cargo.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 3ce16e36591..5ea11be474a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1027,7 +1027,6 @@ dependencies = [ "ci-common", "clap 4.5.50", "duct", - "env_logger 0.10.2", "spacetimedb-guard", "tempfile", ] From cdae211f2b8dd18ef57229f9c5e26de60b9e3385 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sun, 9 Aug 2026 15:43:50 -0400 Subject: [PATCH 34/34] Move cargo ci args to slim package --- Cargo.lock | 38 +-- Cargo.toml | 1 + tools/ci/Cargo.toml | 16 +- tools/ci/args/Cargo.toml | 7 + tools/ci/args/src/lib.rs | 220 ++++++++++++++++++ tools/ci/commands/cla-assistant/Cargo.toml | 1 + tools/ci/commands/cla-assistant/src/lib.rs | 48 ---- tools/ci/commands/cla-assistant/src/main.rs | 2 +- tools/ci/commands/cli-docs/Cargo.toml | 1 + tools/ci/commands/cli-docs/src/lib.rs | 10 - tools/ci/commands/cli-docs/src/main.rs | 2 +- tools/ci/commands/codeowners-check/Cargo.toml | 1 + tools/ci/commands/codeowners-check/src/lib.rs | 12 - .../ci/commands/codeowners-check/src/main.rs | 4 +- .../coordinate-internal-tests/Cargo.toml | 1 + .../coordinate-internal-tests/src/lib.rs | 13 -- .../coordinate-internal-tests/src/main.rs | 2 +- tools/ci/commands/docs/Cargo.toml | 1 + tools/ci/commands/docs/src/lib.rs | 4 - tools/ci/commands/docs/src/main.rs | 2 +- .../ci/commands/global-json-policy/Cargo.toml | 1 + .../ci/commands/global-json-policy/src/lib.rs | 4 - .../commands/global-json-policy/src/main.rs | 2 +- tools/ci/commands/keynote-bench/Cargo.toml | 1 + tools/ci/commands/keynote-bench/src/lib.rs | 4 - tools/ci/commands/keynote-bench/src/main.rs | 2 +- tools/ci/commands/lint/Cargo.toml | 1 + tools/ci/commands/lint/src/lib.rs | 4 - tools/ci/commands/lint/src/main.rs | 2 +- tools/ci/commands/publish-checks/Cargo.toml | 1 + tools/ci/commands/publish-checks/src/lib.rs | 4 - tools/ci/commands/publish-checks/src/main.rs | 2 +- tools/ci/commands/smoketests/Cargo.toml | 1 + tools/ci/commands/smoketests/src/lib.rs | 40 ---- tools/ci/commands/smoketests/src/main.rs | 2 +- tools/ci/commands/test/Cargo.toml | 1 + tools/ci/commands/test/src/lib.rs | 4 - tools/ci/commands/test/src/main.rs | 2 +- tools/ci/commands/typescript-test/Cargo.toml | 1 + tools/ci/commands/typescript-test/src/lib.rs | 4 - tools/ci/commands/typescript-test/src/main.rs | 2 +- tools/ci/commands/update-flow/Cargo.toml | 1 + tools/ci/commands/update-flow/src/lib.rs | 16 -- tools/ci/commands/update-flow/src/main.rs | 2 +- .../commands/version-upgrade-check/Cargo.toml | 1 + .../commands/version-upgrade-check/src/lib.rs | 4 - .../version-upgrade-check/src/main.rs | 2 +- tools/ci/commands/wasm-bindings/Cargo.toml | 1 + tools/ci/commands/wasm-bindings/src/lib.rs | 4 - tools/ci/commands/wasm-bindings/src/main.rs | 2 +- tools/ci/src/main.rs | 30 +-- 51 files changed, 298 insertions(+), 236 deletions(-) create mode 100644 tools/ci/args/Cargo.toml create mode 100644 tools/ci/args/src/lib.rs delete mode 100644 tools/ci/commands/cla-assistant/src/lib.rs delete mode 100644 tools/ci/commands/cli-docs/src/lib.rs delete mode 100644 tools/ci/commands/codeowners-check/src/lib.rs delete mode 100644 tools/ci/commands/coordinate-internal-tests/src/lib.rs delete mode 100644 tools/ci/commands/docs/src/lib.rs delete mode 100644 tools/ci/commands/global-json-policy/src/lib.rs delete mode 100644 tools/ci/commands/keynote-bench/src/lib.rs delete mode 100644 tools/ci/commands/lint/src/lib.rs delete mode 100644 tools/ci/commands/publish-checks/src/lib.rs delete mode 100644 tools/ci/commands/smoketests/src/lib.rs delete mode 100644 tools/ci/commands/test/src/lib.rs delete mode 100644 tools/ci/commands/typescript-test/src/lib.rs delete mode 100644 tools/ci/commands/update-flow/src/lib.rs delete mode 100644 tools/ci/commands/version-upgrade-check/src/lib.rs delete mode 100644 tools/ci/commands/wasm-bindings/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 5ea11be474a..e21ac949cd9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -899,30 +899,24 @@ name = "ci" version = "0.1.0" dependencies = [ "anyhow", - "ci-cla-assistant", - "ci-cli-docs", - "ci-codeowners-check", - "ci-coordinate-internal-tests", - "ci-docs-build", - "ci-global-json-policy", - "ci-keynote-bench", - "ci-lint", - "ci-publish-checks", - "ci-smoketests", - "ci-test", - "ci-typescript-test", - "ci-update-flow", - "ci-version-upgrade-check", - "ci-wasm-bindings", + "ci-args", "clap 4.5.50", "duct", ] +[[package]] +name = "ci-args" +version = "0.1.0" +dependencies = [ + "clap 4.5.50", +] + [[package]] name = "ci-cla-assistant" version = "0.1.0" dependencies = [ "anyhow", + "ci-args", "clap 4.5.50", "reqwest 0.12.24", "serde", @@ -934,6 +928,7 @@ name = "ci-cli-docs" version = "0.1.0" dependencies = [ "anyhow", + "ci-args", "ci-common", "clap 4.5.50", "duct", @@ -944,6 +939,7 @@ name = "ci-codeowners-check" version = "0.1.0" dependencies = [ "anyhow", + "ci-args", "ci-common", "clap 4.5.50", "duct", @@ -964,6 +960,7 @@ name = "ci-coordinate-internal-tests" version = "0.1.0" dependencies = [ "anyhow", + "ci-args", "clap 4.5.50", "duct", "serde", @@ -975,6 +972,7 @@ name = "ci-docs-build" version = "0.1.0" dependencies = [ "anyhow", + "ci-args", "ci-common", "clap 4.5.50", ] @@ -984,6 +982,7 @@ name = "ci-global-json-policy" version = "0.1.0" dependencies = [ "anyhow", + "ci-args", "ci-common", "clap 4.5.50", "duct", @@ -994,6 +993,7 @@ name = "ci-keynote-bench" version = "0.1.0" dependencies = [ "anyhow", + "ci-args", "clap 4.5.50", "keynote-bench-harness", "spacetimedb-guard", @@ -1004,6 +1004,7 @@ name = "ci-lint" version = "0.1.0" dependencies = [ "anyhow", + "ci-args", "ci-common", "clap 4.5.50", "duct", @@ -1015,6 +1016,7 @@ name = "ci-publish-checks" version = "0.1.0" dependencies = [ "anyhow", + "ci-args", "clap 4.5.50", "duct", ] @@ -1024,6 +1026,7 @@ name = "ci-smoketests" version = "0.1.0" dependencies = [ "anyhow", + "ci-args", "ci-common", "clap 4.5.50", "duct", @@ -1036,6 +1039,7 @@ name = "ci-test" version = "0.1.0" dependencies = [ "anyhow", + "ci-args", "ci-common", "clap 4.5.50", "duct", @@ -1046,6 +1050,7 @@ name = "ci-typescript-test" version = "0.1.0" dependencies = [ "anyhow", + "ci-args", "ci-common", "clap 4.5.50", "duct", @@ -1056,6 +1061,7 @@ name = "ci-update-flow" version = "0.1.0" dependencies = [ "anyhow", + "ci-args", "clap 4.5.50", "duct", "tempfile", @@ -1066,6 +1072,7 @@ name = "ci-version-upgrade-check" version = "0.1.0" dependencies = [ "anyhow", + "ci-args", "clap 4.5.50", "duct", ] @@ -1075,6 +1082,7 @@ name = "ci-wasm-bindings" version = "0.1.0" dependencies = [ "anyhow", + "ci-args", "ci-common", "clap 4.5.50", "duct", diff --git a/Cargo.toml b/Cargo.toml index a1a94f15fc3..71778eedf35 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ members = [ "sdks/rust/tests/procedural-view-pk-client", "sdks/rust/tests/event-table-client", "tools/ci", + "tools/ci/args", "tools/ci/commands/test", "tools/ci/commands/lint", "tools/ci/commands/wasm-bindings", diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml index 21f9807674f..1d20cacb17a 100644 --- a/tools/ci/Cargo.toml +++ b/tools/ci/Cargo.toml @@ -7,18 +7,4 @@ edition.workspace = true anyhow.workspace = true clap.workspace = true duct.workspace = true -ci-cla-assistant = { path = "commands/cla-assistant" } -ci-cli-docs = { path = "commands/cli-docs" } -ci-codeowners-check = { path = "commands/codeowners-check" } -ci-coordinate-internal-tests = { path = "commands/coordinate-internal-tests" } -ci-docs-build = { path = "commands/docs" } -ci-global-json-policy = { path = "commands/global-json-policy" } -ci-keynote-bench = { path = "commands/keynote-bench" } -ci-lint = { path = "commands/lint" } -ci-publish-checks = { path = "commands/publish-checks" } -ci-smoketests = { path = "commands/smoketests" } -ci-test = { path = "commands/test" } -ci-typescript-test = { path = "commands/typescript-test" } -ci-update-flow = { path = "commands/update-flow" } -ci-version-upgrade-check = { path = "commands/version-upgrade-check" } -ci-wasm-bindings = { path = "commands/wasm-bindings" } +ci-args = { path = "args" } diff --git a/tools/ci/args/Cargo.toml b/tools/ci/args/Cargo.toml new file mode 100644 index 00000000000..61a7a9333c4 --- /dev/null +++ b/tools/ci/args/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "ci-args" +version = "0.1.0" +edition.workspace = true + +[dependencies] +clap.workspace = true diff --git a/tools/ci/args/src/lib.rs b/tools/ci/args/src/lib.rs new file mode 100644 index 00000000000..d0fd6af3d03 --- /dev/null +++ b/tools/ci/args/src/lib.rs @@ -0,0 +1,220 @@ +pub mod cla_assistant { + use clap::{ArgGroup, Args as ClapArgs, Subcommand}; + + #[derive(Subcommand)] + pub enum ClaAssistantCmd { + /// Retries CLA Assistant if `license/cla` is the only remaining PR blocker. + Retry(RetryArgs), + + /// Returns the `license/cla` status for a pull request or commit SHA. + Status(StatusArgs), + } + + #[derive(ClapArgs)] + pub struct RetryArgs { + /// Pull request number to check. + #[arg(long)] + pub pr_number: u64, + + /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. + #[arg(long)] + pub repo: Option, + } + + #[derive(ClapArgs)] + #[command(group( + ArgGroup::new("target") + .required(true) + .multiple(false) + .args(["pr", "sha"]), + ))] + pub struct StatusArgs { + /// Pull request number whose head commit should be checked. + #[arg(long)] + pub pr: Option, + + /// Commit SHA to check. + #[arg(long)] + pub sha: Option, + + /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. + #[arg(long)] + pub repo: Option, + } + + #[derive(ClapArgs)] + pub struct Args { + #[command(subcommand)] + pub cmd: ClaAssistantCmd, + } +} + +pub mod cli_docs { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args { + #[arg( + long, + long_help = "specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located)" + )] + pub spacetime_path: Option, + } +} + +pub mod codeowners_check { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args { + /// Git ref to compare against, usually origin/. + #[arg(long)] + pub base_ref: String, + + /// Pull request number to inspect for approval state. + #[arg(long)] + pub pr_number: u64, + } +} + +pub mod coordinate_internal_tests { + use clap::Args as ClapArgs; + + /// Selects or starts the private workflow for a public Internal Tests run. + #[derive(ClapArgs)] + pub struct Args { + /// Immutable public commit to test. + #[arg(long)] + pub public_sha: String, + + /// Public pull request number, when coordinating a pull request run. + #[arg(long)] + pub public_pr_number: Option, + } +} + +pub mod docs { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod global_json_policy { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod keynote_bench { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod lint { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod publish_checks { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod smoketests { + use clap::{Args as ClapArgs, Subcommand}; + + #[derive(ClapArgs)] + pub struct SmoketestsArgs { + #[command(subcommand)] + pub cmd: Option, + + /// Run tests against a remote server instead of spawning local servers. + /// + /// When specified, tests will connect to the given URL instead of starting + /// local server instances. Tests that require local server control (like + /// restart tests) will be skipped. + #[arg(long)] + pub server: Option, + + /// Use a SpacetimeAuth-issued login for remote-server tests. + /// + /// This is required for servers that reject direct server-issued logins for privileged operations. + /// + /// Optionally accepts an auth host to pass through to `spacetime login`, + /// for example `--auth-host=https://spacetimedb.com`. + #[arg(long, num_args = 0..=1, require_equals = true, default_missing_value = "")] + pub auth_host: Option, + + #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] + pub dotnet: bool, + + /// Additional arguments to pass to the test runner + #[arg(trailing_var_arg = true)] + pub args: Vec, + } + + #[derive(Subcommand)] + pub enum SmoketestCmd { + /// Only build binaries without running tests + /// + /// Use this before running `cargo test --all` to ensure binaries are built. + Prepare, + CheckModList, + } +} + +pub mod test { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod typescript_test { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod update_flow { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args { + #[arg( + long, + long_help = "Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms." + )] + pub target: Option, + + #[arg( + long, + default_value = "false", + long_help = "Whether to enable github token authentication feature when building the update binary. By default this is disabled." + )] + pub github_token_auth: bool, + } +} + +pub mod version_upgrade_check { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod wasm_bindings { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} diff --git a/tools/ci/commands/cla-assistant/Cargo.toml b/tools/ci/commands/cla-assistant/Cargo.toml index 4096cd8087e..eef51dc60ba 100644 --- a/tools/ci/commands/cla-assistant/Cargo.toml +++ b/tools/ci/commands/cla-assistant/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true [dependencies] anyhow.workspace = true clap.workspace = true +ci-args = { path = "../../args" } reqwest = { workspace = true, features = ["blocking", "json"] } serde.workspace = true serde_json.workspace = true diff --git a/tools/ci/commands/cla-assistant/src/lib.rs b/tools/ci/commands/cla-assistant/src/lib.rs deleted file mode 100644 index d20c55961e7..00000000000 --- a/tools/ci/commands/cla-assistant/src/lib.rs +++ /dev/null @@ -1,48 +0,0 @@ -use clap::{ArgGroup, Args as ClapArgs, Subcommand}; - -#[derive(Subcommand)] -pub enum ClaAssistantCmd { - /// Retries CLA Assistant if `license/cla` is the only remaining PR blocker. - Retry(RetryArgs), - - /// Returns the `license/cla` status for a pull request or commit SHA. - Status(StatusArgs), -} - -#[derive(ClapArgs)] -pub struct RetryArgs { - /// Pull request number to check. - #[arg(long)] - pub pr_number: u64, - - /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. - #[arg(long)] - pub repo: Option, -} - -#[derive(ClapArgs)] -#[command(group( - ArgGroup::new("target") - .required(true) - .multiple(false) - .args(["pr", "sha"]), -))] -pub struct StatusArgs { - /// Pull request number whose head commit should be checked. - #[arg(long)] - pub pr: Option, - - /// Commit SHA to check. - #[arg(long)] - pub sha: Option, - - /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. - #[arg(long)] - pub repo: Option, -} - -#[derive(ClapArgs)] -pub struct Args { - #[command(subcommand)] - pub cmd: ClaAssistantCmd, -} diff --git a/tools/ci/commands/cla-assistant/src/main.rs b/tools/ci/commands/cla-assistant/src/main.rs index 434eedf667c..dcb8d7c6e8d 100644 --- a/tools/ci/commands/cla-assistant/src/main.rs +++ b/tools/ci/commands/cla-assistant/src/main.rs @@ -4,7 +4,7 @@ use std::collections::BTreeMap; use std::env; use anyhow::{anyhow, bail, Context, Result}; -use ci_cla_assistant::{Args, ClaAssistantCmd, RetryArgs, StatusArgs}; +use ci_args::cla_assistant::{Args, ClaAssistantCmd, RetryArgs, StatusArgs}; use clap::Parser; use reqwest::blocking::Client; use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, AUTHORIZATION, USER_AGENT}; diff --git a/tools/ci/commands/cli-docs/Cargo.toml b/tools/ci/commands/cli-docs/Cargo.toml index 9ea97b1cc91..07d7ff20840 100644 --- a/tools/ci/commands/cli-docs/Cargo.toml +++ b/tools/ci/commands/cli-docs/Cargo.toml @@ -7,4 +7,5 @@ edition.workspace = true anyhow.workspace = true duct.workspace = true clap.workspace = true +ci-args = { path = "../../args" } ci-common = { path = "../../common" } diff --git a/tools/ci/commands/cli-docs/src/lib.rs b/tools/ci/commands/cli-docs/src/lib.rs deleted file mode 100644 index 35c2dfed8ca..00000000000 --- a/tools/ci/commands/cli-docs/src/lib.rs +++ /dev/null @@ -1,10 +0,0 @@ -use clap::Args as ClapArgs; - -#[derive(ClapArgs)] -pub struct Args { - #[arg( - long, - long_help = "specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located)" - )] - pub spacetime_path: Option, -} diff --git a/tools/ci/commands/cli-docs/src/main.rs b/tools/ci/commands/cli-docs/src/main.rs index 53f5b1b5ff7..9829bbfbef3 100644 --- a/tools/ci/commands/cli-docs/src/main.rs +++ b/tools/ci/commands/cli-docs/src/main.rs @@ -1,6 +1,6 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; -use ci_cli_docs::Args; +use ci_args::cli_docs::Args; use ci_common::pnpm; use clap::Parser; use duct::cmd; diff --git a/tools/ci/commands/codeowners-check/Cargo.toml b/tools/ci/commands/codeowners-check/Cargo.toml index 19d76f82671..c553c6499b1 100644 --- a/tools/ci/commands/codeowners-check/Cargo.toml +++ b/tools/ci/commands/codeowners-check/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true [dependencies] anyhow.workspace = true clap.workspace = true +ci-args = { path = "../../args" } ci-common = { path = "../../common" } duct.workspace = true regex.workspace = true diff --git a/tools/ci/commands/codeowners-check/src/lib.rs b/tools/ci/commands/codeowners-check/src/lib.rs deleted file mode 100644 index ed27d6fda38..00000000000 --- a/tools/ci/commands/codeowners-check/src/lib.rs +++ /dev/null @@ -1,12 +0,0 @@ -use clap::Args as ClapArgs; - -#[derive(ClapArgs)] -pub struct Args { - /// Git ref to compare against, usually origin/. - #[arg(long)] - pub base_ref: String, - - /// Pull request number to inspect for approval state. - #[arg(long)] - pub pr_number: u64, -} diff --git a/tools/ci/commands/codeowners-check/src/main.rs b/tools/ci/commands/codeowners-check/src/main.rs index 6476648b483..07866923242 100644 --- a/tools/ci/commands/codeowners-check/src/main.rs +++ b/tools/ci/commands/codeowners-check/src/main.rs @@ -1,5 +1,5 @@ use anyhow::{anyhow, bail, Context, Result}; -use ci_codeowners_check::Args; +use ci_args::codeowners_check::Args; use ci_common::ensure_repo_root; use clap::Parser; use duct::cmd; @@ -9,8 +9,8 @@ use std::path::{Path, PathBuf}; const REPO: &str = "clockworklabs/SpacetimeDB"; -#[command(about = "Checks that sensitive CODEOWNERS-controlled files have the required approvals.")] #[derive(Parser)] +#[command(about = "Checks that sensitive CODEOWNERS-controlled files have the required approvals.")] struct Cli { #[command(flatten)] args: Args, diff --git a/tools/ci/commands/coordinate-internal-tests/Cargo.toml b/tools/ci/commands/coordinate-internal-tests/Cargo.toml index c4b71bd705d..da01e6cad03 100644 --- a/tools/ci/commands/coordinate-internal-tests/Cargo.toml +++ b/tools/ci/commands/coordinate-internal-tests/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true [dependencies] anyhow.workspace = true clap.workspace = true +ci-args = { path = "../../args" } duct.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/tools/ci/commands/coordinate-internal-tests/src/lib.rs b/tools/ci/commands/coordinate-internal-tests/src/lib.rs deleted file mode 100644 index a0e066b8cf2..00000000000 --- a/tools/ci/commands/coordinate-internal-tests/src/lib.rs +++ /dev/null @@ -1,13 +0,0 @@ -use clap::Args as ClapArgs; - -/// Selects or starts the private workflow for a public Internal Tests run. -#[derive(ClapArgs)] -pub struct Args { - /// Immutable public commit to test. - #[arg(long)] - pub public_sha: String, - - /// Public pull request number, when coordinating a pull request run. - #[arg(long)] - pub public_pr_number: Option, -} diff --git a/tools/ci/commands/coordinate-internal-tests/src/main.rs b/tools/ci/commands/coordinate-internal-tests/src/main.rs index 0fde3826a76..d913cf94fed 100644 --- a/tools/ci/commands/coordinate-internal-tests/src/main.rs +++ b/tools/ci/commands/coordinate-internal-tests/src/main.rs @@ -1,7 +1,7 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, ensure, Context, Result}; -use ci_coordinate_internal_tests::Args; +use ci_args::coordinate_internal_tests::Args; use clap::Parser; use duct::{cmd, Expression}; use serde::de::DeserializeOwned; diff --git a/tools/ci/commands/docs/Cargo.toml b/tools/ci/commands/docs/Cargo.toml index 156da133e6b..411892ec988 100644 --- a/tools/ci/commands/docs/Cargo.toml +++ b/tools/ci/commands/docs/Cargo.toml @@ -6,4 +6,5 @@ edition.workspace = true [dependencies] anyhow.workspace = true clap.workspace = true +ci-args = { path = "../../args" } ci-common = { path = "../../common" } diff --git a/tools/ci/commands/docs/src/lib.rs b/tools/ci/commands/docs/src/lib.rs deleted file mode 100644 index d129311db6c..00000000000 --- a/tools/ci/commands/docs/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -use clap::Args as ClapArgs; - -#[derive(ClapArgs)] -pub struct Args {} diff --git a/tools/ci/commands/docs/src/main.rs b/tools/ci/commands/docs/src/main.rs index cd3fb33c6bb..fd8e305ccce 100644 --- a/tools/ci/commands/docs/src/main.rs +++ b/tools/ci/commands/docs/src/main.rs @@ -1,7 +1,7 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; +use ci_args::docs::Args; use ci_common::pnpm; -use ci_docs_build::Args; use clap::Parser; /// Builds the docs site. diff --git a/tools/ci/commands/global-json-policy/Cargo.toml b/tools/ci/commands/global-json-policy/Cargo.toml index b45b8ff0cab..644ef6df18c 100644 --- a/tools/ci/commands/global-json-policy/Cargo.toml +++ b/tools/ci/commands/global-json-policy/Cargo.toml @@ -6,5 +6,6 @@ edition.workspace = true [dependencies] anyhow.workspace = true clap.workspace = true +ci-args = { path = "../../args" } duct.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/global-json-policy/src/lib.rs b/tools/ci/commands/global-json-policy/src/lib.rs deleted file mode 100644 index d129311db6c..00000000000 --- a/tools/ci/commands/global-json-policy/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -use clap::Args as ClapArgs; - -#[derive(ClapArgs)] -pub struct Args {} diff --git a/tools/ci/commands/global-json-policy/src/main.rs b/tools/ci/commands/global-json-policy/src/main.rs index 32aedacc1f0..b2e680b4055 100644 --- a/tools/ci/commands/global-json-policy/src/main.rs +++ b/tools/ci/commands/global-json-policy/src/main.rs @@ -1,8 +1,8 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Result}; +use ci_args::global_json_policy::Args; use ci_common::ensure_repo_root; -use ci_global_json_policy::Args; use clap::Parser; use duct::cmd; use std::fs; diff --git a/tools/ci/commands/keynote-bench/Cargo.toml b/tools/ci/commands/keynote-bench/Cargo.toml index c171abbcb2e..508ee26eebf 100644 --- a/tools/ci/commands/keynote-bench/Cargo.toml +++ b/tools/ci/commands/keynote-bench/Cargo.toml @@ -6,5 +6,6 @@ edition.workspace = true [dependencies] anyhow.workspace = true clap.workspace = true +ci-args = { path = "../../args" } keynote-bench-harness = { path = "../../../keynote-bench-harness" } spacetimedb-guard.workspace = true diff --git a/tools/ci/commands/keynote-bench/src/lib.rs b/tools/ci/commands/keynote-bench/src/lib.rs deleted file mode 100644 index d129311db6c..00000000000 --- a/tools/ci/commands/keynote-bench/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -use clap::Args as ClapArgs; - -#[derive(ClapArgs)] -pub struct Args {} diff --git a/tools/ci/commands/keynote-bench/src/main.rs b/tools/ci/commands/keynote-bench/src/main.rs index ede810009e0..cc97ae8cef9 100644 --- a/tools/ci/commands/keynote-bench/src/main.rs +++ b/tools/ci/commands/keynote-bench/src/main.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use ci_keynote_bench::Args; +use ci_args::keynote_bench::Args; use clap::Parser; use keynote_bench_harness::KeynoteBenchConfig; use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; diff --git a/tools/ci/commands/lint/Cargo.toml b/tools/ci/commands/lint/Cargo.toml index 44bfa5776bd..67eef8b8b9d 100644 --- a/tools/ci/commands/lint/Cargo.toml +++ b/tools/ci/commands/lint/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true [dependencies] anyhow.workspace = true clap.workspace = true +ci-args = { path = "../../args" } duct.workspace = true serde_json.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/lint/src/lib.rs b/tools/ci/commands/lint/src/lib.rs deleted file mode 100644 index d129311db6c..00000000000 --- a/tools/ci/commands/lint/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -use clap::Args as ClapArgs; - -#[derive(ClapArgs)] -pub struct Args {} diff --git a/tools/ci/commands/lint/src/main.rs b/tools/ci/commands/lint/src/main.rs index 52a42f86d93..ddf6365fd8b 100644 --- a/tools/ci/commands/lint/src/main.rs +++ b/tools/ci/commands/lint/src/main.rs @@ -1,8 +1,8 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Context, Result}; +use ci_args::lint::Args; use ci_common::{ensure_repo_root, pnpm}; -use ci_lint::Args; use clap::Parser; use duct::cmd; use serde_json::Value; diff --git a/tools/ci/commands/publish-checks/Cargo.toml b/tools/ci/commands/publish-checks/Cargo.toml index 4f2cda1ede6..be73f4263d5 100644 --- a/tools/ci/commands/publish-checks/Cargo.toml +++ b/tools/ci/commands/publish-checks/Cargo.toml @@ -6,4 +6,5 @@ edition.workspace = true [dependencies] anyhow.workspace = true clap.workspace = true +ci-args = { path = "../../args" } duct.workspace = true diff --git a/tools/ci/commands/publish-checks/src/lib.rs b/tools/ci/commands/publish-checks/src/lib.rs deleted file mode 100644 index d129311db6c..00000000000 --- a/tools/ci/commands/publish-checks/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -use clap::Args as ClapArgs; - -#[derive(ClapArgs)] -pub struct Args {} diff --git a/tools/ci/commands/publish-checks/src/main.rs b/tools/ci/commands/publish-checks/src/main.rs index 9a966565b36..3e2b7e71ccd 100644 --- a/tools/ci/commands/publish-checks/src/main.rs +++ b/tools/ci/commands/publish-checks/src/main.rs @@ -1,6 +1,6 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Result}; -use ci_publish_checks::Args; +use ci_args::publish_checks::Args; use clap::Parser; use duct::cmd; diff --git a/tools/ci/commands/smoketests/Cargo.toml b/tools/ci/commands/smoketests/Cargo.toml index f161fe64586..42ec134b129 100644 --- a/tools/ci/commands/smoketests/Cargo.toml +++ b/tools/ci/commands/smoketests/Cargo.toml @@ -7,6 +7,7 @@ edition.workspace = true anyhow.workspace = true duct.workspace = true clap.workspace = true +ci-args = { path = "../../args" } tempfile.workspace = true spacetimedb-guard.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/smoketests/src/lib.rs b/tools/ci/commands/smoketests/src/lib.rs deleted file mode 100644 index 80a4c02f35e..00000000000 --- a/tools/ci/commands/smoketests/src/lib.rs +++ /dev/null @@ -1,40 +0,0 @@ -use clap::{Args as ClapArgs, Subcommand}; - -#[derive(ClapArgs)] -pub struct SmoketestsArgs { - #[command(subcommand)] - pub cmd: Option, - - /// Run tests against a remote server instead of spawning local servers. - /// - /// When specified, tests will connect to the given URL instead of starting - /// local server instances. Tests that require local server control (like - /// restart tests) will be skipped. - #[arg(long)] - pub server: Option, - - /// Use a SpacetimeAuth-issued login for remote-server tests. - /// - /// This is required for servers that reject direct server-issued logins for privileged operations. - /// - /// Optionally accepts an auth host to pass through to `spacetime login`, - /// for example `--auth-host=https://spacetimedb.com`. - #[arg(long, num_args = 0..=1, require_equals = true, default_missing_value = "")] - pub auth_host: Option, - - #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] - pub dotnet: bool, - - /// Additional arguments to pass to the test runner - #[arg(trailing_var_arg = true)] - pub args: Vec, -} - -#[derive(Subcommand)] -pub enum SmoketestCmd { - /// Only build binaries without running tests - /// - /// Use this before running `cargo test --all` to ensure binaries are built. - Prepare, - CheckModList, -} diff --git a/tools/ci/commands/smoketests/src/main.rs b/tools/ci/commands/smoketests/src/main.rs index feb7d3e78d0..2342d35d98a 100644 --- a/tools/ci/commands/smoketests/src/main.rs +++ b/tools/ci/commands/smoketests/src/main.rs @@ -1,7 +1,7 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, ensure, Context, Result}; +use ci_args::smoketests::{SmoketestCmd, SmoketestsArgs}; use ci_common::ensure_repo_root; -use ci_smoketests::{SmoketestCmd, SmoketestsArgs}; use clap::Parser; use duct::cmd; use spacetimedb_guard::ensure_binaries_built; diff --git a/tools/ci/commands/test/Cargo.toml b/tools/ci/commands/test/Cargo.toml index 91d98a7f490..040ad9fa502 100644 --- a/tools/ci/commands/test/Cargo.toml +++ b/tools/ci/commands/test/Cargo.toml @@ -6,5 +6,6 @@ edition.workspace = true [dependencies] anyhow.workspace = true clap.workspace = true +ci-args = { path = "../../args" } duct.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/test/src/lib.rs b/tools/ci/commands/test/src/lib.rs deleted file mode 100644 index d129311db6c..00000000000 --- a/tools/ci/commands/test/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -use clap::Args as ClapArgs; - -#[derive(ClapArgs)] -pub struct Args {} diff --git a/tools/ci/commands/test/src/main.rs b/tools/ci/commands/test/src/main.rs index 6d3fb13dc91..b9f8e72c912 100644 --- a/tools/ci/commands/test/src/main.rs +++ b/tools/ci/commands/test/src/main.rs @@ -1,7 +1,7 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; +use ci_args::test::Args; use ci_common::pnpm; -use ci_test::Args; use clap::Parser; use duct::cmd; diff --git a/tools/ci/commands/typescript-test/Cargo.toml b/tools/ci/commands/typescript-test/Cargo.toml index 170f3618076..e7ca40b5200 100644 --- a/tools/ci/commands/typescript-test/Cargo.toml +++ b/tools/ci/commands/typescript-test/Cargo.toml @@ -6,5 +6,6 @@ edition.workspace = true [dependencies] anyhow.workspace = true clap.workspace = true +ci-args = { path = "../../args" } duct.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/typescript-test/src/lib.rs b/tools/ci/commands/typescript-test/src/lib.rs deleted file mode 100644 index d129311db6c..00000000000 --- a/tools/ci/commands/typescript-test/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -use clap::Args as ClapArgs; - -#[derive(ClapArgs)] -pub struct Args {} diff --git a/tools/ci/commands/typescript-test/src/main.rs b/tools/ci/commands/typescript-test/src/main.rs index 77e01fc9e63..f0a783a675c 100644 --- a/tools/ci/commands/typescript-test/src/main.rs +++ b/tools/ci/commands/typescript-test/src/main.rs @@ -1,7 +1,7 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, Result}; +use ci_args::typescript_test::Args; use ci_common::pnpm; -use ci_typescript_test::Args; use clap::Parser; use duct::cmd; diff --git a/tools/ci/commands/update-flow/Cargo.toml b/tools/ci/commands/update-flow/Cargo.toml index 590c8305711..15a89ac5f6f 100644 --- a/tools/ci/commands/update-flow/Cargo.toml +++ b/tools/ci/commands/update-flow/Cargo.toml @@ -8,3 +8,4 @@ anyhow.workspace = true duct.workspace = true tempfile.workspace = true clap.workspace = true +ci-args = { path = "../../args" } diff --git a/tools/ci/commands/update-flow/src/lib.rs b/tools/ci/commands/update-flow/src/lib.rs deleted file mode 100644 index 5543d9cc3d8..00000000000 --- a/tools/ci/commands/update-flow/src/lib.rs +++ /dev/null @@ -1,16 +0,0 @@ -use clap::Args as ClapArgs; - -#[derive(ClapArgs)] -pub struct Args { - #[arg( - long, - long_help = "Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms." - )] - pub target: Option, - #[arg( - long, - default_value = "false", - long_help = "Whether to enable github token authentication feature when building the update binary. By default this is disabled." - )] - pub github_token_auth: bool, -} diff --git a/tools/ci/commands/update-flow/src/main.rs b/tools/ci/commands/update-flow/src/main.rs index ba70b6e3275..8f0fe9d88c3 100644 --- a/tools/ci/commands/update-flow/src/main.rs +++ b/tools/ci/commands/update-flow/src/main.rs @@ -1,6 +1,6 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; -use ci_update_flow::Args; +use ci_args::update_flow::Args; use clap::Parser; use duct::cmd; diff --git a/tools/ci/commands/version-upgrade-check/Cargo.toml b/tools/ci/commands/version-upgrade-check/Cargo.toml index 4ab568a5789..bd978f6e437 100644 --- a/tools/ci/commands/version-upgrade-check/Cargo.toml +++ b/tools/ci/commands/version-upgrade-check/Cargo.toml @@ -6,4 +6,5 @@ edition.workspace = true [dependencies] anyhow.workspace = true clap.workspace = true +ci-args = { path = "../../args" } duct.workspace = true diff --git a/tools/ci/commands/version-upgrade-check/src/lib.rs b/tools/ci/commands/version-upgrade-check/src/lib.rs deleted file mode 100644 index d129311db6c..00000000000 --- a/tools/ci/commands/version-upgrade-check/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -use clap::Args as ClapArgs; - -#[derive(ClapArgs)] -pub struct Args {} diff --git a/tools/ci/commands/version-upgrade-check/src/main.rs b/tools/ci/commands/version-upgrade-check/src/main.rs index 05b533db0ac..f2c1f7e0bf4 100644 --- a/tools/ci/commands/version-upgrade-check/src/main.rs +++ b/tools/ci/commands/version-upgrade-check/src/main.rs @@ -1,6 +1,6 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; -use ci_version_upgrade_check::Args; +use ci_args::version_upgrade_check::Args; use clap::Parser; use duct::cmd; diff --git a/tools/ci/commands/wasm-bindings/Cargo.toml b/tools/ci/commands/wasm-bindings/Cargo.toml index e807f25ea7e..a255c928130 100644 --- a/tools/ci/commands/wasm-bindings/Cargo.toml +++ b/tools/ci/commands/wasm-bindings/Cargo.toml @@ -6,5 +6,6 @@ edition.workspace = true [dependencies] anyhow.workspace = true clap.workspace = true +ci-args = { path = "../../args" } duct.workspace = true ci-common = { path = "../../common" } diff --git a/tools/ci/commands/wasm-bindings/src/lib.rs b/tools/ci/commands/wasm-bindings/src/lib.rs deleted file mode 100644 index d129311db6c..00000000000 --- a/tools/ci/commands/wasm-bindings/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -use clap::Args as ClapArgs; - -#[derive(ClapArgs)] -pub struct Args {} diff --git a/tools/ci/commands/wasm-bindings/src/main.rs b/tools/ci/commands/wasm-bindings/src/main.rs index ed4aae9ef95..7b562bad2fa 100644 --- a/tools/ci/commands/wasm-bindings/src/main.rs +++ b/tools/ci/commands/wasm-bindings/src/main.rs @@ -1,7 +1,7 @@ #![allow(clippy::disallowed_macros)] use anyhow::Result; +use ci_args::wasm_bindings::Args; use ci_common::pnpm; -use ci_wasm_bindings::Args; use clap::Parser; use duct::cmd; diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index ac31a5b3c47..125768c0450 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -33,46 +33,46 @@ enum CiCmd { /// Runs rust tests, codegens csharp sdk and runs csharp tests. /// This does not include Unreal tests. /// This expects to run in a clean git state. - Test(ci_test::Args), + Test(ci_args::test::Args), /// Lints the codebase /// /// Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there /// are no warnings. - Lint(ci_lint::Args), + Lint(ci_args::lint::Args), /// Tests Wasm bindings /// /// Runs tests for the codegen crate and builds a test module with the wasm bindings. - WasmBindings(ci_wasm_bindings::Args), + WasmBindings(ci_args::wasm_bindings::Args), /// Deprecated; use `cargo regen csharp dlls`. Dlls, /// Runs smoketests /// /// Executes the smoketests suite with some default exclusions. - Smoketests(ci_smoketests::SmoketestsArgs), + Smoketests(ci_args::smoketests::SmoketestsArgs), /// Runs the keynote benchmark as a CI performance regression gate. /// /// Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the /// keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and /// fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. - KeynoteBench(ci_keynote_bench::Args), + KeynoteBench(ci_args::keynote_bench::Args), /// Tests the update flow /// /// Tests the self-update flow by building the spacetimedb-update binary for the specified /// target, by default the current target, and performing a self-install into a temporary /// directory. - UpdateFlow(ci_update_flow::Args), + UpdateFlow(ci_args::update_flow::Args), /// Generates CLI documentation and checks for changes - CliDocs(ci_cli_docs::Args), + CliDocs(ci_args::cli_docs::Args), /// Verify that any non-root global.json files are symlinks to the root global.json. - GlobalJsonPolicy(ci_global_json_policy::Args), + GlobalJsonPolicy(ci_args::global_json_policy::Args), /// Checks that publishable crates satisfy publish constraints. - PublishChecks(ci_publish_checks::Args), + PublishChecks(ci_args::publish_checks::Args), /// Runs TypeScript workspace tests and template build checks. - TypescriptTest(ci_typescript_test::Args), + TypescriptTest(ci_args::typescript_test::Args), /// Verifies that the repository version upgrade tool still works. - VersionUpgradeCheck(ci_version_upgrade_check::Args), + VersionUpgradeCheck(ci_args::version_upgrade_check::Args), /// Builds the docs site. - Docs(ci_docs_build::Args), + Docs(ci_args::docs::Args), OtherWorkflows { #[command(subcommand)] cmd: OtherWorkflowsCmd, @@ -82,11 +82,11 @@ enum CiCmd { #[derive(Subcommand)] enum OtherWorkflowsCmd { /// Selects or starts the private workflow for a public Internal Tests run. - CoordinateInternalTests(ci_coordinate_internal_tests::Args), + CoordinateInternalTests(ci_args::coordinate_internal_tests::Args), /// Checks that sensitive CODEOWNERS-controlled files have the required approvals. - CodeownersCheck(ci_codeowners_check::Args), + CodeownersCheck(ci_args::codeowners_check::Args), /// Interacts with CLA Assistant. - ClaAssistant(ci_cla_assistant::Args), + ClaAssistant(ci_args::cla_assistant::Args), } fn run_package(package: &str, args: &[String]) -> Result<()> {