diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7ed75409a55..18d3516d5f2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,8 +8,8 @@ LICENSE.txt @cloutiertyler /crates/cli/src/ @bfops @cloutiertyler @jdetter /tools/ci/ @bfops @cloutiertyler @jdetter -/tools/ci/src/keynote_bench.rs @joshua-spacetime @cloutiertyler @jdetter -/tools/ci/src/codeowners_check.rs @cloutiertyler +/tools/ci/commands/keynote-bench/ @joshua-spacetime @cloutiertyler @jdetter +/tools/ci/commands/codeowners-check/ @cloutiertyler /tools/upgrade-version/ @bfops @jdetter @cloutiertyler @rekhoff /tools/release/ @bfops @jdetter @cloutiertyler @rekhoff /tools/license-check/ @bfops @jdetter @cloutiertyler diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c39b4a2458e..a01744937dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -681,39 +681,6 @@ jobs: cargo test -- --test-threads=1 ' - ci_command_docs: - needs: [merge_queue_noop] - if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} - name: Check CI command docs - runs-on: spacetimedb-new-runner-2 - env: - RUST_BACKTRACE: full - steps: - - name: Find Git ref - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - run: | - PR_NUMBER="${{ github.event.inputs.pr_number || null }}" - if test -n "${PR_NUMBER}"; then - GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" - else - GIT_REF="${{ github.ref }}" - fi - echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - - - name: Checkout sources - uses: actions/checkout@v4 - with: - ref: ${{ env.GIT_REF }} - - - uses: dsherret/rust-toolchain-file@v1 - - name: Set default rust toolchain - run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - - name: Check for docs change - run: cargo ci self-docs --check - cli_docs: needs: [merge_queue_noop] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} diff --git a/Cargo.lock b/Cargo.lock index bb5abb64fdd..e21ac949cd9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -899,20 +899,195 @@ name = "ci" version = "0.1.0" dependencies = [ "anyhow", - "chrono", + "ci-args", "clap 4.5.50", "duct", - "env_logger 0.10.2", - "keynote-bench-harness", - "log", - "regex", +] + +[[package]] +name = "ci-args" +version = "0.1.0" +dependencies = [ + "clap 4.5.50", +] + +[[package]] +name = "ci-cla-assistant" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-args", + "clap 4.5.50", "reqwest 0.12.24", "serde", "serde_json", +] + +[[package]] +name = "ci-cli-docs" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-args", + "ci-common", + "clap 4.5.50", + "duct", +] + +[[package]] +name = "ci-codeowners-check" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-args", + "ci-common", + "clap 4.5.50", + "duct", + "regex", + "serde_json", +] + +[[package]] +name = "ci-common" +version = "0.1.0" +dependencies = [ + "anyhow", + "duct", +] + +[[package]] +name = "ci-coordinate-internal-tests" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-args", + "clap 4.5.50", + "duct", + "serde", + "serde_json", +] + +[[package]] +name = "ci-docs-build" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-args", + "ci-common", + "clap 4.5.50", +] + +[[package]] +name = "ci-global-json-policy" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-args", + "ci-common", + "clap 4.5.50", + "duct", +] + +[[package]] +name = "ci-keynote-bench" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-args", + "clap 4.5.50", + "keynote-bench-harness", + "spacetimedb-guard", +] + +[[package]] +name = "ci-lint" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-args", + "ci-common", + "clap 4.5.50", + "duct", + "serde_json", +] + +[[package]] +name = "ci-publish-checks" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-args", + "clap 4.5.50", + "duct", +] + +[[package]] +name = "ci-smoketests" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-args", + "ci-common", + "clap 4.5.50", + "duct", "spacetimedb-guard", "tempfile", ] +[[package]] +name = "ci-test" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-args", + "ci-common", + "clap 4.5.50", + "duct", +] + +[[package]] +name = "ci-typescript-test" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-args", + "ci-common", + "clap 4.5.50", + "duct", +] + +[[package]] +name = "ci-update-flow" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-args", + "clap 4.5.50", + "duct", + "tempfile", +] + +[[package]] +name = "ci-version-upgrade-check" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-args", + "clap 4.5.50", + "duct", +] + +[[package]] +name = "ci-wasm-bindings" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-args", + "ci-common", + "clap 4.5.50", + "duct", +] + [[package]] name = "ciborium" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index c31354942a5..71778eedf35 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,23 @@ members = [ "sdks/rust/tests/procedural-view-pk-client", "sdks/rust/tests/event-table-client", "tools/ci", + "tools/ci/args", + "tools/ci/commands/test", + "tools/ci/commands/lint", + "tools/ci/commands/wasm-bindings", + "tools/ci/commands/smoketests", + "tools/ci/commands/keynote-bench", + "tools/ci/commands/update-flow", + "tools/ci/commands/cli-docs", + "tools/ci/commands/global-json-policy", + "tools/ci/commands/publish-checks", + "tools/ci/commands/typescript-test", + "tools/ci/commands/version-upgrade-check", + "tools/ci/commands/docs", + "tools/ci/commands/codeowners-check", + "tools/ci/commands/cla-assistant", + "tools/ci/commands/coordinate-internal-tests", + "tools/ci/common", "tools/keynote-bench-harness", "tools/license-check", "tools/replace-spacetimedb", diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml index 3374b66b34d..1d20cacb17a 100644 --- a/tools/ci/Cargo.toml +++ b/tools/ci/Cargo.toml @@ -4,16 +4,7 @@ version = "0.1.0" edition.workspace = true [dependencies] -log.workspace = true anyhow.workspace = true -chrono = { workspace = true, features=["clock"] } clap.workspace = true -regex.workspace = true -reqwest = { workspace = true, features = ["blocking", "json"] } -serde.workspace = true -serde_json.workspace = true duct.workspace = true -tempfile.workspace = true -env_logger.workspace = true -keynote-bench-harness = { path = "../keynote-bench-harness" } -spacetimedb-guard.workspace = true +ci-args = { path = "args" } diff --git a/tools/ci/README.md b/tools/ci/README.md deleted file mode 100644 index f0563733ee2..00000000000 --- a/tools/ci/README.md +++ /dev/null @@ -1,378 +0,0 @@ -# SpacetimeDB's cargo ci - -## Overview - -This document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options. - -## `cargo ci` - -SpacetimeDB CI tasks - -This tool provides several subcommands for automating CI workflows in SpacetimeDB. - -It may be invoked via `cargo ci `, or simply `cargo ci` to run all subcommands in sequence. It is mostly designed to be run in CI environments via the github workflows, but can also be run locally - -**Usage:** -```bash -Usage: cargo ci [OPTIONS] [COMMAND] -``` - -**Options:** - -- `--skip `: Skip specified subcommands when running all - -When no subcommand is specified, all subcommands are run in sequence. This option allows specifying subcommands to skip when running all. For example, to skip the `unreal-tests` subcommand, use `--skip unreal-tests`. - -- `--help`: Print help (see a summary with '-h') - -### `test` - -Runs tests - -Runs rust tests, codegens csharp sdk and runs csharp tests. This does not include Unreal tests. This expects to run in a clean git state. - -**Usage:** -```bash -Usage: test -``` - -**Options:** - -- `--help`: Print help (see a summary with '-h') - -### `lint` - -Lints the codebase - -Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there are no warnings. - -**Usage:** -```bash -Usage: lint -``` - -**Options:** - -- `--help`: Print help (see a summary with '-h') - -### `wasm-bindings` - -Tests Wasm bindings - -Runs tests for the codegen crate and builds a test module with the wasm bindings. - -**Usage:** -```bash -Usage: wasm-bindings -``` - -**Options:** - -- `--help`: Print help (see a summary with '-h') - -### `dlls` - -Deprecated; use `cargo regen csharp dlls`. - -Builds and packs C# DLLs and NuGet packages for local Unity workflows. - -**Usage:** -```bash -Usage: dlls -``` - -**Options:** - -- `--help`: Print help (see a summary with '-h') - -### `smoketests` - -Runs smoketests - -Executes the smoketests suite with some default exclusions. - -**Usage:** -```bash -Usage: smoketests [OPTIONS] [ARGS]... [COMMAND] -``` - -**Options:** - -- `--server `: Run tests against a remote server instead of spawning local servers. - -When specified, tests will connect to the given URL instead of starting local server instances. Tests that require local server control (like restart tests) will be skipped. - -- `--auth-host `: Use a SpacetimeAuth-issued login for remote-server tests. - -This is required for servers that reject direct server-issued logins for privileged operations. - -Optionally accepts an auth host to pass through to `spacetime login`, for example `--auth-host=https://spacetimedb.com`. - -- `--dotnet `: -- `args `: Additional arguments to pass to the test runner -- `--help`: Print help (see a summary with '-h') - -#### `prepare` - -Only build binaries without running tests - -Use this before running `cargo test --all` to ensure binaries are built. - -**Usage:** -```bash -Usage: prepare -``` - -**Options:** - -- `--help`: Print help (see a summary with '-h') - -#### `check-mod-list` - -**Usage:** -```bash -Usage: check-mod-list -``` - -**Options:** - -- `--help`: Print help - -#### `help` - -**Usage:** -```bash -Usage: help [COMMAND]... -``` - -**Options:** - -- `subcommand `: Print help for the subcommand(s) - -### `keynote-bench` - -Runs the keynote benchmark as a CI performance regression gate. - -Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. - -**Usage:** -```bash -Usage: keynote-bench -``` - -**Options:** - -- `--help`: Print help (see a summary with '-h') - -### `update-flow` - -Tests the update flow - -Tests the self-update flow by building the spacetimedb-update binary for the specified target, by default the current target, and performing a self-install into a temporary directory. - -**Usage:** -```bash -Usage: update-flow [OPTIONS] -``` - -**Options:** - -- `--target `: Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms. -- `--github-token-auth`: Whether to enable github token authentication feature when building the update binary. By default this is disabled. -- `--help`: Print help (see a summary with '-h') - -### `cli-docs` - -**Usage:** -```bash -Usage: cli-docs [OPTIONS] -``` - -**Options:** - -- `--spacetime-path `: specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located) -- `--help`: Print help (see a summary with '-h') - -### `self-docs` - -**Usage:** -```bash -Usage: self-docs [OPTIONS] -``` - -**Options:** - -- `--check`: Only check for changes, do not generate the docs -- `--help`: Print help (see a summary with '-h') - -### `global-json-policy` - -**Usage:** -```bash -Usage: global-json-policy -``` - -**Options:** - -- `--help`: Print help - -### `publish-checks` - -**Usage:** -```bash -Usage: publish-checks -``` - -**Options:** - -- `--help`: Print help - -### `typescript-test` - -**Usage:** -```bash -Usage: typescript-test -``` - -**Options:** - -- `--help`: Print help - -### `version-upgrade-check` - -**Usage:** -```bash -Usage: version-upgrade-check -``` - -**Options:** - -- `--help`: Print help - -### `docs` - -**Usage:** -```bash -Usage: docs -``` - -**Options:** - -- `--help`: Print help - -### `other-workflows` - -**Usage:** -```bash -Usage: other-workflows -``` - -**Options:** - -- `--help`: Print help - -#### `coordinate-internal-tests` - -**Usage:** -```bash -Usage: coordinate-internal-tests [OPTIONS] --public-sha -``` - -**Options:** - -- `--public-sha `: Immutable public commit to test -- `--public-pr-number `: Public pull request number, when coordinating a pull request run -- `--help`: Print help - -#### `codeowners-check` - -**Usage:** -```bash -Usage: codeowners-check --base-ref --pr-number -``` - -**Options:** - -- `--base-ref `: Git ref to compare against, usually origin/ -- `--pr-number `: Pull request number to inspect for approval state -- `--help`: Print help - -#### `cla-assistant` - -**Usage:** -```bash -Usage: cla-assistant -``` - -**Options:** - -- `--help`: Print help - -##### `retry` - -**Usage:** -```bash -Usage: retry [OPTIONS] --pr-number -``` - -**Options:** - -- `--pr-number `: Pull request number to check -- `--repo `: Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY -- `--help`: Print help - -##### `status` - -**Usage:** -```bash -Usage: status [OPTIONS] <--pr |--sha > -``` - -**Options:** - -- `--pr `: Pull request number whose head commit should be checked -- `--sha `: Commit SHA to check -- `--repo `: Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY -- `--help`: Print help - -##### `help` - -**Usage:** -```bash -Usage: help [COMMAND]... -``` - -**Options:** - -- `subcommand `: Print help for the subcommand(s) - -#### `help` - -**Usage:** -```bash -Usage: help [COMMAND]... -``` - -**Options:** - -- `subcommand `: Print help for the subcommand(s) - -### `help` - -**Usage:** -```bash -Usage: help [COMMAND]... -``` - -**Options:** - -- `subcommand `: Print help for the subcommand(s) - - ---- - -This document is auto-generated by running: - -```bash -cargo ci self-docs -``` \ No newline at end of file diff --git a/tools/ci/args/Cargo.toml b/tools/ci/args/Cargo.toml new file mode 100644 index 00000000000..61a7a9333c4 --- /dev/null +++ b/tools/ci/args/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "ci-args" +version = "0.1.0" +edition.workspace = true + +[dependencies] +clap.workspace = true diff --git a/tools/ci/args/src/lib.rs b/tools/ci/args/src/lib.rs new file mode 100644 index 00000000000..d0fd6af3d03 --- /dev/null +++ b/tools/ci/args/src/lib.rs @@ -0,0 +1,220 @@ +pub mod cla_assistant { + use clap::{ArgGroup, Args as ClapArgs, Subcommand}; + + #[derive(Subcommand)] + pub enum ClaAssistantCmd { + /// Retries CLA Assistant if `license/cla` is the only remaining PR blocker. + Retry(RetryArgs), + + /// Returns the `license/cla` status for a pull request or commit SHA. + Status(StatusArgs), + } + + #[derive(ClapArgs)] + pub struct RetryArgs { + /// Pull request number to check. + #[arg(long)] + pub pr_number: u64, + + /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. + #[arg(long)] + pub repo: Option, + } + + #[derive(ClapArgs)] + #[command(group( + ArgGroup::new("target") + .required(true) + .multiple(false) + .args(["pr", "sha"]), + ))] + pub struct StatusArgs { + /// Pull request number whose head commit should be checked. + #[arg(long)] + pub pr: Option, + + /// Commit SHA to check. + #[arg(long)] + pub sha: Option, + + /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. + #[arg(long)] + pub repo: Option, + } + + #[derive(ClapArgs)] + pub struct Args { + #[command(subcommand)] + pub cmd: ClaAssistantCmd, + } +} + +pub mod cli_docs { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args { + #[arg( + long, + long_help = "specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located)" + )] + pub spacetime_path: Option, + } +} + +pub mod codeowners_check { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args { + /// Git ref to compare against, usually origin/. + #[arg(long)] + pub base_ref: String, + + /// Pull request number to inspect for approval state. + #[arg(long)] + pub pr_number: u64, + } +} + +pub mod coordinate_internal_tests { + use clap::Args as ClapArgs; + + /// Selects or starts the private workflow for a public Internal Tests run. + #[derive(ClapArgs)] + pub struct Args { + /// Immutable public commit to test. + #[arg(long)] + pub public_sha: String, + + /// Public pull request number, when coordinating a pull request run. + #[arg(long)] + pub public_pr_number: Option, + } +} + +pub mod docs { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod global_json_policy { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod keynote_bench { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod lint { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod publish_checks { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod smoketests { + use clap::{Args as ClapArgs, Subcommand}; + + #[derive(ClapArgs)] + pub struct SmoketestsArgs { + #[command(subcommand)] + pub cmd: Option, + + /// Run tests against a remote server instead of spawning local servers. + /// + /// When specified, tests will connect to the given URL instead of starting + /// local server instances. Tests that require local server control (like + /// restart tests) will be skipped. + #[arg(long)] + pub server: Option, + + /// Use a SpacetimeAuth-issued login for remote-server tests. + /// + /// This is required for servers that reject direct server-issued logins for privileged operations. + /// + /// Optionally accepts an auth host to pass through to `spacetime login`, + /// for example `--auth-host=https://spacetimedb.com`. + #[arg(long, num_args = 0..=1, require_equals = true, default_missing_value = "")] + pub auth_host: Option, + + #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] + pub dotnet: bool, + + /// Additional arguments to pass to the test runner + #[arg(trailing_var_arg = true)] + pub args: Vec, + } + + #[derive(Subcommand)] + pub enum SmoketestCmd { + /// Only build binaries without running tests + /// + /// Use this before running `cargo test --all` to ensure binaries are built. + Prepare, + CheckModList, + } +} + +pub mod test { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod typescript_test { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod update_flow { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args { + #[arg( + long, + long_help = "Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms." + )] + pub target: Option, + + #[arg( + long, + default_value = "false", + long_help = "Whether to enable github token authentication feature when building the update binary. By default this is disabled." + )] + pub github_token_auth: bool, + } +} + +pub mod version_upgrade_check { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} + +pub mod wasm_bindings { + use clap::Args as ClapArgs; + + #[derive(ClapArgs)] + pub struct Args {} +} diff --git a/tools/ci/commands/cla-assistant/Cargo.toml b/tools/ci/commands/cla-assistant/Cargo.toml new file mode 100644 index 00000000000..eef51dc60ba --- /dev/null +++ b/tools/ci/commands/cla-assistant/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ci-cla-assistant" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +ci-args = { path = "../../args" } +reqwest = { workspace = true, features = ["blocking", "json"] } +serde.workspace = true +serde_json.workspace = true diff --git a/tools/ci/src/cla_assistant.rs b/tools/ci/commands/cla-assistant/src/main.rs similarity index 84% rename from tools/ci/src/cla_assistant.rs rename to tools/ci/commands/cla-assistant/src/main.rs index 065db03ef70..dcb8d7c6e8d 100644 --- a/tools/ci/src/cla_assistant.rs +++ b/tools/ci/commands/cla-assistant/src/main.rs @@ -1,8 +1,11 @@ +#![allow(clippy::disallowed_macros)] + use std::collections::BTreeMap; use std::env; use anyhow::{anyhow, bail, Context, Result}; -use clap::{ArgGroup, Args, Subcommand}; +use ci_args::cla_assistant::{Args, ClaAssistantCmd, RetryArgs, StatusArgs}; +use clap::Parser; use reqwest::blocking::Client; use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, AUTHORIZATION, USER_AGENT}; use serde::de::DeserializeOwned; @@ -10,49 +13,15 @@ use serde::{Deserialize, Serialize}; const CLA_CONTEXT: &str = "license/cla"; -#[derive(Subcommand)] -pub(crate) enum ClaAssistantCmd { - /// Retries CLA Assistant if `license/cla` is the only remaining PR blocker. - Retry(RetryArgs), - - /// Returns the `license/cla` status for a pull request or commit SHA. - Status(StatusArgs), -} - -#[derive(Args)] -pub(crate) struct RetryArgs { - /// Pull request number to check. - #[arg(long)] - pub(crate) pr_number: u64, - - /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. - #[arg(long)] - pub(crate) repo: Option, -} - -#[derive(Args)] -#[command(group( - ArgGroup::new("target") - .required(true) - .multiple(false) - .args(["pr", "sha"]), -))] -pub(crate) struct StatusArgs { - /// Pull request number whose head commit should be checked. - #[arg(long)] - pub(crate) pr: Option, - - /// Commit SHA to check. - #[arg(long)] - pub(crate) sha: Option, - - /// Repository in `owner/name` form. Defaults to GITHUB_REPOSITORY. - #[arg(long)] - pub(crate) repo: Option, +#[derive(Parser)] +#[command(about = "Interacts with CLA Assistant.")] +struct Cli { + #[command(flatten)] + args: Args, } -pub(crate) fn run(cmd: ClaAssistantCmd) -> Result<()> { - match cmd { +fn main() -> Result<()> { + match Cli::parse().args.cmd { ClaAssistantCmd::Retry(args) => retry(args), ClaAssistantCmd::Status(args) => status(args), } diff --git a/tools/ci/commands/cli-docs/Cargo.toml b/tools/ci/commands/cli-docs/Cargo.toml new file mode 100644 index 00000000000..07d7ff20840 --- /dev/null +++ b/tools/ci/commands/cli-docs/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ci-cli-docs" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +clap.workspace = true +ci-args = { path = "../../args" } +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/cli-docs/src/main.rs b/tools/ci/commands/cli-docs/src/main.rs new file mode 100644 index 00000000000..9829bbfbef3 --- /dev/null +++ b/tools/ci/commands/cli-docs/src/main.rs @@ -0,0 +1,37 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::Result; +use ci_args::cli_docs::Args; +use ci_common::pnpm; +use clap::Parser; +use duct::cmd; +use std::env; + +#[derive(Parser)] +#[command(about = "Generates CLI documentation and checks for changes")] +struct Cli { + #[command(flatten)] + args: Args, +} + +fn main() -> Result<()> { + let Args { spacetime_path } = Cli::parse().args; + if let Some(path) = spacetime_path { + env::set_current_dir(path).ok(); + } + let current_dir = env::current_dir().expect("No current directory!"); + let dir_name = current_dir.file_name().expect("No current directory!"); + if dir_name != "SpacetimeDB" && dir_name != "public" { + anyhow::bail!("You must execute this binary from inside of the SpacetimeDB directory, or use --spacetime-path"); + } + + pnpm(["install", "--recursive"]).run()?; + pnpm(["generate-cli-docs"]).dir("docs").run()?; + let out = cmd!("git", "status", "--porcelain", "--", "docs").read()?; + if out.is_empty() { + println!("No docs changes detected"); + } else { + anyhow::bail!("CLI docs are out of date:\n{out}"); + } + + Ok(()) +} diff --git a/tools/ci/commands/codeowners-check/Cargo.toml b/tools/ci/commands/codeowners-check/Cargo.toml new file mode 100644 index 00000000000..c553c6499b1 --- /dev/null +++ b/tools/ci/commands/codeowners-check/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "ci-codeowners-check" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +ci-args = { path = "../../args" } +ci-common = { path = "../../common" } +duct.workspace = true +regex.workspace = true +serde_json.workspace = true diff --git a/tools/ci/src/codeowners_check.rs b/tools/ci/commands/codeowners-check/src/main.rs similarity index 94% rename from tools/ci/src/codeowners_check.rs rename to tools/ci/commands/codeowners-check/src/main.rs index 006d597cf87..07866923242 100644 --- a/tools/ci/src/codeowners_check.rs +++ b/tools/ci/commands/codeowners-check/src/main.rs @@ -1,4 +1,7 @@ use anyhow::{anyhow, bail, Context, Result}; +use ci_args::codeowners_check::Args; +use ci_common::ensure_repo_root; +use clap::Parser; use duct::cmd; use serde_json::Value; use std::collections::HashMap; @@ -6,8 +9,15 @@ use std::path::{Path, PathBuf}; const REPO: &str = "clockworklabs/SpacetimeDB"; -pub fn run(base_ref: &str, pr_number: u64) -> Result<()> { - super::ensure_repo_root()?; +#[derive(Parser)] +#[command(about = "Checks that sensitive CODEOWNERS-controlled files have the required approvals.")] +struct Cli { + #[command(flatten)] + args: Args, +} + +fn run(base_ref: &str, pr_number: u64) -> Result<()> { + ensure_repo_root()?; fetch_base_ref(base_ref)?; let review = ReviewStatus::fetch(pr_number)?; @@ -20,6 +30,11 @@ pub fn run(base_ref: &str, pr_number: u64) -> Result<()> { Ok(()) } +fn main() -> Result<()> { + let args = Cli::parse().args; + run(&args.base_ref, args.pr_number) +} + fn file_review_requirements(base_ref: &str, path: &Path, review: &ReviewStatus) -> Result<()> { if license::is_license(path) && !license::is_trivial_change(base_ref, path)? { review.require("cloutiertyler")?; diff --git a/tools/ci/commands/coordinate-internal-tests/Cargo.toml b/tools/ci/commands/coordinate-internal-tests/Cargo.toml new file mode 100644 index 00000000000..da01e6cad03 --- /dev/null +++ b/tools/ci/commands/coordinate-internal-tests/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ci-coordinate-internal-tests" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +ci-args = { path = "../../args" } +duct.workspace = true +serde.workspace = true +serde_json.workspace = true diff --git a/tools/ci/src/internal_tests.rs b/tools/ci/commands/coordinate-internal-tests/src/main.rs similarity index 97% rename from tools/ci/src/internal_tests.rs rename to tools/ci/commands/coordinate-internal-tests/src/main.rs index 78d7f7cfb66..d913cf94fed 100644 --- a/tools/ci/src/internal_tests.rs +++ b/tools/ci/commands/coordinate-internal-tests/src/main.rs @@ -1,5 +1,8 @@ +#![allow(clippy::disallowed_macros)] + use anyhow::{bail, ensure, Context, Result}; -use clap::Args; +use ci_args::coordinate_internal_tests::Args; +use clap::Parser; use duct::{cmd, Expression}; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -14,16 +17,11 @@ const PRIVATE_REPO: &str = "clockworklabs/SpacetimeDBPrivate"; const PRIVATE_WORKFLOW: &str = "ci.yml"; const PRIVATE_DEFAULT_BRANCH: &str = "master"; -#[derive(Args)] -/// Selects or starts the private workflow for a public Internal Tests run. -pub(crate) struct CoordinateArgs { - /// Immutable public commit to test. - #[arg(long)] - public_sha: String, - - /// Public pull request number, when coordinating a pull request run. - #[arg(long)] - public_pr_number: Option, +#[derive(Parser)] +#[command(about = "Selects or starts the private workflow for a public Internal Tests run.")] +struct Cli { + #[command(flatten)] + args: Args, } #[derive(Debug)] @@ -451,7 +449,8 @@ fn write_github_output(name: &str, value: impl std::fmt::Display) -> Result<()> } /// Coordinates the public Internal Tests run without checking out or executing private code. -pub(crate) fn coordinate(args: CoordinateArgs) -> Result<()> { +fn main() -> Result<()> { + let args = Cli::parse().args; let private_source = resolve_private_source(args.public_pr_number)?; let coordinated = match private_source { diff --git a/tools/ci/commands/docs/Cargo.toml b/tools/ci/commands/docs/Cargo.toml new file mode 100644 index 00000000000..411892ec988 --- /dev/null +++ b/tools/ci/commands/docs/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "ci-docs-build" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +ci-args = { path = "../../args" } +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/docs/src/main.rs b/tools/ci/commands/docs/src/main.rs new file mode 100644 index 00000000000..fd8e305ccce --- /dev/null +++ b/tools/ci/commands/docs/src/main.rs @@ -0,0 +1,20 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::Result; +use ci_args::docs::Args; +use ci_common::pnpm; +use clap::Parser; + +/// Builds the docs site. +#[derive(Parser)] +struct Cli { + #[command(flatten)] + args: Args, +} + +fn main() -> Result<()> { + Cli::parse(); + + pnpm(["install"]).dir("docs").run()?; + pnpm(["build"]).dir("docs").run()?; + Ok(()) +} diff --git a/tools/ci/commands/global-json-policy/Cargo.toml b/tools/ci/commands/global-json-policy/Cargo.toml new file mode 100644 index 00000000000..644ef6df18c --- /dev/null +++ b/tools/ci/commands/global-json-policy/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ci-global-json-policy" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +ci-args = { path = "../../args" } +duct.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/global-json-policy/src/main.rs b/tools/ci/commands/global-json-policy/src/main.rs new file mode 100644 index 00000000000..b2e680b4055 --- /dev/null +++ b/tools/ci/commands/global-json-policy/src/main.rs @@ -0,0 +1,60 @@ +#![allow(clippy::disallowed_macros)] + +use anyhow::{bail, Result}; +use ci_args::global_json_policy::Args; +use ci_common::ensure_repo_root; +use clap::Parser; +use duct::cmd; +use std::fs; +use std::path::{Path, PathBuf}; + +/// Verify that any non-root global.json files are symlinks to the root global.json. +#[derive(Parser)] +struct Cli { + #[command(flatten)] + args: Args, +} + +fn git_tracked_files(pathspec: &str) -> Result> { + let output = cmd!("git", "ls-files", pathspec).read()?; + Ok(output.lines().map(PathBuf::from).collect()) +} + +fn main() -> Result<()> { + Cli::parse(); + + ensure_repo_root()?; + + let root_json = Path::new("global.json"); + let root_contents = fs::read_to_string(root_json)?; + + let globals = git_tracked_files(":(glob)**/global.json")?; + + let mut ok = true; + for p in globals { + let meta = fs::symlink_metadata(&p)?; + let is_symlink = meta.file_type().is_symlink(); + let is_template_global_json = p.strip_prefix(".").unwrap_or(&p).starts_with(Path::new("templates")); + if is_template_global_json && is_symlink { + eprintln!( + "Error: {} is a symlink. Template files must not be symlinks; they are copied literally and this will break if the CLI is built under Windows where symlinks are not supported.", + p.display() + ); + ok = false; + } + + let contents = fs::read_to_string(&p)?; + if contents != root_contents { + eprintln!("Error: {} does not match the root global.json contents", p.display()); + ok = false; + } else if !is_template_global_json || !is_symlink { + println!("OK: {}", p.display()); + } + } + + if !ok { + bail!("global.json policy check failed"); + } + + Ok(()) +} diff --git a/tools/ci/commands/keynote-bench/Cargo.toml b/tools/ci/commands/keynote-bench/Cargo.toml new file mode 100644 index 00000000000..508ee26eebf --- /dev/null +++ b/tools/ci/commands/keynote-bench/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ci-keynote-bench" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +ci-args = { path = "../../args" } +keynote-bench-harness = { path = "../../../keynote-bench-harness" } +spacetimedb-guard.workspace = true diff --git a/tools/ci/commands/keynote-bench/src/main.rs b/tools/ci/commands/keynote-bench/src/main.rs new file mode 100644 index 00000000000..cc97ae8cef9 --- /dev/null +++ b/tools/ci/commands/keynote-bench/src/main.rs @@ -0,0 +1,26 @@ +use anyhow::Result; +use ci_args::keynote_bench::Args; +use clap::Parser; +use keynote_bench_harness::KeynoteBenchConfig; +use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; + +/// Runs the keynote benchmark as a CI performance regression gate. +/// +/// Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the +/// keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and +/// fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. +#[derive(Parser)] +struct Cli { + #[command(flatten)] + args: Args, +} + +fn main() -> Result<()> { + Cli::parse(); + + let cli_path = ensure_binaries_built(); + let server = SpacetimeDbGuard::spawn_in_temp_data_dir(); + let server_url = server.host_url.clone(); + + keynote_bench_harness::run(KeynoteBenchConfig::standalone(".", cli_path, server_url)) +} diff --git a/tools/ci/commands/lint/Cargo.toml b/tools/ci/commands/lint/Cargo.toml new file mode 100644 index 00000000000..67eef8b8b9d --- /dev/null +++ b/tools/ci/commands/lint/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ci-lint" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +ci-args = { path = "../../args" } +duct.workspace = true +serde_json.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/lint/src/main.rs b/tools/ci/commands/lint/src/main.rs new file mode 100644 index 00000000000..ddf6365fd8b --- /dev/null +++ b/tools/ci/commands/lint/src/main.rs @@ -0,0 +1,359 @@ +#![allow(clippy::disallowed_macros)] + +use anyhow::{bail, Context, Result}; +use ci_args::lint::Args; +use ci_common::{ensure_repo_root, pnpm}; +use clap::Parser; +use duct::cmd; +use serde_json::Value; +use std::collections::BTreeSet; +use std::ffi::OsString; +use std::fs; +use std::path::Path; +use std::path::PathBuf; + +/// Lints the codebase +/// +/// Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there +/// are no warnings. +#[derive(Parser)] +struct Cli { + #[command(flatten)] + args: Args, +} + +fn package_json_pnpm_version(package_manager: &str) -> Option<&str> { + package_manager.strip_prefix("pnpm@") +} + +fn git_tracked_files(pathspec: &str) -> Result> { + let output = cmd!("git", "ls-files", pathspec).read()?; + Ok(output.lines().map(PathBuf::from).collect()) +} + +fn package_json_string_value(package_json: &Value, key: &str) -> Option { + package_json.get(key)?.as_str().map(str::to_owned) +} + +fn package_json_engines_pnpm(package_json: &Value) -> Option { + package_json.get("engines")?.get("pnpm")?.as_str().map(str::to_owned) +} + +fn read_package_json(path: &Path) -> Result { + let contents = fs::read_to_string(path)?; + Ok(serde_json::from_str(&contents)?) +} + +fn is_npm_package_json(package_json: &Value) -> bool { + [ + "bin", + "dependencies", + "devDependencies", + "exports", + "main", + "optionalDependencies", + "packageManager", + "peerDependencies", + "scripts", + "type", + ] + .iter() + .any(|key| package_json.get(key).is_some()) +} + +fn is_template_path(path: &Path) -> bool { + path.starts_with("templates") +} + +fn minimum_release_age(path: &Path) -> Result { + let workspace = fs::read_to_string(path)?; + workspace + .lines() + .find_map(|line| { + let line = line.trim(); + let value = line.strip_prefix("minimumReleaseAge:")?.trim(); + value.parse::().ok() + }) + .ok_or_else(|| anyhow::anyhow!("{} is missing minimumReleaseAge", path.display())) +} + +fn npmrc_minimum_release_age(path: &Path, expected_minimum_release_age: u64) -> Result { + let contents = fs::read_to_string(path).map_err(|err| { + if err.kind() == std::io::ErrorKind::NotFound { + anyhow::anyhow!( + "{} is tracked but missing from the working tree. Restore it with:\nminimum-release-age={}", + path.display(), + expected_minimum_release_age + ) + } else { + anyhow::anyhow!( + "failed to read {} while checking pnpm minimum package age: {err}", + path.display() + ) + } + })?; + contents + .lines() + .find_map(|line| { + let line = line.trim(); + let value = line.strip_prefix("minimum-release-age=")?.trim(); + value.parse::().ok() + }) + .ok_or_else(|| { + anyhow::anyhow!( + "{} must contain `minimum-release-age={}` to match root pnpm-workspace.yaml", + path.display(), + expected_minimum_release_age + ) + }) +} + +fn check_pnpm_release_age_policy() -> Result<()> { + ensure_repo_root()?; + + let root_package_json_path = Path::new("package.json"); + let root_package_json = read_package_json(root_package_json_path)?; + let package_manager = package_json_string_value(&root_package_json, "packageManager") + .ok_or_else(|| anyhow::anyhow!("package.json is missing packageManager"))?; + let package_manager_version = package_json_pnpm_version(&package_manager) + .ok_or_else(|| anyhow::anyhow!("packageManager must be pnpm@, found {package_manager:?}"))?; + + let expected_engine_pnpm = format!(">={package_manager_version}"); + let engine_pnpm = package_json_engines_pnpm(&root_package_json) + .ok_or_else(|| anyhow::anyhow!("package.json engines is missing pnpm"))?; + if engine_pnpm != expected_engine_pnpm { + bail!("package.json engines.pnpm must be {expected_engine_pnpm:?}, found {engine_pnpm:?}"); + } + + for package_json_path in git_tracked_files(":(glob)**/package.json")? { + let package_json = read_package_json(&package_json_path)?; + let Some(found_package_manager) = package_json_string_value(&package_json, "packageManager") else { + continue; + }; + if found_package_manager != package_manager { + bail!( + "{} packageManager must match root package.json: expected {:?}, found {:?}", + package_json_path.display(), + package_manager, + found_package_manager + ); + } + } + + let root_workspace_path = Path::new("pnpm-workspace.yaml"); + let root_minimum_release_age = minimum_release_age(root_workspace_path)?; + for workspace_path in git_tracked_files(":(glob)**/pnpm-workspace.yaml")? { + let found_minimum_release_age = minimum_release_age(&workspace_path)?; + if found_minimum_release_age != root_minimum_release_age { + bail!( + "{} minimumReleaseAge must match root pnpm-workspace.yaml: expected {}, found {}", + workspace_path.display(), + root_minimum_release_age, + found_minimum_release_age + ); + } + } + + for npmrc_path in git_tracked_files(":(glob)**/.npmrc")? { + // Template package roots are copied into projects created by `spacetime init`. + // They must not embed this repo's package-age policy; smoketests enforce it + // at the pnpm process boundary instead. + if is_template_path(&npmrc_path) { + continue; + } + let found_minimum_release_age = npmrc_minimum_release_age(&npmrc_path, root_minimum_release_age)?; + if found_minimum_release_age != root_minimum_release_age { + bail!( + "{} minimum-release-age must match root pnpm-workspace.yaml: expected {}, found {}", + npmrc_path.display(), + root_minimum_release_age, + found_minimum_release_age + ); + } + } + + for package_json_path in git_tracked_files(":(glob)**/package.json")? { + // Template package roots are copied into projects created by `spacetime init`. + // They must not require adjacent .npmrc files for this repo's package-age + // policy; smoketests enforce it at the pnpm process boundary instead. + if is_template_path(&package_json_path) { + continue; + } + let package_json = read_package_json(&package_json_path)?; + if !is_npm_package_json(&package_json) { + continue; + } + let package_dir = package_json_path + .parent() + .expect("git-tracked package.json path should have a parent"); + let npmrc_path = package_dir.join(".npmrc"); + if !npmrc_path.is_file() { + bail!( + "{} is required because {} is an npm/pnpm package manifest.\nAdd {} containing:\nminimum-release-age={}", + npmrc_path.display(), + package_json_path.display(), + npmrc_path.display(), + root_minimum_release_age + ); + } + let found_minimum_release_age = npmrc_minimum_release_age(&npmrc_path, root_minimum_release_age)?; + if found_minimum_release_age != root_minimum_release_age { + bail!( + "{} minimum-release-age must match root pnpm-workspace.yaml: expected {}, found {}", + npmrc_path.display(), + root_minimum_release_age, + found_minimum_release_age + ); + } + } + + for workflow_path in git_tracked_files(".github/workflows/*")? { + let contents = fs::read_to_string(&workflow_path)?; + if contents.contains("pnpm/action-setup@v4") { + bail!( + "{} must use ./.github/actions/setup-pnpm instead of pnpm/action-setup@v4", + workflow_path.display() + ); + } + } + + Ok(()) +} + +/// Codex plugin ships a copy of `skills/`, because plugin installers do not follow symlinks, +/// this checks if the copy is in sync +fn check_codex_plugin_skills_sync() -> Result<()> { + let source = Path::new("skills"); + let copy = Path::new("codex-plugin/plugins/spacetimedb/skills"); + + if fs::symlink_metadata(copy) + .with_context(|| format!("reading {}", copy.display()))? + .file_type() + .is_symlink() + { + bail!( + "{} must be a real directory, not a symlink, plugin installers do not follow symlinks", + copy.display() + ); + } + + fn walk(root: &Path, dir: &Path, out: &mut BTreeSet) -> Result<()> { + for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { + let entry = entry?; + if entry.file_type()?.is_dir() { + walk(root, &entry.path(), out)?; + } else { + let rel = entry + .path() + .strip_prefix(root) + .expect("walked path should be under its root") + .to_path_buf(); + out.insert(rel); + } + } + Ok(()) + } + + let mut source_files = BTreeSet::new(); + walk(source, source, &mut source_files)?; + let mut copy_files = BTreeSet::new(); + walk(copy, copy, &mut copy_files)?; + + let mut drift = Vec::new(); + for rel in &source_files { + if !copy_files.contains(rel) { + drift.push(format!("missing from copy: {}", rel.display())); + } else if fs::read(source.join(rel))? != fs::read(copy.join(rel))? { + drift.push(format!("differs: {}", rel.display())); + } + } + for rel in ©_files { + if !source_files.contains(rel) { + drift.push(format!("extraneous in copy: {}", rel.display())); + } + } + + if !drift.is_empty() { + bail!( + "codex-plugin skills copy is out of sync with skills/:\n {}\nRun: node codex-plugin/scripts/check-skills-sync.ts --fix", + drift.join("\n ") + ); + } + Ok(()) +} + +fn tracked_rs_files_under(path: &str) -> Result> { + let output = cmd!("git", "ls-files", "--", path).read()?; + Ok(output + .lines() + .filter(|line| line.ends_with(".rs")) + .map(PathBuf::from) + .collect()) +} + +fn main() -> Result<()> { + Cli::parse(); + + ensure_repo_root()?; + check_pnpm_release_age_policy()?; + check_codex_plugin_skills_sync()?; + // `cargo fmt --all` only checks files that Cargo discovers through workspace/package targets. + // However, we also keep Rust sources in a locations that are tracked but not part of our workspace, + // so this approach properly catches all the files, where `cargo fmt` does not. + let mut files = Vec::new(); + files.extend(tracked_rs_files_under(".")?); + const RUSTFMT_BATCH_SIZE: usize = 200; + for batch in files.chunks(RUSTFMT_BATCH_SIZE) { + let mut args = Vec::::with_capacity(batch.len() + 1); + args.push("--check".into()); + args.extend(batch.iter().map(|path| path.as_os_str().to_os_string())); + cmd("rustfmt", args).run()?; + } + cmd!( + "cargo", + "clippy", + "--timings", + "--all", + "--tests", + "--benches", + "--", + "-D", + "warnings", + ) + .run()?; + cmd!( + "cargo", + "clippy", + "--timings", + "--no-default-features", + "--features=browser", + "-pspacetimedb-sdk", + "--tests", + "--benches", + "--", + "-D", + "warnings", + ) + .run()?; + cmd!("dotnet", "tool", "restore").dir("crates/bindings-csharp").run()?; + cmd!("dotnet", "csharpier", "--check", ".") + .dir("crates/bindings-csharp") + .run()?; + pnpm(["lint"]).run()?; + cmd!("cargo", "test", "--doc", "--target", "wasm32-unknown-unknown") + .dir("crates/bindings") + .run()?; + cmd!("cargo", "test", "--doc").dir("crates/bindings").run()?; + // `bindings` is the only crate we care strongly about documenting, + // since we link to its docs.rs from our website. + // We won't pass `--no-deps`, though, + // since we want everything reachable through it to also work. + // This includes `sats` and `lib`. + cmd!("cargo", "doc") + .dir("crates/bindings") + // Make `cargo doc` exit with error on warnings, most notably broken links + .env("RUSTDOCFLAGS", "--deny warnings") + .run()?; + + Ok(()) +} diff --git a/tools/ci/commands/publish-checks/Cargo.toml b/tools/ci/commands/publish-checks/Cargo.toml new file mode 100644 index 00000000000..be73f4263d5 --- /dev/null +++ b/tools/ci/commands/publish-checks/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "ci-publish-checks" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +ci-args = { path = "../../args" } +duct.workspace = true diff --git a/tools/ci/commands/publish-checks/src/main.rs b/tools/ci/commands/publish-checks/src/main.rs new file mode 100644 index 00000000000..3e2b7e71ccd --- /dev/null +++ b/tools/ci/commands/publish-checks/src/main.rs @@ -0,0 +1,44 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::{bail, Result}; +use ci_args::publish_checks::Args; +use clap::Parser; +use duct::cmd; + +/// Checks that publishable crates satisfy publish constraints. +#[derive(Parser)] +struct Cli { + #[command(flatten)] + args: Args, +} + +fn main() -> Result<()> { + Cli::parse(); + + cmd!("bash", "-lc", "test -d venv || python3 -m venv venv").run()?; + cmd!("venv/bin/pip3", "install", "argparse", "toml").run()?; + + let crates = cmd!( + "venv/bin/python3", + "tools/find-publish-list.py", + "--recursive", + "--directories", + "--quiet", + "spacetimedb", + "spacetimedb-sdk" + ) + .read()?; + + let mut failed = Vec::new(); + for crate_dir in crates.split_whitespace() { + if let Err(err) = cmd!("venv/bin/python3", "tools/crate-publish-checks.py", crate_dir).run() { + eprintln!("crate publish checks failed for {crate_dir}: {err}"); + failed.push(crate_dir.to_string()); + } + } + + if !failed.is_empty() { + bail!("crate publish checks failed for: {}", failed.join(", ")); + } + + Ok(()) +} diff --git a/tools/ci/commands/smoketests/Cargo.toml b/tools/ci/commands/smoketests/Cargo.toml new file mode 100644 index 00000000000..42ec134b129 --- /dev/null +++ b/tools/ci/commands/smoketests/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "ci-smoketests" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +clap.workspace = true +ci-args = { path = "../../args" } +tempfile.workspace = true +spacetimedb-guard.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/src/smoketest.rs b/tools/ci/commands/smoketests/src/main.rs similarity index 88% rename from tools/ci/src/smoketest.rs rename to tools/ci/commands/smoketests/src/main.rs index 2a35b91ca59..2342d35d98a 100644 --- a/tools/ci/src/smoketest.rs +++ b/tools/ci/commands/smoketests/src/main.rs @@ -1,6 +1,8 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, ensure, Context, Result}; -use clap::{Args, Subcommand}; +use ci_args::smoketests::{SmoketestCmd, SmoketestsArgs}; +use ci_common::ensure_repo_root; +use clap::Parser; use duct::cmd; use spacetimedb_guard::ensure_binaries_built; use std::ffi::OsStr; @@ -9,51 +11,21 @@ use std::process::{Command, Stdio}; use std::{env, fs}; use tempfile::TempDir; -use crate::util; - -#[derive(Args)] +#[derive(Parser)] +#[command( + about = "Runs smoketests", + long_about = "Runs smoketests\n\nExecutes the smoketests suite with some default exclusions." +)] /// This command builds the binaries needed by the smoketests, then runs them. This prevents /// race conditions when running tests in parallel with nextest, where multiple test processes /// might try to build the same binaries simultaneously. -pub struct SmoketestsArgs { - #[command(subcommand)] - cmd: Option, - - /// Run tests against a remote server instead of spawning local servers. - /// - /// When specified, tests will connect to the given URL instead of starting - /// local server instances. Tests that require local server control (like - /// restart tests) will be skipped. - #[arg(long)] - server: Option, - - /// Use a SpacetimeAuth-issued login for remote-server tests. - /// - /// This is required for servers that reject direct server-issued logins for privileged operations. - /// - /// Optionally accepts an auth host to pass through to `spacetime login`, - /// for example `--auth-host=https://spacetimedb.com`. - #[arg(long, num_args = 0..=1, require_equals = true, default_missing_value = "")] - auth_host: Option, - - #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] - dotnet: bool, - - /// Additional arguments to pass to the test runner - #[arg(trailing_var_arg = true)] - args: Vec, -} - -#[derive(Subcommand)] -enum SmoketestCmd { - /// Only build binaries without running tests - /// - /// Use this before running `cargo test --all` to ensure binaries are built. - Prepare, - CheckModList, +struct Cli { + #[command(flatten)] + args: SmoketestsArgs, } -pub fn run(args: SmoketestsArgs) -> Result<()> { +fn main() -> Result<()> { + let args = Cli::parse().args; match args.cmd { Some(SmoketestCmd::Prepare) => { build_cli()?; @@ -292,7 +264,7 @@ fn set_env(cmd: &mut Command, server: Option, dotnet: bool, auth_host: b } fn check_smoketests_mod_rs_complete() -> Result<()> { - util::ensure_repo_root()?; + ensure_repo_root()?; let expected_dir = Path::new("crates/smoketests/tests/smoketests"); let mut expected = std::collections::BTreeSet::::new(); diff --git a/tools/ci/commands/test/Cargo.toml b/tools/ci/commands/test/Cargo.toml new file mode 100644 index 00000000000..040ad9fa502 --- /dev/null +++ b/tools/ci/commands/test/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ci-test" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +ci-args = { path = "../../args" } +duct.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/test/src/main.rs b/tools/ci/commands/test/src/main.rs new file mode 100644 index 00000000000..b9f8e72c912 --- /dev/null +++ b/tools/ci/commands/test/src/main.rs @@ -0,0 +1,132 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::Result; +use ci_args::test::Args; +use ci_common::pnpm; +use clap::Parser; +use duct::cmd; + +/// Runs tests +/// +/// Runs rust tests, codegens csharp sdk and runs csharp tests. +/// This does not include Unreal tests. +/// This expects to run in a clean git state. +#[derive(Parser)] +struct Cli { + #[command(flatten)] + args: Args, +} + +fn main() -> Result<()> { + Cli::parse(); + + pnpm(["build"]).dir("crates/bindings-typescript").run()?; + + // TODO: This doesn't work on at least user Linux machines, because something here apparently uses `sudo`? + + // Exclude smoketests from `cargo test --all` since they require pre-built binaries. + // Smoketests have their own dedicated command: `cargo ci smoketests` + cmd!( + "cargo", + "test", + "--all", + "--exclude", + "spacetimedb-smoketests", + "--exclude", + "spacetimedb-sdk", + "--exclude", + "spacetimedb", + "--", + "--test-threads=2", + "--skip", + "unreal" + ) + .run()?; + // Bindings snapshot tests rely on the unstable feature, + // as they compile and test APIs which are gated behind that feature, + // e.g. procedures, HTTP handlers. + cmd!( + "cargo", + "test", + "-p", + "spacetimedb", + "--features", + "unstable", + "--", + "--test-threads=2", + ) + .run()?; + // The SDK test harness uses the same child-process server guard as smoketests, + // which expects release CLI/standalone binaries to already exist. + cmd!( + "cargo", + "build", + "--release", + "-p", + "spacetimedb-cli", + "-p", + "spacetimedb-standalone", + "--features", + "spacetimedb-standalone/allow_loopback_http_for_tests", + ) + .run()?; + // SDK procedure tests intentionally make localhost HTTP requests. + cmd!( + "cargo", + "test", + "-p", + "spacetimedb-sdk", + "--features", + "allow_loopback_http_for_tests", + "--", + "--test-threads=2", + "--skip", + "unreal" + ) + .run()?; + // Run the same SDK suite against wasm/browser test clients. + cmd!( + "cargo", + "test", + "-p", + "spacetimedb-sdk", + "--features", + "allow_loopback_http_for_tests,browser", + "--", + "--test-threads=2", + "--skip", + "unreal" + ) + .run()?; + // TODO: This should check for a diff at the start. If there is one, we should alert the user + // that we're disabling diff checks because they have a dirty git repo, and to re-run in a clean one + // if they want those checks. + + // The fallocate tests have been flakely when running in parallel + cmd!( + "cargo", + "test", + "-p", + "spacetimedb-durability", + "--features", + "fallocate", + "--", + "--test-threads=1", + ) + .run()?; + cmd!("bash", "tools/check-diff.sh").run()?; + cmd!( + "cargo", + "run", + "-p", + "spacetimedb-codegen", + "--example", + "regen-csharp-moduledef", + ) + .run()?; + cmd!("bash", "tools/check-diff.sh", "crates/bindings-csharp").run()?; + cmd!("dotnet", "test", "-warnaserror") + .dir("crates/bindings-csharp") + .run()?; + + Ok(()) +} diff --git a/tools/ci/commands/typescript-test/Cargo.toml b/tools/ci/commands/typescript-test/Cargo.toml new file mode 100644 index 00000000000..e7ca40b5200 --- /dev/null +++ b/tools/ci/commands/typescript-test/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ci-typescript-test" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +ci-args = { path = "../../args" } +duct.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/typescript-test/src/main.rs b/tools/ci/commands/typescript-test/src/main.rs new file mode 100644 index 00000000000..f0a783a675c --- /dev/null +++ b/tools/ci/commands/typescript-test/src/main.rs @@ -0,0 +1,38 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::{bail, Result}; +use ci_args::typescript_test::Args; +use ci_common::pnpm; +use clap::Parser; +use duct::cmd; + +/// Runs TypeScript workspace tests and template build checks. +#[derive(Parser)] +struct Cli { + #[command(flatten)] + args: Args, +} + +fn main() -> Result<()> { + Cli::parse(); + + pnpm(["build"]).dir("crates/bindings-typescript").run()?; + pnpm(["test"]).dir("crates/bindings-typescript").run()?; + pnpm(["generate"]).dir("templates/chat-react-ts").run()?; + let diff_status = cmd!( + "bash", + "tools/check-diff.sh", + "templates/chat-react-ts/src/module_bindings" + ) + .run()?; + if !diff_status.status.success() { + bail!("Bindings are dirty. Please generate bindings again and commit them to this branch."); + } + pnpm(["build"]).dir("templates/chat-react-ts").run()?; + pnpm(["-r", "--filter", "./**", "run", "build"]) + .dir("templates") + .run()?; + pnpm(["-r", "--filter", "./**", "run", "build"]) + .dir("crates/bindings-typescript") + .run()?; + Ok(()) +} diff --git a/tools/ci/commands/update-flow/Cargo.toml b/tools/ci/commands/update-flow/Cargo.toml new file mode 100644 index 00000000000..15a89ac5f6f --- /dev/null +++ b/tools/ci/commands/update-flow/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ci-update-flow" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true +tempfile.workspace = true +clap.workspace = true +ci-args = { path = "../../args" } diff --git a/tools/ci/commands/update-flow/src/main.rs b/tools/ci/commands/update-flow/src/main.rs new file mode 100644 index 00000000000..8f0fe9d88c3 --- /dev/null +++ b/tools/ci/commands/update-flow/src/main.rs @@ -0,0 +1,61 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::Result; +use ci_args::update_flow::Args; +use clap::Parser; +use duct::cmd; + +#[derive(Parser)] +#[command( + about = "Tests the update flow", + long_about = "Tests the update flow\n\nTests the self-update flow by building the spacetimedb-update binary for the specified target, by default the current target, and performing a self-install into a temporary directory." +)] +struct Cli { + #[command(flatten)] + args: Args, +} + +fn main() -> Result<()> { + let Args { + target, + github_token_auth, + } = Cli::parse().args; + let mut common_args = vec![]; + if let Some(target) = target.as_ref() { + common_args.push("--target"); + common_args.push(target); + } + if github_token_auth { + common_args.push("--features"); + common_args.push("github-token-auth"); + } + + cmd( + "cargo", + ["build", "-p", "spacetimedb-update"] + .into_iter() + .chain(common_args.clone()), + ) + .run()?; + // NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`. + // My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it + // happens very frequently on the `macos-runner`, but we haven't seen it on any others). + let root_dir = tempfile::tempdir()?; + let root_dir_string = root_dir.path().to_string_lossy().to_string(); + let root_arg = format!("--root-dir={}", root_dir_string); + cmd( + "cargo", + ["run", "-p", "spacetimedb-update"] + .into_iter() + .chain(common_args.clone()) + .chain(["--", "self-install", &root_arg, "--yes"].into_iter()), + ) + .run()?; + + let mut spacetime_path = root_dir.path().join("spacetime"); + if !std::env::consts::EXE_EXTENSION.is_empty() { + spacetime_path.set_extension(std::env::consts::EXE_EXTENSION); + } + cmd(spacetime_path, [&root_arg, "help"]).run()?; + + Ok(()) +} diff --git a/tools/ci/commands/version-upgrade-check/Cargo.toml b/tools/ci/commands/version-upgrade-check/Cargo.toml new file mode 100644 index 00000000000..bd978f6e437 --- /dev/null +++ b/tools/ci/commands/version-upgrade-check/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "ci-version-upgrade-check" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +ci-args = { path = "../../args" } +duct.workspace = true diff --git a/tools/ci/commands/version-upgrade-check/src/main.rs b/tools/ci/commands/version-upgrade-check/src/main.rs new file mode 100644 index 00000000000..f2c1f7e0bf4 --- /dev/null +++ b/tools/ci/commands/version-upgrade-check/src/main.rs @@ -0,0 +1,33 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::Result; +use ci_args::version_upgrade_check::Args; +use clap::Parser; +use duct::cmd; + +/// Verifies that the repository version upgrade tool still works. +#[derive(Parser)] +struct Cli { + #[command(flatten)] + args: Args, +} + +fn run_version_upgrade_check() -> Result<()> { + cmd!( + "cargo", + "bump-versions", + "123.456.789", + "--rust-and-cli", + "--csharp", + "--typescript", + "--cpp", + "--accept-snapshots" + ) + .run()?; + Ok(()) +} + +fn main() -> Result<()> { + Cli::parse(); + + run_version_upgrade_check() +} diff --git a/tools/ci/commands/wasm-bindings/Cargo.toml b/tools/ci/commands/wasm-bindings/Cargo.toml new file mode 100644 index 00000000000..a255c928130 --- /dev/null +++ b/tools/ci/commands/wasm-bindings/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ci-wasm-bindings" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +ci-args = { path = "../../args" } +duct.workspace = true +ci-common = { path = "../../common" } diff --git a/tools/ci/commands/wasm-bindings/src/main.rs b/tools/ci/commands/wasm-bindings/src/main.rs new file mode 100644 index 00000000000..7b562bad2fa --- /dev/null +++ b/tools/ci/commands/wasm-bindings/src/main.rs @@ -0,0 +1,45 @@ +#![allow(clippy::disallowed_macros)] +use anyhow::Result; +use ci_args::wasm_bindings::Args; +use ci_common::pnpm; +use clap::Parser; +use duct::cmd; + +/// Tests Wasm bindings +/// +/// Runs tests for the codegen crate and builds a test module with the wasm bindings. +#[derive(Parser)] +struct Cli { + #[command(flatten)] + args: Args, +} + +fn main() -> Result<()> { + Cli::parse(); + + pnpm([ + "install", + "--filter", + "./crates/bindings-typescript...", + "--filter", + "./modules/module-test-ts...", + ]) + .run()?; + pnpm(["build"]).dir("crates/bindings-typescript").run()?; + cmd!("cargo", "test", "-p", "spacetimedb-codegen").run()?; + // Pre-build the CLI so that it _doesn't_ get `cargo update`d, since that may break the build. + cmd!("cargo", "build", "-p", "spacetimedb-cli").run()?; + // Make sure the `Cargo.lock` file reflects the latest available versions. + // This is what users would end up with on a fresh module, so we want to + // catch any compile errors arising from a different transitive closure + // of dependencies than what is in the workspace lock file. + // + // For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714 + cmd!("cargo", "update").run()?; + let cli_path = ci_common::repo_root() + .join("target/debug/spacetimedb-cli") + .with_extension(std::env::consts::EXE_EXTENSION); + cmd!(cli_path, "build", "--module-path", "modules/module-test",).run()?; + + Ok(()) +} diff --git a/tools/ci/common/Cargo.toml b/tools/ci/common/Cargo.toml new file mode 100644 index 00000000000..2e45f5dbe66 --- /dev/null +++ b/tools/ci/common/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "ci-common" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true diff --git a/tools/ci/common/src/lib.rs b/tools/ci/common/src/lib.rs new file mode 100644 index 00000000000..4dfafe8e03b --- /dev/null +++ b/tools/ci/common/src/lib.rs @@ -0,0 +1,36 @@ +use anyhow::{bail, Result}; +use duct::{cmd, Expression}; +use std::ffi::OsStr; +use std::path::Path; +use std::path::PathBuf; + +pub fn ensure_repo_root() -> Result<()> { + if !Path::new("Cargo.toml").exists() { + bail!("You must execute this command from the SpacetimeDB repository root (where Cargo.toml is located)"); + } + Ok(()) +} + +pub fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(|p| p.parent()) + .and_then(|p| p.parent()) + .expect("failed to find repo root") + .to_path_buf() +} + +pub fn pnpm(args: I) -> Expression +where + I: IntoIterator, + S: AsRef, +{ + let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); + if cfg!(windows) { + let mut full: Vec = vec!["/c".into(), "pnpm".into()]; + full.extend(args); + cmd("cmd", full) + } else { + cmd("pnpm", args) + } +} diff --git a/tools/ci/src/ci_docs.rs b/tools/ci/src/ci_docs.rs deleted file mode 100644 index 3fadcdaf0cc..00000000000 --- a/tools/ci/src/ci_docs.rs +++ /dev/null @@ -1,85 +0,0 @@ -use clap::{Command, CommandFactory}; - -use crate::Cli; - -// TODO: use clap_markdown instead of this custom implementation in the future -pub fn generate_cli_docs() -> String { - let mut cli = Cli::command(); - let usage = generate_markdown(&mut cli, 2); - - format!( - "\ -# SpacetimeDB's cargo ci - -## Overview - -This document provides an overview of the `cargo ci` command-line tool, and documentation for each of its subcommands and options. - -{} ---- - -This document is auto-generated by running: - -```bash -cargo ci self-docs -```", - usage - ) -} - -fn generate_markdown(cmd: &mut Command, heading_level: usize) -> String { - let mut out = String::new(); - - let heading = "#".repeat(heading_level); - out.push_str(&format!("{} `{}`\n\n", heading, cmd.get_name())); - - if let Some(long_about) = cmd.get_long_about() { - out.push_str(&format!("{}\n\n", long_about)); - } - - out.push_str(&format!("**Usage:**\n```bash\n{}\n```\n\n", cmd.render_usage())); - - let mut options = String::new(); - for arg in cmd.get_arguments() { - let mut names = arg - .get_long() - .map(|l| format!("--{}", l)) - .or_else(|| arg.get_short().map(|s| format!("-{}", s))) - .unwrap_or_else(|| arg.get_id().to_string()); - if let Some(value_names) = arg.get_value_names().filter(|_| arg.get_action().takes_values()) { - for value_name in value_names { - names.push_str(&format!(" <{value_name}>")); - } - } - let help = arg - .get_long_help() - .or_else(|| arg.get_help()) - .map(|help| help.to_string()) - .unwrap_or_else(|| { - eprintln!("Warning: argument `{}` is missing help text", arg.get_id()); - "".to_string() - }); - if help.is_empty() { - options.push_str(&format!("- `{names}`:\n")); - } else { - options.push_str(&format!( - "- `{}`: {}\n{}", - names, - help, - if help.lines().count() > 1 { "\n" } else { "" } - )); - } - } - - if !options.is_empty() { - out.push_str("**Options:**\n\n"); - out.push_str(&options); - out.push('\n'); - } - - for sub in cmd.get_subcommands_mut() { - out.push_str(&generate_markdown(sub, heading_level + 1)); - } - - out -} diff --git a/tools/ci/src/keynote_bench.rs b/tools/ci/src/keynote_bench.rs deleted file mode 100644 index ed49bdd8185..00000000000 --- a/tools/ci/src/keynote_bench.rs +++ /dev/null @@ -1,11 +0,0 @@ -use anyhow::Result; -use keynote_bench_harness::KeynoteBenchConfig; -use spacetimedb_guard::{ensure_binaries_built, SpacetimeDbGuard}; - -pub fn run() -> Result<()> { - let cli_path = ensure_binaries_built(); - let server = SpacetimeDbGuard::spawn_in_temp_data_dir(); - let server_url = server.host_url.clone(); - - keynote_bench_harness::run(KeynoteBenchConfig::standalone(".", cli_path, server_url)) -} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 469d085bf5e..125768c0450 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,44 +1,8 @@ #![allow(clippy::disallowed_macros)] -use anyhow::{bail, Context, Result}; +use anyhow::Result; use clap::{CommandFactory, Parser, Subcommand}; -use duct::{cmd, Expression}; -use serde_json::Value; -use std::collections::BTreeSet; -use std::ffi::OsString; -use std::path::Path; -use std::path::PathBuf; -use std::{env, fs}; - -const README_PATH: &str = "tools/ci/README.md"; - -mod ci_docs; -mod cla_assistant; -mod codeowners_check; -mod internal_tests; -mod keynote_bench; -mod smoketest; -mod util; - -use util::ensure_repo_root; - -/// On Windows, `pnpm` is installed as a `.cmd` shim which `CreateProcess` cannot -/// find without going through the shell. Wrapping with `cmd /c` fixes this. -/// On Unix, we invoke `pnpm` directly. -fn pnpm(args: I) -> Expression -where - I: IntoIterator, - S: AsRef, -{ - let args: Vec = args.into_iter().map(|a| a.as_ref().to_os_string()).collect(); - if cfg!(windows) { - let mut full: Vec = vec!["/c".into(), "pnpm".into()]; - full.extend(args); - cmd("cmd", full) - } else { - cmd("pnpm", args) - } -} +use duct::cmd; /// SpacetimeDB CI tasks /// @@ -62,303 +26,6 @@ struct Cli { skip: Vec, } -fn check_global_json_policy() -> Result<()> { - ensure_repo_root()?; - - let root_json = Path::new("global.json"); - let root_contents = fs::read_to_string(root_json)?; - - let globals = git_tracked_files(":(glob)**/global.json")?; - - let mut ok = true; - for p in globals { - let meta = fs::symlink_metadata(&p)?; - let is_symlink = meta.file_type().is_symlink(); - let is_template_global_json = p.strip_prefix(".").unwrap_or(&p).starts_with(Path::new("templates")); - if is_template_global_json && is_symlink { - eprintln!( - "Error: {} is a symlink. Template files must not be symlinks; they are copied literally and this will break if the CLI is built under Windows where symlinks are not supported.", - p.display() - ); - ok = false; - } - - let contents = fs::read_to_string(&p)?; - if contents != root_contents { - eprintln!("Error: {} does not match the root global.json contents", p.display()); - ok = false; - } else if !is_template_global_json || !is_symlink { - println!("OK: {}", p.display()); - } - } - - if !ok { - bail!("global.json policy check failed"); - } - - Ok(()) -} - -fn package_json_pnpm_version(package_manager: &str) -> Option<&str> { - package_manager.strip_prefix("pnpm@") -} - -fn git_tracked_files(pathspec: &str) -> Result> { - let output = cmd!("git", "ls-files", pathspec).read()?; - Ok(output.lines().map(PathBuf::from).collect()) -} - -fn package_json_string_value(package_json: &Value, key: &str) -> Option { - package_json.get(key)?.as_str().map(str::to_owned) -} - -fn package_json_engines_pnpm(package_json: &Value) -> Option { - package_json.get("engines")?.get("pnpm")?.as_str().map(str::to_owned) -} - -fn read_package_json(path: &Path) -> Result { - let contents = fs::read_to_string(path)?; - Ok(serde_json::from_str(&contents)?) -} - -fn is_npm_package_json(package_json: &Value) -> bool { - [ - "bin", - "dependencies", - "devDependencies", - "exports", - "main", - "optionalDependencies", - "packageManager", - "peerDependencies", - "scripts", - "type", - ] - .iter() - .any(|key| package_json.get(key).is_some()) -} - -fn is_template_path(path: &Path) -> bool { - path.starts_with("templates") -} - -fn minimum_release_age(path: &Path) -> Result { - let workspace = fs::read_to_string(path)?; - workspace - .lines() - .find_map(|line| { - let line = line.trim(); - let value = line.strip_prefix("minimumReleaseAge:")?.trim(); - value.parse::().ok() - }) - .ok_or_else(|| anyhow::anyhow!("{} is missing minimumReleaseAge", path.display())) -} - -fn npmrc_minimum_release_age(path: &Path, expected_minimum_release_age: u64) -> Result { - let contents = fs::read_to_string(path).map_err(|err| { - if err.kind() == std::io::ErrorKind::NotFound { - anyhow::anyhow!( - "{} is tracked but missing from the working tree. Restore it with:\nminimum-release-age={}", - path.display(), - expected_minimum_release_age - ) - } else { - anyhow::anyhow!( - "failed to read {} while checking pnpm minimum package age: {err}", - path.display() - ) - } - })?; - contents - .lines() - .find_map(|line| { - let line = line.trim(); - let value = line.strip_prefix("minimum-release-age=")?.trim(); - value.parse::().ok() - }) - .ok_or_else(|| { - anyhow::anyhow!( - "{} must contain `minimum-release-age={}` to match root pnpm-workspace.yaml", - path.display(), - expected_minimum_release_age - ) - }) -} - -fn check_pnpm_release_age_policy() -> Result<()> { - ensure_repo_root()?; - - let root_package_json_path = Path::new("package.json"); - let root_package_json = read_package_json(root_package_json_path)?; - let package_manager = package_json_string_value(&root_package_json, "packageManager") - .ok_or_else(|| anyhow::anyhow!("package.json is missing packageManager"))?; - let package_manager_version = package_json_pnpm_version(&package_manager) - .ok_or_else(|| anyhow::anyhow!("packageManager must be pnpm@, found {package_manager:?}"))?; - - let expected_engine_pnpm = format!(">={package_manager_version}"); - let engine_pnpm = package_json_engines_pnpm(&root_package_json) - .ok_or_else(|| anyhow::anyhow!("package.json engines is missing pnpm"))?; - if engine_pnpm != expected_engine_pnpm { - bail!("package.json engines.pnpm must be {expected_engine_pnpm:?}, found {engine_pnpm:?}"); - } - - for package_json_path in git_tracked_files(":(glob)**/package.json")? { - let package_json = read_package_json(&package_json_path)?; - let Some(found_package_manager) = package_json_string_value(&package_json, "packageManager") else { - continue; - }; - if found_package_manager != package_manager { - bail!( - "{} packageManager must match root package.json: expected {:?}, found {:?}", - package_json_path.display(), - package_manager, - found_package_manager - ); - } - } - - let root_workspace_path = Path::new("pnpm-workspace.yaml"); - let root_minimum_release_age = minimum_release_age(root_workspace_path)?; - for workspace_path in git_tracked_files(":(glob)**/pnpm-workspace.yaml")? { - let found_minimum_release_age = minimum_release_age(&workspace_path)?; - if found_minimum_release_age != root_minimum_release_age { - bail!( - "{} minimumReleaseAge must match root pnpm-workspace.yaml: expected {}, found {}", - workspace_path.display(), - root_minimum_release_age, - found_minimum_release_age - ); - } - } - - for npmrc_path in git_tracked_files(":(glob)**/.npmrc")? { - // Template package roots are copied into projects created by `spacetime init`. - // They must not embed this repo's package-age policy; smoketests enforce it - // at the pnpm process boundary instead. - if is_template_path(&npmrc_path) { - continue; - } - let found_minimum_release_age = npmrc_minimum_release_age(&npmrc_path, root_minimum_release_age)?; - if found_minimum_release_age != root_minimum_release_age { - bail!( - "{} minimum-release-age must match root pnpm-workspace.yaml: expected {}, found {}", - npmrc_path.display(), - root_minimum_release_age, - found_minimum_release_age - ); - } - } - - for package_json_path in git_tracked_files(":(glob)**/package.json")? { - // Template package roots are copied into projects created by `spacetime init`. - // They must not require adjacent .npmrc files for this repo's package-age - // policy; smoketests enforce it at the pnpm process boundary instead. - if is_template_path(&package_json_path) { - continue; - } - let package_json = read_package_json(&package_json_path)?; - if !is_npm_package_json(&package_json) { - continue; - } - let package_dir = package_json_path - .parent() - .expect("git-tracked package.json path should have a parent"); - let npmrc_path = package_dir.join(".npmrc"); - if !npmrc_path.is_file() { - bail!( - "{} is required because {} is an npm/pnpm package manifest.\nAdd {} containing:\nminimum-release-age={}", - npmrc_path.display(), - package_json_path.display(), - npmrc_path.display(), - root_minimum_release_age - ); - } - let found_minimum_release_age = npmrc_minimum_release_age(&npmrc_path, root_minimum_release_age)?; - if found_minimum_release_age != root_minimum_release_age { - bail!( - "{} minimum-release-age must match root pnpm-workspace.yaml: expected {}, found {}", - npmrc_path.display(), - root_minimum_release_age, - found_minimum_release_age - ); - } - } - - for workflow_path in git_tracked_files(".github/workflows/*")? { - let contents = fs::read_to_string(&workflow_path)?; - if contents.contains("pnpm/action-setup@v4") { - bail!( - "{} must use ./.github/actions/setup-pnpm instead of pnpm/action-setup@v4", - workflow_path.display() - ); - } - } - - Ok(()) -} - -/// Codex plugin ships a copy of `skills/`, because plugin installers do not follow symlinks, -/// this checks if the copy is in sync -fn check_codex_plugin_skills_sync() -> Result<()> { - let source = Path::new("skills"); - let copy = Path::new("codex-plugin/plugins/spacetimedb/skills"); - - if fs::symlink_metadata(copy) - .with_context(|| format!("reading {}", copy.display()))? - .file_type() - .is_symlink() - { - bail!( - "{} must be a real directory, not a symlink, plugin installers do not follow symlinks", - copy.display() - ); - } - - fn walk(root: &Path, dir: &Path, out: &mut BTreeSet) -> Result<()> { - for entry in fs::read_dir(dir).with_context(|| format!("reading {}", dir.display()))? { - let entry = entry?; - if entry.file_type()?.is_dir() { - walk(root, &entry.path(), out)?; - } else { - let rel = entry - .path() - .strip_prefix(root) - .expect("walked path should be under its root") - .to_path_buf(); - out.insert(rel); - } - } - Ok(()) - } - - let mut source_files = BTreeSet::new(); - walk(source, source, &mut source_files)?; - let mut copy_files = BTreeSet::new(); - walk(copy, copy, &mut copy_files)?; - - let mut drift = Vec::new(); - for rel in &source_files { - if !copy_files.contains(rel) { - drift.push(format!("missing from copy: {}", rel.display())); - } else if fs::read(source.join(rel))? != fs::read(copy.join(rel))? { - drift.push(format!("differs: {}", rel.display())); - } - } - for rel in ©_files { - if !source_files.contains(rel) { - drift.push(format!("extraneous in copy: {}", rel.display())); - } - } - - if !drift.is_empty() { - bail!( - "codex-plugin skills copy is out of sync with skills/:\n {}\nRun: node codex-plugin/scripts/check-skills-sync.ts --fix", - drift.join("\n ") - ); - } - Ok(()) -} - #[derive(Subcommand)] enum CiCmd { /// Runs tests @@ -366,76 +33,46 @@ enum CiCmd { /// Runs rust tests, codegens csharp sdk and runs csharp tests. /// This does not include Unreal tests. /// This expects to run in a clean git state. - Test, + Test(ci_args::test::Args), /// Lints the codebase /// /// Runs rustfmt, clippy, csharpier, TypeScript lint, and generates rust docs to ensure there /// are no warnings. - Lint, + Lint(ci_args::lint::Args), /// Tests Wasm bindings /// /// Runs tests for the codegen crate and builds a test module with the wasm bindings. - WasmBindings, + WasmBindings(ci_args::wasm_bindings::Args), /// Deprecated; use `cargo regen csharp dlls`. - /// - /// Builds and packs C# DLLs and NuGet packages for local Unity workflows. Dlls, /// Runs smoketests /// /// Executes the smoketests suite with some default exclusions. - Smoketests(smoketest::SmoketestsArgs), + Smoketests(ci_args::smoketests::SmoketestsArgs), /// Runs the keynote benchmark as a CI performance regression gate. /// /// Assumes release SpacetimeDB binaries and the TypeScript SDK are already built, runs the /// keynote SpacetimeDB benchmark for 60 seconds against the TypeScript and Rust modules, and /// fails if throughput is below 275K TPS for TypeScript or 300K TPS for Rust. - KeynoteBench, + KeynoteBench(ci_args::keynote_bench::Args), /// Tests the update flow /// /// Tests the self-update flow by building the spacetimedb-update binary for the specified /// target, by default the current target, and performing a self-install into a temporary /// directory. - UpdateFlow { - #[arg( - long, - long_help = "Target triple to build for, by default the current target. Used by github workflows to check the update flow on multiple platforms." - )] - target: Option, - #[arg( - long, - default_value = "false", - long_help = "Whether to enable github token authentication feature when building the update binary. By default this is disabled." - )] - github_token_auth: bool, - }, + UpdateFlow(ci_args::update_flow::Args), /// Generates CLI documentation and checks for changes - CliDocs { - #[arg( - long, - long_help = "specify a custom path to the SpacetimeDB repository root (where the main Cargo.toml is located)" - )] - spacetime_path: Option, - }, - SelfDocs { - #[arg( - long, - default_value_t = false, - long_help = "Only check for changes, do not generate the docs" - )] - check: bool, - }, - + CliDocs(ci_args::cli_docs::Args), /// Verify that any non-root global.json files are symlinks to the root global.json. - GlobalJsonPolicy, + GlobalJsonPolicy(ci_args::global_json_policy::Args), /// Checks that publishable crates satisfy publish constraints. - PublishChecks, + PublishChecks(ci_args::publish_checks::Args), /// Runs TypeScript workspace tests and template build checks. - TypescriptTest, + TypescriptTest(ci_args::typescript_test::Args), /// Verifies that the repository version upgrade tool still works. - VersionUpgradeCheck, + VersionUpgradeCheck(ci_args::version_upgrade_check::Args), /// Builds the docs site. - Docs, - /// Workflows should leave here if they should not be run as part of a no-subcommand invocation of `cargo ci`. + Docs(ci_args::docs::Args), OtherWorkflows { #[command(subcommand)] cmd: OtherWorkflowsCmd, @@ -445,471 +82,107 @@ enum CiCmd { #[derive(Subcommand)] enum OtherWorkflowsCmd { /// Selects or starts the private workflow for a public Internal Tests run. - CoordinateInternalTests(internal_tests::CoordinateArgs), + CoordinateInternalTests(ci_args::coordinate_internal_tests::Args), /// Checks that sensitive CODEOWNERS-controlled files have the required approvals. - CodeownersCheck { - /// Git ref to compare against, usually origin/. - #[arg(long)] - base_ref: String, - /// Pull request number to inspect for approval state. - #[arg(long)] - pr_number: u64, - }, + CodeownersCheck(ci_args::codeowners_check::Args), /// Interacts with CLA Assistant. - ClaAssistant { - #[command(subcommand)] - cmd: cla_assistant::ClaAssistantCmd, - }, + ClaAssistant(ci_args::cla_assistant::Args), } -fn run_all_clap_subcommands(skips: &[String]) -> Result<()> { - let subcmds = Cli::command() - .get_subcommands() - .map(|sc| sc.get_name().to_string()) - .collect::>(); - - for subcmd in subcmds { - if skips.contains(&subcmd) { - log::info!("skipping {subcmd} as requested"); - continue; - } - log::info!("executing cargo ci {subcmd}"); - cmd!("cargo", "ci", &subcmd).run()?; - } +fn run_package(package: &str, args: &[String]) -> Result<()> { + let mut cargo_args = vec!["run", "--package", package, "--"]; + cargo_args.extend(args.iter().map(String::as_str)); + cmd("cargo", cargo_args).run()?; Ok(()) } -fn tracked_rs_files_under(path: &str) -> Result> { - let output = cmd!("git", "ls-files", "--", path).read()?; - Ok(output - .lines() - .filter(|line| line.ends_with(".rs")) - .map(PathBuf::from) - .collect()) -} - -fn run_publish_checks() -> Result<()> { - cmd!("bash", "-lc", "test -d venv || python3 -m venv venv").run()?; - cmd!("venv/bin/pip3", "install", "argparse", "toml").run()?; - - let crates = cmd!( - "venv/bin/python3", - "tools/find-publish-list.py", - "--recursive", - "--directories", - "--quiet", - "spacetimedb", - "spacetimedb-sdk" - ) - .read()?; - - let mut failed = Vec::new(); - for crate_dir in crates.split_whitespace() { - if let Err(err) = cmd!("venv/bin/python3", "tools/crate-publish-checks.py", crate_dir).run() { - eprintln!("crate publish checks failed for {crate_dir}: {err}"); - failed.push(crate_dir.to_string()); +fn forwarded_args(raw_args: &[String], path: &[&str]) -> Vec { + let args = &raw_args[1..]; + let mut start = None; + let mut idx = 0; + while idx < args.len() { + match args[idx].as_str() { + "--skip" => idx += 2, + arg if arg.starts_with("--skip=") => idx += 1, + _ => { + if args[idx] == path[0] + && args + .get(idx..idx + path.len()) + .is_some_and(|window| window.iter().map(String::as_str).eq(path.iter().copied())) + { + start = Some(idx); + break; + } + idx += 1; + } } } - if !failed.is_empty() { - bail!("crate publish checks failed for: {}", failed.join(", ")); - } - - Ok(()) -} - -fn run_typescript_tests() -> Result<()> { - pnpm(["build"]).dir("crates/bindings-typescript").run()?; - pnpm(["test"]).dir("crates/bindings-typescript").run()?; - pnpm(["generate"]).dir("templates/chat-react-ts").run()?; - let diff_status = cmd!( - "bash", - "tools/check-diff.sh", - "templates/chat-react-ts/src/module_bindings" - ) - .run()?; - if !diff_status.status.success() { - bail!("Bindings are dirty. Please generate bindings again and commit them to this branch."); - } - pnpm(["build"]).dir("templates/chat-react-ts").run()?; - pnpm(["-r", "--filter", "./**", "run", "build"]) - .dir("templates") - .run()?; - pnpm(["-r", "--filter", "./**", "run", "build"]) - .dir("crates/bindings-typescript") - .run()?; - Ok(()) + let start = start.unwrap_or_else(|| panic!("missing command path `{}` in raw argv", path.join(" "))); + args[start + path.len()..].to_vec() } -fn run_docs_build() -> Result<()> { - pnpm(["install"]).dir("docs").run()?; - pnpm(["build"]).dir("docs").run()?; +fn run_dlls() -> Result<()> { + eprintln!("warning: `cargo ci dlls` is deprecated; use `cargo regen csharp dlls` instead"); + cmd!("cargo", "regen", "csharp", "dlls").run()?; Ok(()) } -fn run_version_upgrade_check() -> Result<()> { - cmd!( - "cargo", - "bump-versions", - "123.456.789", - "--rust-and-cli", - "--csharp", - "--typescript", - "--cpp", - "--accept-snapshots" - ) - .run()?; +fn run_command(cmd: CiCmd, raw_args: &[String]) -> Result<()> { + match cmd { + CiCmd::Test(_) => run_package("ci-test", &forwarded_args(raw_args, &["test"])), + CiCmd::Lint(_) => run_package("ci-lint", &forwarded_args(raw_args, &["lint"])), + CiCmd::WasmBindings(_) => run_package("ci-wasm-bindings", &forwarded_args(raw_args, &["wasm-bindings"])), + CiCmd::Dlls => run_dlls(), + CiCmd::Smoketests(_) => run_package("ci-smoketests", &forwarded_args(raw_args, &["smoketests"])), + CiCmd::KeynoteBench(_) => run_package("ci-keynote-bench", &forwarded_args(raw_args, &["keynote-bench"])), + CiCmd::UpdateFlow(_) => run_package("ci-update-flow", &forwarded_args(raw_args, &["update-flow"])), + CiCmd::CliDocs(_) => run_package("ci-cli-docs", &forwarded_args(raw_args, &["cli-docs"])), + CiCmd::GlobalJsonPolicy(_) => run_package( + "ci-global-json-policy", + &forwarded_args(raw_args, &["global-json-policy"]), + ), + CiCmd::PublishChecks(_) => run_package("ci-publish-checks", &forwarded_args(raw_args, &["publish-checks"])), + CiCmd::TypescriptTest(_) => run_package("ci-typescript-test", &forwarded_args(raw_args, &["typescript-test"])), + CiCmd::VersionUpgradeCheck(_) => run_package( + "ci-version-upgrade-check", + &forwarded_args(raw_args, &["version-upgrade-check"]), + ), + CiCmd::Docs(_) => run_package("ci-docs-build", &forwarded_args(raw_args, &["docs"])), + CiCmd::OtherWorkflows { cmd } => match cmd { + OtherWorkflowsCmd::CoordinateInternalTests(_) => run_package( + "ci-coordinate-internal-tests", + &forwarded_args(raw_args, &["other-workflows", "coordinate-internal-tests"]), + ), + OtherWorkflowsCmd::CodeownersCheck(_) => run_package( + "ci-codeowners-check", + &forwarded_args(raw_args, &["other-workflows", "codeowners-check"]), + ), + OtherWorkflowsCmd::ClaAssistant(_) => run_package( + "ci-cla-assistant", + &forwarded_args(raw_args, &["other-workflows", "cla-assistant"]), + ), + }, + } +} + +fn run_all_clap_subcommands(skip: &[String]) -> Result<()> { + for subcommand in Cli::command().get_subcommands() { + let name = subcommand.get_name(); + if skip.iter().any(|skip| skip == name) { + continue; + } + cmd!("cargo", "ci", name).run()?; + } Ok(()) } fn main() -> Result<()> { - env_logger::init(); - - let cli = Cli::parse(); + let raw_args = std::env::args().collect::>(); + let cli = Cli::parse_from(&raw_args); match cli.cmd { - Some(CiCmd::Test) => { - pnpm(["build"]).dir("crates/bindings-typescript").run()?; - - // TODO: This doesn't work on at least user Linux machines, because something here apparently uses `sudo`? - - // Exclude smoketests from `cargo test --all` since they require pre-built binaries. - // Smoketests have their own dedicated command: `cargo ci smoketests` - cmd!( - "cargo", - "test", - "--all", - "--exclude", - "spacetimedb-smoketests", - "--exclude", - "spacetimedb-sdk", - "--exclude", - "spacetimedb", - "--", - "--test-threads=2", - "--skip", - "unreal" - ) - .run()?; - // Bindings snapshot tests rely on the unstable feature, - // as they compile and test APIs which are gated behind that feature, - // e.g. procedures, HTTP handlers. - cmd!( - "cargo", - "test", - "-p", - "spacetimedb", - "--features", - "unstable", - "--", - "--test-threads=2", - ) - .run()?; - // The SDK test harness uses the same child-process server guard as smoketests, - // which expects release CLI/standalone binaries to already exist. - cmd!( - "cargo", - "build", - "--release", - "-p", - "spacetimedb-cli", - "-p", - "spacetimedb-standalone", - "--features", - "spacetimedb-standalone/allow_loopback_http_for_tests", - ) - .run()?; - // SDK procedure tests intentionally make localhost HTTP requests. - cmd!( - "cargo", - "test", - "-p", - "spacetimedb-sdk", - "--features", - "allow_loopback_http_for_tests", - "--", - "--test-threads=2", - "--skip", - "unreal" - ) - .run()?; - // Run the same SDK suite against wasm/browser test clients. - cmd!( - "cargo", - "test", - "-p", - "spacetimedb-sdk", - "--features", - "allow_loopback_http_for_tests,browser", - "--", - "--test-threads=2", - "--skip", - "unreal" - ) - .run()?; - // TODO: This should check for a diff at the start. If there is one, we should alert the user - // that we're disabling diff checks because they have a dirty git repo, and to re-run in a clean one - // if they want those checks. - - // The fallocate tests have been flakely when running in parallel - cmd!( - "cargo", - "test", - "-p", - "spacetimedb-durability", - "--features", - "fallocate", - "--", - "--test-threads=1", - ) - .run()?; - cmd!("bash", "tools/check-diff.sh").run()?; - cmd!( - "cargo", - "run", - "-p", - "spacetimedb-codegen", - "--example", - "regen-csharp-moduledef", - ) - .run()?; - cmd!("bash", "tools/check-diff.sh", "crates/bindings-csharp").run()?; - cmd!("dotnet", "test", "-warnaserror") - .dir("crates/bindings-csharp") - .run()?; - } - - Some(CiCmd::Lint) => { - ensure_repo_root()?; - check_pnpm_release_age_policy()?; - check_codex_plugin_skills_sync()?; - // `cargo fmt --all` only checks files that Cargo discovers through workspace/package targets. - // However, we also keep Rust sources in a locations that are tracked but not part of our workspace, - // so this approach properly catches all the files, where `cargo fmt` does not. - let mut files = Vec::new(); - files.extend(tracked_rs_files_under(".")?); - const RUSTFMT_BATCH_SIZE: usize = 200; - for batch in files.chunks(RUSTFMT_BATCH_SIZE) { - let mut args = Vec::::with_capacity(batch.len() + 1); - args.push("--check".into()); - args.extend(batch.iter().map(|path| path.as_os_str().to_os_string())); - cmd("rustfmt", args).run()?; - } - cmd!( - "cargo", - "clippy", - "--timings", - "--all", - "--tests", - "--benches", - "--", - "-D", - "warnings", - ) - .run()?; - cmd!( - "cargo", - "clippy", - "--timings", - "--no-default-features", - "--features=browser", - "-pspacetimedb-sdk", - "--tests", - "--benches", - "--", - "-D", - "warnings", - ) - .run()?; - cmd!("dotnet", "tool", "restore").dir("crates/bindings-csharp").run()?; - cmd!("dotnet", "csharpier", "--check", ".") - .dir("crates/bindings-csharp") - .run()?; - pnpm(["lint"]).run()?; - cmd!("cargo", "test", "--doc", "--target", "wasm32-unknown-unknown") - .dir("crates/bindings") - .run()?; - cmd!("cargo", "test", "--doc").dir("crates/bindings").run()?; - // `bindings` is the only crate we care strongly about documenting, - // since we link to its docs.rs from our website. - // We won't pass `--no-deps`, though, - // since we want everything reachable through it to also work. - // This includes `sats` and `lib`. - cmd!("cargo", "doc") - .dir("crates/bindings") - // Make `cargo doc` exit with error on warnings, most notably broken links - .env("RUSTDOCFLAGS", "--deny warnings") - .run()?; - } - - Some(CiCmd::WasmBindings) => { - pnpm([ - "install", - "--filter", - "./crates/bindings-typescript...", - "--filter", - "./modules/module-test-ts...", - ]) - .run()?; - pnpm(["build"]).dir("crates/bindings-typescript").run()?; - cmd!("cargo", "test", "-p", "spacetimedb-codegen").run()?; - // Pre-build the CLI so that it _doesn't_ get `cargo update`d, since that may break the build. - cmd!("cargo", "build", "-p", "spacetimedb-cli").run()?; - // Make sure the `Cargo.lock` file reflects the latest available versions. - // This is what users would end up with on a fresh module, so we want to - // catch any compile errors arising from a different transitive closure - // of dependencies than what is in the workspace lock file. - // - // For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714 - cmd!("cargo", "update").run()?; - let cli_path = Path::new(env!("CARGO_MANIFEST_DIR")) - .ancestors() - .nth(2) - .unwrap() - .join("target/debug/spacetimedb-cli") - .with_extension(std::env::consts::EXE_EXTENSION); - cmd!(cli_path, "build", "--module-path", "modules/module-test",).run()?; - } - - Some(CiCmd::Dlls) => { - eprintln!("warning: `cargo ci dlls` is deprecated; use `cargo regen csharp dlls` instead"); - cmd!("cargo", "regen", "csharp", "dlls").run()?; - } - - Some(CiCmd::Smoketests(args)) => { - ensure_repo_root()?; - smoketest::run(args)?; - } - - Some(CiCmd::KeynoteBench) => { - ensure_repo_root()?; - keynote_bench::run()?; - } - - Some(CiCmd::UpdateFlow { - target, - github_token_auth, - }) => { - let mut common_args = vec![]; - if let Some(target) = target.as_ref() { - common_args.push("--target"); - common_args.push(target); - log::info!("checking update flow for target: {target}"); - } else { - log::info!("checking update flow"); - } - if github_token_auth { - common_args.push("--features"); - common_args.push("github-token-auth"); - } - - cmd( - "cargo", - ["build", "-p", "spacetimedb-update"] - .into_iter() - .chain(common_args.clone()), - ) - .run()?; - // NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`. - // My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it - // happens very frequently on the `macos-runner`, but we haven't seen it on any others). - let root_dir = tempfile::tempdir()?; - let root_dir_string = root_dir.path().to_string_lossy().to_string(); - let root_arg = format!("--root-dir={}", root_dir_string); - cmd( - "cargo", - ["run", "-p", "spacetimedb-update"] - .into_iter() - .chain(common_args.clone()) - .chain(["--", "self-install", &root_arg, "--yes"].into_iter()), - ) - .run()?; - - let mut spacetime_path = root_dir.path().join("spacetime"); - if !std::env::consts::EXE_EXTENSION.is_empty() { - spacetime_path.set_extension(std::env::consts::EXE_EXTENSION); - } - cmd(spacetime_path, [&root_arg, "help"]).run()?; - } - - Some(CiCmd::CliDocs { spacetime_path }) => { - if let Some(path) = spacetime_path { - env::set_current_dir(path).ok(); - } - let current_dir = env::current_dir().expect("No current directory!"); - let dir_name = current_dir.file_name().expect("No current directory!"); - if dir_name != "SpacetimeDB" && dir_name != "public" { - anyhow::bail!( - "You must execute this binary from inside of the SpacetimeDB directory, or use --spacetime-path" - ); - } - - pnpm(["install", "--recursive"]).run()?; - pnpm(["generate-cli-docs"]).dir("docs").run()?; - let out = cmd!("git", "status", "--porcelain", "--", "docs").read()?; - if out.is_empty() { - log::info!("No docs changes detected"); - } else { - anyhow::bail!("CLI docs are out of date:\n{out}"); - } - } - - Some(CiCmd::SelfDocs { check }) => { - let readme_content = ci_docs::generate_cli_docs(); - let path = Path::new(README_PATH); - - if check { - let existing = fs::read_to_string(path).unwrap_or_default(); - if existing != readme_content { - bail!("README.md is out of date. Please run `cargo ci self-docs` to update it."); - } else { - log::info!("README.md is up to date."); - } - } else { - fs::write(path, readme_content)?; - log::info!("Wrote CLI docs to {}", path.display()); - } - } - - Some(CiCmd::GlobalJsonPolicy) => { - check_global_json_policy()?; - } - - Some(CiCmd::OtherWorkflows { - cmd: OtherWorkflowsCmd::CoordinateInternalTests(args), - }) => { - internal_tests::coordinate(args)?; - } - Some(CiCmd::OtherWorkflows { - cmd: OtherWorkflowsCmd::CodeownersCheck { base_ref, pr_number }, - }) => { - codeowners_check::run(&base_ref, pr_number)?; - } - - Some(CiCmd::PublishChecks) => { - run_publish_checks()?; - } - - Some(CiCmd::TypescriptTest) => { - run_typescript_tests()?; - } - - Some(CiCmd::VersionUpgradeCheck) => { - run_version_upgrade_check()?; - } - - Some(CiCmd::Docs) => { - run_docs_build()?; - } - - Some(CiCmd::OtherWorkflows { - cmd: OtherWorkflowsCmd::ClaAssistant { cmd }, - }) => { - cla_assistant::run(cmd)?; - } - - None => run_all_clap_subcommands(&cli.skip)?, + Some(cmd) => run_command(cmd, &raw_args), + None => run_all_clap_subcommands(&cli.skip), } - - Ok(()) } diff --git a/tools/ci/src/util.rs b/tools/ci/src/util.rs deleted file mode 100644 index ac35a7a465b..00000000000 --- a/tools/ci/src/util.rs +++ /dev/null @@ -1,11 +0,0 @@ -#![allow(clippy::disallowed_macros)] - -use anyhow::{bail, Result}; -use std::path::Path; - -pub fn ensure_repo_root() -> Result<()> { - if !Path::new("Cargo.toml").exists() { - bail!("You must execute this command from the SpacetimeDB repository root (where Cargo.toml is located)"); - } - Ok(()) -}