Load testing foundations: YAML contract, engine levers, CLI load mode, run-flows Action (Phase 0+1)

.github/workflows/action-test.yaml

@@ -0,0 +1,65 @@
+name: Action / run-flows
+
+on:
+  pull_request:
+    paths:
+      - 'actions/**'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Test run-flows (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run smoke flow
+        id: run-flows
+        uses: ./actions/run-flows
+        with:
+          file: actions/run-flows/testdata/smoke.yamlflow.yaml
+          version: latest
+
+      # Do NOT assert on $GITHUB_STEP_SUMMARY here: GitHub gives every step its
+      # own summary file, so what the action appended inside its own step is
+      # invisible to this one (it would read its own empty file and always
+      # fail). Assert on report.json instead — write-summary.sh renders its
+      # table from exactly the fields checked below, so the summary's data is
+      # covered transitively; its formatting is covered by the script's tests.
+      - name: Assert outputs and report contents
+        shell: bash
+        env:
+          JSON_REPORT: ${{ steps.run-flows.outputs.json-report }}
+          JUNIT_REPORT: ${{ steps.run-flows.outputs.junit-report }}
+          SUCCESS: ${{ steps.run-flows.outputs.success }}
+        run: |
+          set -euo pipefail
+
+          echo "json-report=$JSON_REPORT"
+          echo "junit-report=$JUNIT_REPORT"
+          echo "success=$SUCCESS"
+
+          test -f "$JSON_REPORT"
+          test -f "$JUNIT_REPORT"
+
+          [ "$SUCCESS" = "true" ]
+
+          echo "report contents:"
+          jq -c '[.[] | {flow_name, status, duration}]' "$JSON_REPORT"
+
+          # Both flows from testdata/smoke.yamlflow.yaml ran, succeeded, and
+          # recorded a non-zero duration (nanoseconds).
+          jq -e 'length == 2' "$JSON_REPORT" > /dev/null
+          for flow in fetch-user create-post; do
+            jq -e --arg flow "$flow" \
+              'any(.[]; .flow_name == $flow and .status == "success" and .duration > 0)' \
+              "$JSON_REPORT" > /dev/null
+          done

.prettierignore

@@ -1,5 +1,7 @@
 .ai/
+.context/
 .golangci.yml
+.superpowers/
 *.har
 AGENTS.md
 CHANGELOG.md

actions/run-flows/README.md

@@ -0,0 +1,116 @@
+# run-flows
+
+Composite GitHub Action that runs a DevTools `.yamlflow.yaml` file with the
+released `devtoolscli` binary, publishes a job summary, and produces JSON /
+JUnit reports as step outputs.
+
+It downloads the `devtoolscli` release binary for the runner's OS/arch itself
+— the consuming workflow only needs to check out its own repo (the one
+containing the `.yamlflow.yaml` file), not this one:
+
+```yaml
+- uses: the-dev-tools/dev-tools/actions/run-flows@main
+  with:
+    file: flows/smoke.yamlflow.yaml
+```
+
+Pin `@main` to a commit SHA (or a `cli@<version>` tag, which is a normal git
+tag on this monorepo) if you want the action's own behavior to stay fixed
+independently of the `version` input.
+
+## Supported runners
+
+Linux and macOS only (`ubuntu-*`, `macos-*` runners), `x64` and `arm64`.
+Windows is out of scope for this action: `devtoolscli` does publish Windows
+release assets (see `.github/workflows/release-go.yaml`), but this action
+does not resolve or invoke them, and every step is `shell: bash`. Windows CI
+should use the manual install steps in [`docs/cli.md`](../../docs/cli.md)
+instead.
+
+## Inputs
+
+| Name            | Required | Default             | Description                                                                                                 |
+| --------------- | -------- | ------------------- | ----------------------------------------------------------------------------------------------------------- |
+| `file`          | yes      | —                   | Path to the `.yamlflow.yaml` file to run.                                                                   |
+| `flow`          | no       | _(unset)_           | Single flow name to run. Defaults to the file's top-level `run:` block.                                     |
+| `version`       | no       | `latest`            | `devtoolscli` release to install: `latest`, a release tag (`cli@1.0.3`), or a bare version (`1.0.3`).       |
+| `report-dir`    | no       | `.devtools-reports` | Directory to write the JSON and JUnit reports into.                                                         |
+| `fail-on-error` | no       | `true`              | Fail this step if any flow fails. Set to `'false'` to always exit 0 and check the `success` output instead. |
+
+## Outputs
+
+| Name           | Description                                                       |
+| -------------- | ----------------------------------------------------------------- |
+| `json-report`  | Path to the JSON report (empty string if none was produced).      |
+| `junit-report` | Path to the JUnit XML report (empty string if none was produced). |
+| `success`      | `'true'` if every flow in the run succeeded, `'false'` otherwise. |
+
+A job summary table (flow name, ✅/❌ status, duration) is always published to
+the job's summary, even when `fail-on-error: 'false'` or the run fails —
+useful for `if: always()` follow-up steps.
+
+## Examples
+
+### PR check
+
+```yaml
+name: API flows
+on:
+  pull_request:
+jobs:
+  flows:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: the-dev-tools/dev-tools/actions/run-flows@main
+        with:
+          file: flows/smoke.yamlflow.yaml
+```
+
+### Nightly cron against staging
+
+```yaml
+name: Nightly flow check
+on:
+  schedule:
+    - cron: '0 6 * * *'
+jobs:
+  flows:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: the-dev-tools/dev-tools/actions/run-flows@main
+        id: flows
+        with:
+          file: flows/nightly.yamlflow.yaml
+          version: cli@1.0.3
+          fail-on-error: 'false'
+        env:
+          LOGIN_EMAIL: ${{ secrets.LOGIN_EMAIL }}
+          LOGIN_PASSWORD: ${{ secrets.LOGIN_PASSWORD }}
+      - name: Notify on failure
+        if: steps.flows.outputs.success != 'true'
+        run: echo "Nightly flows failed — see the job summary for details" # replace with a real notification step
+```
+
+`env:` overrides for the YAML file's `#env:NAME` placeholders work the same
+way as in the manual CLI usage documented in
+[`docs/cli.md`](../../docs/cli.md#environment-variable-overrides).
+
+## How it works
+
+1. Resolves `version` to a release tag (`cli@<version>`) and downloads the
+   matching `devtools-cli-<version>-<os>-<arch>` asset from this repo's
+   GitHub Releases into `$RUNNER_TEMP/devtools/bin`. `latest` resolves to the
+   highest `cli@*` tag via `git ls-remote` (the repo also cuts `desktop@`/
+   `web@`/etc. releases, so a plain "latest release" API lookup would not be
+   specific enough).
+2. Runs `devtoolscli flow run <file> [flow] --report console --report json:<report-dir>/report.json --report junit:<report-dir>/junit.xml`.
+3. Publishes a job summary table from the JSON report, whether or not the run
+   succeeded.
+4. Sets the `json-report` / `junit-report` / `success` outputs, then fails
+   the step if `fail-on-error` is `'true'` (the default) and any flow failed.
+
+See `apps/cli/internal/reporter/reporter.go` for the JSON report schema and
+[`docs/cli.md`](../../docs/cli.md) for the YAML flow format and the
+`{{ ... }}` interpolation syntax used in `testdata/smoke.yamlflow.yaml`.

actions/run-flows/action.yml

@@ -0,0 +1,78 @@
+name: 'DevTools Run Flows'
+description: 'Run a DevTools YAML flow with the released devtoolscli CLI and publish JSON/JUnit reports plus a job summary.'
+author: 'the-dev-tools'
+
+inputs:
+  file:
+    description: 'Path to the .yamlflow.yaml file to run.'
+    required: true
+  flow:
+    description: "Single flow name to run. Defaults to the file's top-level run: block."
+    required: false
+  version:
+    description: "devtoolscli release to install: 'latest' or a release tag, e.g. cli@1.0.3."
+    required: false
+    default: 'latest'
+  report-dir:
+    description: 'Directory to write the JSON and JUnit reports into.'
+    required: false
+    default: '.devtools-reports'
+  fail-on-error:
+    description: "Fail this step if any flow fails. Set to 'false' to always exit 0 and inspect the success output instead."
+    required: false
+    default: 'true'
+
+outputs:
+  json-report:
+    description: 'Path to the JSON report (empty string if none was produced).'
+    value: ${{ steps.finalize.outputs.json-report }}
+  junit-report:
+    description: 'Path to the JUnit XML report (empty string if none was produced).'
+    value: ${{ steps.finalize.outputs.junit-report }}
+  success:
+    description: "'true' if every flow in the run succeeded, 'false' otherwise."
+    value: ${{ steps.finalize.outputs.success }}
+
+runs:
+  using: composite
+  steps:
+    # linux/macOS only — devtoolscli release-go.yaml does not publish a
+    # Windows-friendly layout for this action to consume; see README.md.
+    - name: Download devtoolscli
+      id: download
+      shell: bash
+      run: '"$GITHUB_ACTION_PATH/scripts/download-cli.sh"'
+      env:
+        VERSION: ${{ inputs.version }}
+
+    # continue-on-error: a failing flow run must not skip the summary/output
+    # steps below — they need to run via if: always() regardless.
+    - name: Run flow
+      id: run
+      shell: bash
+      continue-on-error: true
+      run: '"$GITHUB_ACTION_PATH/scripts/run-flow.sh"'
+      env:
+        CLI_BIN: ${{ steps.download.outputs.bin }}
+        FILE: ${{ inputs.file }}
+        FLOW: ${{ inputs.flow }}
+        REPORT_DIR: ${{ inputs.report-dir }}
+
+    - name: Write job summary
+      id: summary
+      if: always()
+      shell: bash
+      run: '"$GITHUB_ACTION_PATH/scripts/write-summary.sh"'
+      env:
+        REPORT_DIR: ${{ inputs.report-dir }}
+        RUN_OUTCOME: ${{ steps.run.outcome }}
+
+    - name: Set outputs
+      id: finalize
+      if: always()
+      shell: bash
+      run: '"$GITHUB_ACTION_PATH/scripts/finalize.sh"'
+      env:
+        REPORT_DIR: ${{ inputs.report-dir }}
+        RUN_OUTCOME: ${{ steps.run.outcome }}
+        FAIL_ON_ERROR: ${{ inputs.fail-on-error }}

actions/run-flows/scripts/download-cli.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+# Resolves the devtoolscli release for this runner's OS/arch and downloads it
+# into $RUNNER_TEMP/devtools/bin. Reused by actions/run-flows/action.yml.
+#
+# Env in:
+#   VERSION        - "latest" or a release tag, e.g. "cli@1.0.3" (also accepts a
+#                    bare version like "1.0.3")
+#   RUNNER_OS       - set by GitHub Actions ("Linux" / "macOS" / "Windows")
+#   RUNNER_ARCH     - set by GitHub Actions ("X64" / "ARM64" / ...)
+#   RUNNER_TEMP     - set by GitHub Actions; falls back to /tmp for local runs
+#   GITHUB_PATH     - GitHub Actions path file (appended to)
+#   GITHUB_OUTPUT   - GitHub Actions output file (appended to)
+#
+# Outputs (via $GITHUB_OUTPUT):
+#   bin     - absolute path to the installed devtoolscli binary
+#   version - resolved version number (without the "cli@" prefix)
+set -euo pipefail
+
+REPO_OWNER='the-dev-tools'
+REPO_NAME='dev-tools'
+REPO_URL="https://github.com/${REPO_OWNER}/${REPO_NAME}"
+
+version_input="${VERSION:-latest}"
+
+case "${RUNNER_OS:-}" in
+  Linux) os='linux' ;;
+  macOS) os='darwin' ;;
+  *)
+    echo "::error::run-flows only supports Linux and macOS runners (got RUNNER_OS='${RUNNER_OS:-<unset>}'). Windows is out of scope — see actions/run-flows/README.md." >&2
+    exit 1
+    ;;
+esac
+
+case "${RUNNER_ARCH:-}" in
+  X64) arch='x64' ;;
+  ARM64) arch='arm64' ;;
+  *)
+    echo "::error::run-flows only supports X64 and ARM64 runners (got RUNNER_ARCH='${RUNNER_ARCH:-<unset>}')." >&2
+    exit 1
+    ;;
+esac
+
+platform="${os}-${arch}"
+
+# Resolve "latest"/bare-version/tag input into a concrete release tag. Releases
+# for the CLI are tagged "cli@<version>" (see tools/gha-scripts/src/cli.ts and
+# .github/workflows/release-go.yaml); the repo also cuts desktop@/web@ releases
+# on the same tracker, so "latest release" APIs can't be used as-is.
+if [[ "$version_input" == 'latest' ]]; then
+  set +e
+  tag=$(git ls-remote --tags --refs "${REPO_URL}.git" 'cli@*' 2>/dev/null | sed 's#.*refs/tags/##' | sort -V | tail -n1)
+  set -e
+  if [[ -z "$tag" ]]; then
+    echo "::error::Could not resolve the latest devtoolscli release: no cli@* tags found on ${REPO_URL} (or the network request failed)." >&2
+    exit 1
+  fi
+elif [[ "$version_input" == cli@* ]]; then
+  tag="$version_input"
+else
+  tag="cli@${version_input}"
+fi
+version_number="${tag#cli@}"
+
+asset_name="devtools-cli-${version_number}-${platform}"
+download_url="${REPO_URL}/releases/download/${tag}/${asset_name}"
+
+echo "Resolving devtoolscli ${tag} for ${platform}..."
+
+if ! curl -fsSI -o /dev/null "$download_url"; then
+  echo "::error::devtoolscli release asset not found: ${download_url}" >&2
+  echo "::error::Checked tag '${tag}' (from version input '${version_input}'). Confirm it exists at ${REPO_URL}/releases/tag/${tag} and that asset naming still matches 'devtools-cli-<version>-<os>-<arch>' (see .github/workflows/release-go.yaml)." >&2
+  exit 1
+fi
+
+bin_dir="${RUNNER_TEMP:-/tmp}/devtools/bin"
+bin_path="${bin_dir}/devtoolscli"
+mkdir -p "$bin_dir"
+
+if ! curl -fsSL -o "$bin_path" "$download_url"; then
+  echo "::error::Failed to download ${download_url}" >&2
+  exit 1
+fi
+chmod +x "$bin_path"
+
+echo "Installed devtoolscli ${version_number} -> ${bin_path}"
+"$bin_path" version
+
+echo "$bin_dir" >> "$GITHUB_PATH"
+{
+  echo "bin=${bin_path}"
+  echo "version=${version_number}"
+} >> "$GITHUB_OUTPUT"

actions/run-flows/scripts/finalize.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Sets the action's outputs (json-report, junit-report, success) and enforces
+# the fail-on-error input. Runs with `if: always()` in action.yml so outputs
+# are always set, even when the flow run failed. Deliberately does not use
+# `set -e`: a failure here should degrade gracefully rather than abort the
+# composite action before outputs get set, and the `[[ ... ]] &&` idiom at
+# the success-assignment line below is a `set -e` footgun (a false test
+# exits the whole expression non-zero, which -e treats as fatal).
+#
+# Env in:
+#   REPORT_DIR     - directory containing report.json/junit.xml (inputs.report-dir)
+#   RUN_OUTCOME    - outcome of the "Run flow" step ("success"/"failure"/"")
+#   FAIL_ON_ERROR  - inputs.fail-on-error ("true"/"false")
+set -uo pipefail
+
+json_report="${REPORT_DIR:-.devtools-reports}/report.json"
+junit_report="${REPORT_DIR:-.devtools-reports}/junit.xml"
+
+[[ -f "$json_report" ]] || json_report=''
+[[ -f "$junit_report" ]] || junit_report=''
+
+success='false'
+[[ "${RUN_OUTCOME:-}" == 'success' ]] && success='true'
+
+{
+  echo "json-report=${json_report}"
+  echo "junit-report=${junit_report}"
+  echo "success=${success}"
+} >> "$GITHUB_OUTPUT"
+
+if [[ "$success" == 'false' && "${FAIL_ON_ERROR:-true}" == 'true' ]]; then
+  echo "::error::devtoolscli flow run did not complete successfully (run step outcome: ${RUN_OUTCOME:-unknown}) and fail-on-error is 'true'." >&2
+  exit 1
+fi
+
+exit 0