test: test harness, testscript CLI tests, coverage floor and nightly smoke test

.github/workflows/checks.yml

@@ -50,7 +50,12 @@ jobs:
       - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
         with:
           go-version-file: go.mod
-      - run: go test -race ./...
+      - if: matrix.os != 'ubuntu-latest'
+        run: go test -race ./...
+      # On Linux the same tests run with coverage, and fail below the floor.
+      - name: Tests with coverage floor
+        if: matrix.os == 'ubuntu-latest'
+        run: make cover-check
       - run: go build -trimpath -o "$BIN" ./cmd/devlyft
       - run: '"./$BIN" version'
       - name: Completion scripts generate
@@ -66,10 +71,11 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
-      - uses: golang/govulncheck-action@032d45514ae346b1db93c04b0c90b841c370344f # v1.1.0
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
         with:
           go-version-file: go.mod
-          go-package: ./...
+      # The Makefile scopes the scan to shipped code (see the vuln target).
+      - run: make vuln
 
   # Builds every release target without signing or publishing, so changes to
   # .goreleaser.yaml are caught before a tag is pushed.

.github/workflows/smoke.yml

@@ -0,0 +1,48 @@
+name: Smoke
+
+# End-to-end smoke test against the production API, using a dedicated test
+# organization. It needs secrets, so it never runs for pull requests (forks
+# must not reach them): only nightly, by hand, or called from another
+# workflow in this repository (e.g. a release).
+#
+# Secrets live in the protected `smoke` environment (restricted to main,
+# with required reviewers if wanted): DEVLYFT_SMOKE_API_TOKEN (an org API
+# token for the test org) and DEVLYFT_SMOKE_ORG (the test org's UUID). See
+# docs/testing.md.
+on:
+  schedule:
+    - cron: "17 3 * * *"
+  workflow_dispatch:
+  workflow_call:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: smoke
+  cancel-in-progress: false
+
+jobs:
+  smoke:
+    name: smoke
+    if: github.repository == 'devlyft/cli'
+    runs-on: ubuntu-latest
+    environment: smoke
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+      - name: Smoke test
+        env:
+          DEVLYFT_API_TOKEN: ${{ secrets.DEVLYFT_SMOKE_API_TOKEN }}
+          DEVLYFT_SMOKE_ORG: ${{ secrets.DEVLYFT_SMOKE_ORG }}
+        run: |
+          if [ -z "$DEVLYFT_API_TOKEN" ] || [ -z "$DEVLYFT_SMOKE_ORG" ]; then
+            echo "::error::The smoke environment is missing DEVLYFT_SMOKE_API_TOKEN or DEVLYFT_SMOKE_ORG"
+            exit 1
+          fi
+          go test -tags smoke -count=1 -v ./test/smoke/

.golangci.yml

@@ -1,5 +1,9 @@
 version: "2"
 
+run:
+  # Lint the smoke tests too (test/smoke).
+  build-tags: [smoke]
+
 linters:
   enable:
     - errorlint

Makefile

@@ -8,26 +8,37 @@ VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
 LDFLAGS := -s -w -X github.com/devlyft/cli/internal/version.Version=$(VERSION)
 
 .DEFAULT_GOAL := build
-.PHONY: build test cover lint fmt vuln tidy generate vendor-specs check snapshot clean help
+.PHONY: build test cover cover-check lint fmt vuln tidy generate vendor-specs check snapshot clean help
 
 build: ## Build the devlyft binary into bin/
 	go build -trimpath -ldflags '$(LDFLAGS)' -o $(BIN) ./cmd/devlyft
 
 test: ## Run unit tests with the race detector
 	go test -race ./...
 
-cover: ## Run tests and write coverage.out
-	go test -race -coverprofile=coverage.out -covermode=atomic ./...
-	go tool cover -func=coverage.out | tail -1
+COVER_MIN     ?= 80
+# Generated code, mocks and the test harness itself don't count.
+COVER_EXCLUDE ?= /internal/(api/(gen|mock)|testutil)/
+
+cover: ## Run tests and write coverage.out (all of internal/, across packages)
+	go test -race -covermode=atomic -coverpkg=./internal/... -coverprofile=coverage.out ./...
+
+cover-check: cover ## Fail if coverage of internal/ is below COVER_MIN percent
+	go run ./hack/covercheck -profile coverage.out -min $(COVER_MIN) -exclude '$(COVER_EXCLUDE)' -v
 
 lint: ## Run golangci-lint
 	go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) run
 
 fmt: ## Format code
 	go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) fmt
 
-vuln: ## Check dependencies for known vulnerabilities
-	go run golang.org/x/vuln/cmd/govulncheck@$(GOVULNCHECK_VERSION) ./...
+# govulncheck checks the code that ships. Like _test.go files, the test
+# harness in internal/testutil is left out: it runs kin-openapi's validator
+# over our own fixtures, and kin-openapi is pinned by oapi-codegen.
+VULN_PACKAGES = $(shell go list ./... | grep -v '/internal/testutil/')
+
+vuln: ## Check dependencies for known vulnerabilities in shipped code
+	go run golang.org/x/vuln/cmd/govulncheck@$(GOVULNCHECK_VERSION) $(VULN_PACKAGES)
 
 tidy: ## Tidy go.mod and go.sum
 	go mod tidy
@@ -39,7 +50,7 @@ vendor-specs: ## Re-vendor pruned specs from the platform repo (needs access; RE
 	hack/vendor-specs.sh $(REF)
 	$(MAKE) generate
 
-check: lint test vuln ## Run everything CI runs
+check: lint cover-check vuln ## Run everything CI runs
 
 snapshot: ## Build unsigned release archives for every platform into dist/
 	go run github.com/goreleaser/goreleaser/v2@$(GORELEASER_VERSION) release --snapshot --clean --skip=sign,sbom

README.md

@@ -73,9 +73,10 @@ Requires Go 1.26 or later.
 ```sh
 make build        # builds bin/devlyft
 make test         # unit tests with -race
+make cover-check  # tests plus the 80% coverage floor on internal/
 make lint         # golangci-lint
 make vuln         # govulncheck
-make check        # everything CI runs
+make check        # everything CI runs (see docs/testing.md)
 ```
 
 Layout:

docs/testing.md

@@ -0,0 +1,109 @@
+# Testing
+
+There is no local DevLyft stack, so the CLI is tested in four layers that need none, plus a smoke test against production. Everything except the smoke test runs on every pull request on Linux, macOS and Windows.
+
+| Layer | What it proves | Where | Runs |
+|---|---|---|---|
+| Unit | Commands do the right thing with the API's answers | `*_test.go` next to the code, gomock client | `make test`, every PR |
+| Contract | Every request the facade sends, and every canned response, matches the vendored OpenAPI specs | `internal/api/contract_test.go`, `internal/testutil/apitest` | every PR |
+| CLI behaviour | What a user sees: stdout, stderr and exit codes | golden files and testscript scripts in `internal/cli/testdata` | every PR |
+| Smoke | The real binary works against the real API | `test/smoke` (build tag `smoke`) | nightly, by hand, from a release |
+
+`make check` runs lint, tests, the coverage floor and govulncheck, as CI does.
+
+## Unit tests: gomock, never hand-rolled fakes
+
+Commands depend on `api.Client`, never on HTTP. Tests use the generated mock in `internal/api/mock` (regenerated by `make generate`), so a change to the interface breaks the build instead of silently leaving a fake behind.
+
+Commands get the client from `cmdutil.Factory.Client()`. `internal/testutil/clitest` runs the whole CLI in-process with a mock injected, buffered streams, and an optional terminal:
+
+```go
+func TestProjectsList(t *testing.T) {
+	m := mock.NewMockClient(gomock.NewController(t))
+	m.EXPECT().ListProjects(gomock.Any(), orgID).Return(projects, nil)
+
+	res := clitest.Run(t, &clitest.Options{Args: []string{"projects", "list", "--org", orgID}, API: m})
+	golden.Assert(t, "testdata/projects_list.golden", res.Transcript())
+}
+```
+
+`clitest.Options` also takes `TTY` (stdin, stdout and stderr look like a terminal, so tables, colour and prompts behave as they would interactively), `Stdin` (answers to prompts), `Env` (set with `t.Setenv`, so the test cannot be parallel), `Context` (for example a cancelled one, to test Ctrl-C) and `Commands` (extra commands, for fixtures).
+
+Tests of package `cli` that use `clitest` live in package `cli_test`, because `clitest` imports `cli`.
+
+## Contract tests
+
+`internal/testutil/apitest` starts an `httptest` server from the vendored specs in `internal/api/specs`. Each request must match an operation and its parameters and body; each canned 2xx response must match the documented schema. Anything else fails the test.
+
+```go
+srv := apitest.NewServer(t, map[string]apitest.Response{
+	"GetV1Organizations": {Body: `{"organizations":[]}`},
+	"GetV1DeploysDeployId": {Status: 404, Body: `{"error":"not found"}`},
+})
+c, _ := api.New(api.Options{BaseURL: srv.URL, HTTPClient: srv.Client()})
+```
+
+`TestContract` in `internal/api` calls every facade method and checks that every vendored operation is exercised, so the allowlist (`internal/api/specs/allowlist.yaml`) never carries an endpoint the CLI does not use. When a spec update changes a request or response shape the CLI relies on, this is the test that fails.
+
+## CLI behaviour: golden files and scripts
+
+**Golden files** (`internal/cli/testdata/*.golden`) pin the full output of an invocation: the command line, the exit code, stdout and stderr. They cover every output format, every error code and exit code, usage errors and confirmation prompts. After an intended change to output, regenerate and review:
+
+```sh
+go test ./internal/cli/ -run TestGolden -update
+git diff internal/cli/testdata
+```
+
+The `-update` flag comes from `internal/testutil/golden` and works in any package that uses `golden.Assert`.
+
+**Scripts** (`internal/cli/testdata/script/*.txtar`) run the compiled CLI as a separate process with [testscript](https://pkg.go.dev/github.com/rogpeppe/go-internal/testscript), so they see real exit codes, real stdout and stderr, and the environment. Besides the standard testscript commands they have:
+
+- `exitcode N devlyft ...`: run and require exit code N; stdout and stderr can then be checked with `stdout` and `stderr`.
+- `fakeapi`: serve the script's `api/` directory as a fake API and set `DEVLYFT_API_URL`. `GET /v1/projects` is answered with `api/GET/v1/projects.json`, anything else with 404. Unlike `apitest`, it does not validate against the specs; use it for end-to-end flows, and `apitest` for request shapes.
+
+```
+exitcode 2 devlyft nope
+! stdout .
+stderr '^Error: unknown command "nope" for "devlyft"$'
+```
+
+Run one script with `go test ./internal/cli/ -run TestScripts/version`.
+
+## Coverage
+
+CI fails if statement coverage of `internal/` falls below 80% (`make cover-check`). Coverage counts across packages, so code exercised through the CLI counts for the package it lives in. Generated clients, mocks and the test harness are excluded. `make cover` writes `coverage.out`; `go tool cover -html=coverage.out` shows what is missing.
+
+## Smoke test against production
+
+`test/smoke` builds the binary and talks to the production API as a dedicated test organization. It is behind the `smoke` build tag, so `go test ./...` never runs it, and its workflow (`.github/workflows/smoke.yml`) runs nightly, by hand, or when called from another workflow in this repository. It never runs for pull requests, so forks cannot reach its secrets.
+
+What it checks:
+
+- `TestBinary`: the CLI builds and `devlyft version` works.
+- `TestRead`: the token is accepted and the test organization and its projects can be read.
+- `TestDeployLifecycle`: creates a build and a throwaway deploy (`smoke-<unix time>`), reads it back, and deletes the deploy in cleanup. It changes production state, so it only runs with `DEVLYFT_SMOKE_WRITE=1` and is off in the workflow for now. The API has no build deletion, so each run leaves one build behind in the test project.
+
+Run it locally:
+
+```sh
+DEVLYFT_API_TOKEN=<org API token> DEVLYFT_SMOKE_ORG=<org UUID> \
+  go test -tags smoke -count=1 -v ./test/smoke/
+```
+
+Without those variables every test skips. `DEVLYFT_API_URL` points it elsewhere. The write test also needs `DEVLYFT_SMOKE_WRITE=1`, `DEVLYFT_SMOKE_PROJECT`, `DEVLYFT_SMOKE_ENV`, `DEVLYFT_SMOKE_LOCATION` and `DEVLYFT_SMOKE_IMAGE` (`repo:tag`).
+
+### One-off setup
+
+1. Create a dedicated organization for smoke tests, with one project and environment, and nothing else in it.
+2. Create an organization API token for it, with the shortest expiry that is practical, and a calendar reminder to rotate it.
+3. In `devlyft/cli` → Settings → Environments, create an environment named `smoke`:
+   - Deployment branches and tags: selected branches, `main` only.
+   - Required reviewers are optional; with them, nightly runs wait for approval.
+4. Add two environment secrets to `smoke`: `DEVLYFT_SMOKE_API_TOKEN` (the token) and `DEVLYFT_SMOKE_ORG` (the organization's UUID).
+5. Run the workflow once by hand (Actions → Smoke → Run workflow) to confirm.
+
+To run the smoke test on every release, add a job to `release.yml` with `uses: ./.github/workflows/smoke.yml` and `secrets: inherit` after the release job.
+
+## Vulnerability scanning
+
+`make vuln` runs govulncheck over the code that ships. Like `_test.go` files, the test harness in `internal/testutil` is left out: it runs kin-openapi's request validator over our own fixtures, and kin-openapi is held at v0.133.0 because oapi-codegen v2.5.1 (the platform's version) does not build against newer releases. Revisit when the platform moves to a newer oapi-codegen.

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/itchyny/gojq v0.12.19
 	github.com/oapi-codegen/runtime v1.6.0
+	github.com/rogpeppe/go-internal v1.16.0
 	github.com/spf13/cobra v1.10.2
 	github.com/zalando/go-keyring v0.2.8
 	go.uber.org/mock v0.6.0

go.sum

@@ -102,6 +102,8 @@ github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX
 github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g=
+github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=