diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2dbb608..1a300ba 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -50,7 +50,12 @@ jobs: - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod - - run: go test -race ./... + - if: matrix.os != 'ubuntu-latest' + run: go test -race ./... + # On Linux the same tests run with coverage, and fail below the floor. + - name: Tests with coverage floor + if: matrix.os == 'ubuntu-latest' + run: make cover-check - run: go build -trimpath -o "$BIN" ./cmd/devlyft - run: '"./$BIN" version' - name: Completion scripts generate @@ -66,10 +71,11 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: golang/govulncheck-action@032d45514ae346b1db93c04b0c90b841c370344f # v1.1.0 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod - go-package: ./... + # The Makefile scopes the scan to shipped code (see the vuln target). + - run: make vuln # Builds every release target without signing or publishing, so changes to # .goreleaser.yaml are caught before a tag is pushed. diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml new file mode 100644 index 0000000..fa2647e --- /dev/null +++ b/.github/workflows/smoke.yml @@ -0,0 +1,48 @@ +name: Smoke + +# End-to-end smoke test against the production API, using a dedicated test +# organization. It needs secrets, so it never runs for pull requests (forks +# must not reach them): only nightly, by hand, or called from another +# workflow in this repository (e.g. a release). +# +# Secrets live in the protected `smoke` environment (restricted to main, +# with required reviewers if wanted): DEVLYFT_SMOKE_API_TOKEN (an org API +# token for the test org) and DEVLYFT_SMOKE_ORG (the test org's UUID). See +# docs/testing.md. +on: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + workflow_call: + +permissions: + contents: read + +concurrency: + group: smoke + cancel-in-progress: false + +jobs: + smoke: + name: smoke + if: github.repository == 'devlyft/cli' + runs-on: ubuntu-latest + environment: smoke + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - name: Smoke test + env: + DEVLYFT_API_TOKEN: ${{ secrets.DEVLYFT_SMOKE_API_TOKEN }} + DEVLYFT_SMOKE_ORG: ${{ secrets.DEVLYFT_SMOKE_ORG }} + run: | + if [ -z "$DEVLYFT_API_TOKEN" ] || [ -z "$DEVLYFT_SMOKE_ORG" ]; then + echo "::error::The smoke environment is missing DEVLYFT_SMOKE_API_TOKEN or DEVLYFT_SMOKE_ORG" + exit 1 + fi + go test -tags smoke -count=1 -v ./test/smoke/ diff --git a/.golangci.yml b/.golangci.yml index 1713570..8aacb89 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,5 +1,9 @@ version: "2" +run: + # Lint the smoke tests too (test/smoke). + build-tags: [smoke] + linters: enable: - errorlint diff --git a/Makefile b/Makefile index fc1a9ec..ac6f956 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) LDFLAGS := -s -w -X github.com/devlyft/cli/internal/version.Version=$(VERSION) .DEFAULT_GOAL := build -.PHONY: build test cover lint fmt vuln tidy generate vendor-specs check snapshot clean help +.PHONY: build test cover cover-check lint fmt vuln tidy generate vendor-specs check snapshot clean help build: ## Build the devlyft binary into bin/ go build -trimpath -ldflags '$(LDFLAGS)' -o $(BIN) ./cmd/devlyft @@ -16,9 +16,15 @@ build: ## Build the devlyft binary into bin/ test: ## Run unit tests with the race detector go test -race ./... -cover: ## Run tests and write coverage.out - go test -race -coverprofile=coverage.out -covermode=atomic ./... - go tool cover -func=coverage.out | tail -1 +COVER_MIN ?= 80 +# Generated code, mocks and the test harness itself don't count. +COVER_EXCLUDE ?= /internal/(api/(gen|mock)|testutil)/ + +cover: ## Run tests and write coverage.out (all of internal/, across packages) + go test -race -covermode=atomic -coverpkg=./internal/... -coverprofile=coverage.out ./... + +cover-check: cover ## Fail if coverage of internal/ is below COVER_MIN percent + go run ./hack/covercheck -profile coverage.out -min $(COVER_MIN) -exclude '$(COVER_EXCLUDE)' -v lint: ## Run golangci-lint go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) run @@ -26,8 +32,13 @@ lint: ## Run golangci-lint fmt: ## Format code go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) fmt -vuln: ## Check dependencies for known vulnerabilities - go run golang.org/x/vuln/cmd/govulncheck@$(GOVULNCHECK_VERSION) ./... +# govulncheck checks the code that ships. Like _test.go files, the test +# harness in internal/testutil is left out: it runs kin-openapi's validator +# over our own fixtures, and kin-openapi is pinned by oapi-codegen. +VULN_PACKAGES = $(shell go list ./... | grep -v '/internal/testutil/') + +vuln: ## Check dependencies for known vulnerabilities in shipped code + go run golang.org/x/vuln/cmd/govulncheck@$(GOVULNCHECK_VERSION) $(VULN_PACKAGES) tidy: ## Tidy go.mod and go.sum go mod tidy @@ -39,7 +50,7 @@ vendor-specs: ## Re-vendor pruned specs from the platform repo (needs access; RE hack/vendor-specs.sh $(REF) $(MAKE) generate -check: lint test vuln ## Run everything CI runs +check: lint cover-check vuln ## Run everything CI runs snapshot: ## Build unsigned release archives for every platform into dist/ go run github.com/goreleaser/goreleaser/v2@$(GORELEASER_VERSION) release --snapshot --clean --skip=sign,sbom diff --git a/README.md b/README.md index 3de8b6d..2f85de1 100644 --- a/README.md +++ b/README.md @@ -73,9 +73,10 @@ Requires Go 1.26 or later. ```sh make build # builds bin/devlyft make test # unit tests with -race +make cover-check # tests plus the 80% coverage floor on internal/ make lint # golangci-lint make vuln # govulncheck -make check # everything CI runs +make check # everything CI runs (see docs/testing.md) ``` Layout: diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..878aebf --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,109 @@ +# Testing + +There is no local DevLyft stack, so the CLI is tested in four layers that need none, plus a smoke test against production. Everything except the smoke test runs on every pull request on Linux, macOS and Windows. + +| Layer | What it proves | Where | Runs | +|---|---|---|---| +| Unit | Commands do the right thing with the API's answers | `*_test.go` next to the code, gomock client | `make test`, every PR | +| Contract | Every request the facade sends, and every canned response, matches the vendored OpenAPI specs | `internal/api/contract_test.go`, `internal/testutil/apitest` | every PR | +| CLI behaviour | What a user sees: stdout, stderr and exit codes | golden files and testscript scripts in `internal/cli/testdata` | every PR | +| Smoke | The real binary works against the real API | `test/smoke` (build tag `smoke`) | nightly, by hand, from a release | + +`make check` runs lint, tests, the coverage floor and govulncheck, as CI does. + +## Unit tests: gomock, never hand-rolled fakes + +Commands depend on `api.Client`, never on HTTP. Tests use the generated mock in `internal/api/mock` (regenerated by `make generate`), so a change to the interface breaks the build instead of silently leaving a fake behind. + +Commands get the client from `cmdutil.Factory.Client()`. `internal/testutil/clitest` runs the whole CLI in-process with a mock injected, buffered streams, and an optional terminal: + +```go +func TestProjectsList(t *testing.T) { + m := mock.NewMockClient(gomock.NewController(t)) + m.EXPECT().ListProjects(gomock.Any(), orgID).Return(projects, nil) + + res := clitest.Run(t, &clitest.Options{Args: []string{"projects", "list", "--org", orgID}, API: m}) + golden.Assert(t, "testdata/projects_list.golden", res.Transcript()) +} +``` + +`clitest.Options` also takes `TTY` (stdin, stdout and stderr look like a terminal, so tables, colour and prompts behave as they would interactively), `Stdin` (answers to prompts), `Env` (set with `t.Setenv`, so the test cannot be parallel), `Context` (for example a cancelled one, to test Ctrl-C) and `Commands` (extra commands, for fixtures). + +Tests of package `cli` that use `clitest` live in package `cli_test`, because `clitest` imports `cli`. + +## Contract tests + +`internal/testutil/apitest` starts an `httptest` server from the vendored specs in `internal/api/specs`. Each request must match an operation and its parameters and body; each canned 2xx response must match the documented schema. Anything else fails the test. + +```go +srv := apitest.NewServer(t, map[string]apitest.Response{ + "GetV1Organizations": {Body: `{"organizations":[]}`}, + "GetV1DeploysDeployId": {Status: 404, Body: `{"error":"not found"}`}, +}) +c, _ := api.New(api.Options{BaseURL: srv.URL, HTTPClient: srv.Client()}) +``` + +`TestContract` in `internal/api` calls every facade method and checks that every vendored operation is exercised, so the allowlist (`internal/api/specs/allowlist.yaml`) never carries an endpoint the CLI does not use. When a spec update changes a request or response shape the CLI relies on, this is the test that fails. + +## CLI behaviour: golden files and scripts + +**Golden files** (`internal/cli/testdata/*.golden`) pin the full output of an invocation: the command line, the exit code, stdout and stderr. They cover every output format, every error code and exit code, usage errors and confirmation prompts. After an intended change to output, regenerate and review: + +```sh +go test ./internal/cli/ -run TestGolden -update +git diff internal/cli/testdata +``` + +The `-update` flag comes from `internal/testutil/golden` and works in any package that uses `golden.Assert`. + +**Scripts** (`internal/cli/testdata/script/*.txtar`) run the compiled CLI as a separate process with [testscript](https://pkg.go.dev/github.com/rogpeppe/go-internal/testscript), so they see real exit codes, real stdout and stderr, and the environment. Besides the standard testscript commands they have: + +- `exitcode N devlyft ...`: run and require exit code N; stdout and stderr can then be checked with `stdout` and `stderr`. +- `fakeapi`: serve the script's `api/` directory as a fake API and set `DEVLYFT_API_URL`. `GET /v1/projects` is answered with `api/GET/v1/projects.json`, anything else with 404. Unlike `apitest`, it does not validate against the specs; use it for end-to-end flows, and `apitest` for request shapes. + +``` +exitcode 2 devlyft nope +! stdout . +stderr '^Error: unknown command "nope" for "devlyft"$' +``` + +Run one script with `go test ./internal/cli/ -run TestScripts/version`. + +## Coverage + +CI fails if statement coverage of `internal/` falls below 80% (`make cover-check`). Coverage counts across packages, so code exercised through the CLI counts for the package it lives in. Generated clients, mocks and the test harness are excluded. `make cover` writes `coverage.out`; `go tool cover -html=coverage.out` shows what is missing. + +## Smoke test against production + +`test/smoke` builds the binary and talks to the production API as a dedicated test organization. It is behind the `smoke` build tag, so `go test ./...` never runs it, and its workflow (`.github/workflows/smoke.yml`) runs nightly, by hand, or when called from another workflow in this repository. It never runs for pull requests, so forks cannot reach its secrets. + +What it checks: + +- `TestBinary`: the CLI builds and `devlyft version` works. +- `TestRead`: the token is accepted and the test organization and its projects can be read. +- `TestDeployLifecycle`: creates a build and a throwaway deploy (`smoke-`), reads it back, and deletes the deploy in cleanup. It changes production state, so it only runs with `DEVLYFT_SMOKE_WRITE=1` and is off in the workflow for now. The API has no build deletion, so each run leaves one build behind in the test project. + +Run it locally: + +```sh +DEVLYFT_API_TOKEN= DEVLYFT_SMOKE_ORG= \ + go test -tags smoke -count=1 -v ./test/smoke/ +``` + +Without those variables every test skips. `DEVLYFT_API_URL` points it elsewhere. The write test also needs `DEVLYFT_SMOKE_WRITE=1`, `DEVLYFT_SMOKE_PROJECT`, `DEVLYFT_SMOKE_ENV`, `DEVLYFT_SMOKE_LOCATION` and `DEVLYFT_SMOKE_IMAGE` (`repo:tag`). + +### One-off setup + +1. Create a dedicated organization for smoke tests, with one project and environment, and nothing else in it. +2. Create an organization API token for it, with the shortest expiry that is practical, and a calendar reminder to rotate it. +3. In `devlyft/cli` → Settings → Environments, create an environment named `smoke`: + - Deployment branches and tags: selected branches, `main` only. + - Required reviewers are optional; with them, nightly runs wait for approval. +4. Add two environment secrets to `smoke`: `DEVLYFT_SMOKE_API_TOKEN` (the token) and `DEVLYFT_SMOKE_ORG` (the organization's UUID). +5. Run the workflow once by hand (Actions → Smoke → Run workflow) to confirm. + +To run the smoke test on every release, add a job to `release.yml` with `uses: ./.github/workflows/smoke.yml` and `secrets: inherit` after the release job. + +## Vulnerability scanning + +`make vuln` runs govulncheck over the code that ships. Like `_test.go` files, the test harness in `internal/testutil` is left out: it runs kin-openapi's request validator over our own fixtures, and kin-openapi is held at v0.133.0 because oapi-codegen v2.5.1 (the platform's version) does not build against newer releases. Revisit when the platform moves to a newer oapi-codegen. diff --git a/go.mod b/go.mod index cdd7a83..fd28e01 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/google/uuid v1.6.0 github.com/itchyny/gojq v0.12.19 github.com/oapi-codegen/runtime v1.6.0 + github.com/rogpeppe/go-internal v1.16.0 github.com/spf13/cobra v1.10.2 github.com/zalando/go-keyring v0.2.8 go.uber.org/mock v0.6.0 diff --git a/go.sum b/go.sum index d75dac4..b6d23ec 100644 --- a/go.sum +++ b/go.sum @@ -102,6 +102,8 @@ github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= +github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= diff --git a/hack/covercheck/main.go b/hack/covercheck/main.go new file mode 100644 index 0000000..7d4fbd4 --- /dev/null +++ b/hack/covercheck/main.go @@ -0,0 +1,137 @@ +// Command covercheck enforces a minimum statement coverage on a Go cover +// profile, ignoring files that match -exclude. +// +// go test -coverpkg=./internal/... -coverprofile=coverage.out ./... +// go run ./hack/covercheck -profile coverage.out -min 80 -exclude '/internal/api/(gen|mock)/' +// +// With -coverpkg a block can appear once per test binary; a statement +// counts as covered if any binary covered it. +package main + +import ( + "bufio" + "bytes" + "flag" + "fmt" + "io" + "os" + "regexp" + "sort" + "strconv" + "strings" +) + +type block struct { + stmts int + covered bool +} + +func main() { + profile := flag.String("profile", "coverage.out", "cover profile to read") + minPct := flag.Float64("min", 80, "minimum total statement coverage, in percent") + exclude := flag.String("exclude", "", "regexp of file paths to ignore") + verbose := flag.Bool("v", false, "print coverage per package") + flag.Parse() + + var re *regexp.Regexp + if *exclude != "" { + re = regexp.MustCompile(*exclude) + } + data, err := os.ReadFile(*profile) + if err != nil { + fail(err) + } + + blocks, err := parse(bytes.NewReader(data), re) + if err != nil { + fail(err) + } + total, covered, perPkg := summarise(blocks) + if total == 0 { + fail(fmt.Errorf("%s: no statements after exclusions", *profile)) + } + if *verbose { + pkgs := make([]string, 0, len(perPkg)) + for p := range perPkg { + pkgs = append(pkgs, p) + } + sort.Strings(pkgs) + for _, p := range pkgs { + c := perPkg[p] + fmt.Printf("%6.1f%% %s\n", pct(c[1], c[0]), p) + } + } + got := pct(covered, total) + fmt.Printf("total coverage: %.1f%% of %d statements (minimum %.1f%%)\n", got, total, *minPct) + if got < *minPct { + fmt.Fprintf(os.Stderr, "coverage %.1f%% is below the %.1f%% floor\n", got, *minPct) + os.Exit(1) + } +} + +// parse reads a cover profile, merging duplicate blocks. +func parse(r io.Reader, exclude *regexp.Regexp) (map[string]*block, error) { + blocks := make(map[string]*block) + sc := bufio.NewScanner(r) + for sc.Scan() { + line := sc.Text() + if strings.HasPrefix(line, "mode:") || line == "" { + continue + } + // file.go:start.col,end.col numStmts count + fields := strings.Fields(line) + if len(fields) != 3 { + return nil, fmt.Errorf("malformed profile line %q", line) + } + file := fields[0][:strings.LastIndex(fields[0], ":")] + if exclude != nil && exclude.MatchString(file) { + continue + } + stmts, err := strconv.Atoi(fields[1]) + if err != nil { + return nil, fmt.Errorf("malformed profile line %q", line) + } + count, err := strconv.Atoi(fields[2]) + if err != nil { + return nil, fmt.Errorf("malformed profile line %q", line) + } + b, ok := blocks[fields[0]] + if !ok { + b = &block{stmts: stmts} + blocks[fields[0]] = b + } + b.covered = b.covered || count > 0 + } + return blocks, sc.Err() +} + +// summarise totals statements overall and per package directory; per +// package values are {total, covered}. +func summarise(blocks map[string]*block) (total, covered int, perPkg map[string][2]int) { + perPkg = make(map[string][2]int) + for key, b := range blocks { + file := key[:strings.LastIndex(key, ":")] + pkg := file[:strings.LastIndex(file, "/")] + c := perPkg[pkg] + c[0] += b.stmts + total += b.stmts + if b.covered { + c[1] += b.stmts + covered += b.stmts + } + perPkg[pkg] = c + } + return total, covered, perPkg +} + +func pct(covered, total int) float64 { + if total == 0 { + return 100 + } + return 100 * float64(covered) / float64(total) +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, "covercheck:", err) + os.Exit(1) +} diff --git a/hack/covercheck/main_test.go b/hack/covercheck/main_test.go new file mode 100644 index 0000000..41789aa --- /dev/null +++ b/hack/covercheck/main_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "regexp" + "strings" + "testing" +) + +func TestParseMergesAndExcludes(t *testing.T) { + t.Parallel() + + profile := `mode: atomic +github.com/devlyft/cli/internal/a/a.go:1.1,2.1 3 0 +github.com/devlyft/cli/internal/a/a.go:1.1,2.1 3 1 +github.com/devlyft/cli/internal/a/a.go:3.1,4.1 1 0 +github.com/devlyft/cli/internal/b/b.go:1.1,2.1 4 2 +github.com/devlyft/cli/internal/api/gen/x/x.go:1.1,2.1 100 0 +` + blocks, err := parse(strings.NewReader(profile), regexp.MustCompile(`/internal/api/gen/`)) + if err != nil { + t.Fatal(err) + } + total, covered, perPkg := summarise(blocks) + if total != 8 || covered != 7 { + t.Errorf("total, covered = %d, %d; want 8, 7", total, covered) + } + if got := perPkg["github.com/devlyft/cli/internal/a"]; got != [2]int{4, 3} { + t.Errorf("package a = %v, want [4 3]", got) + } +} + +func TestParseRejectsMalformedLines(t *testing.T) { + t.Parallel() + + for _, line := range []string{"x.go:1.1,2.1 3", "x.go:1.1,2.1 three 1", "x.go:1.1,2.1 3 one"} { + if _, err := parse(strings.NewReader(line+"\n"), nil); err == nil { + t.Errorf("parse(%q) succeeded, want error", line) + } + } +} + +func TestPct(t *testing.T) { + t.Parallel() + + if got := pct(0, 0); got != 100 { + t.Errorf("pct(0, 0) = %v, want 100", got) + } + if got := pct(1, 4); got != 25 { + t.Errorf("pct(1, 4) = %v, want 25", got) + } +} diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go index fde33ef..fd55a2d 100644 --- a/internal/api/contract_test.go +++ b/internal/api/contract_test.go @@ -1,21 +1,10 @@ package api import ( - "bytes" - "context" - "io" - "net/http" - "net/http/httptest" - "path/filepath" "slices" - "strings" - "sync" "testing" - "github.com/getkin/kin-openapi/openapi3" - "github.com/getkin/kin-openapi/openapi3filter" - "github.com/getkin/kin-openapi/routers" - "github.com/getkin/kin-openapi/routers/legacy" + "github.com/devlyft/cli/internal/testutil/apitest" ) // Contract tests: every request the facade sends must match an operation @@ -77,106 +66,19 @@ var responses = map[string]string{ "GetV1PodsPodIdMetrics": metricsJSON, } -// contractServer validates every exchange against the vendored specs. -type contractServer struct { - t *testing.T - routers []routers.Router - mu sync.Mutex - seen []string -} - -func newContractServer(t *testing.T) (*contractServer, *httptest.Server) { - t.Helper() - cs := &contractServer{t: t} - srv := httptest.NewServer(cs) - t.Cleanup(srv.Close) - - for _, name := range []string{"deploys", "org", "logs", "metrics"} { - loader := openapi3.NewLoader() - doc, err := loader.LoadFromFile(filepath.Join("specs", name+".yaml")) - if err != nil { - t.Fatalf("loading %s spec: %v", name, err) - } - if err := doc.Validate(loader.Context); err != nil { - t.Fatalf("%s spec is invalid: %v", name, err) - } - doc.Servers = openapi3.Servers{{URL: srv.URL}} - r, err := legacy.NewRouter(doc) - if err != nil { - t.Fatal(err) - } - cs.routers = append(cs.routers, r) - } - return cs, srv -} - -func (cs *contractServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { - // Server-side URLs have no scheme or host; the router matches them - // against the spec's server URL. - req.URL.Scheme, req.URL.Host = "http", req.Host - var route *routers.Route - var params map[string]string - for _, r := range cs.routers { - if rt, p, err := r.FindRoute(req); err == nil { - route, params = rt, p - break - } - } - if route == nil { - cs.t.Errorf("%s %s matches no vendored operation", req.Method, req.URL.Path) - http.Error(w, "no route", http.StatusNotFound) - return - } - id := route.Operation.OperationID - cs.mu.Lock() - cs.seen = append(cs.seen, id) - cs.mu.Unlock() - - ctx := context.Background() //nolint:usetesting // validation outlives no test - in := &openapi3filter.RequestValidationInput{ - Request: req, - PathParams: params, - Route: route, - Options: &openapi3filter.Options{MultiError: true}, - } - if err := openapi3filter.ValidateRequest(ctx, in); err != nil { - cs.t.Errorf("%s: request does not match the spec: %v", id, err) - } - - body := responses[id] - status := successStatus(route.Operation) - out := &openapi3filter.ResponseValidationInput{ - RequestValidationInput: in, - Status: status, - Header: http.Header{"Content-Type": {"application/json"}}, - Body: io.NopCloser(strings.NewReader(body)), - Options: &openapi3filter.Options{MultiError: true, IncludeResponseStatus: true}, - } - if err := openapi3filter.ValidateResponse(ctx, out); err != nil { - cs.t.Errorf("%s: canned response does not match the spec: %v", id, err) - } - - if body != "" { - w.Header().Set("Content-Type", "application/json") - } - w.WriteHeader(status) - _, _ = io.Copy(w, bytes.NewBufferString(body)) -} - -// successStatus is the operation's 2xx response code. -func successStatus(op *openapi3.Operation) int { - for _, code := range []int{200, 201, 204} { - if op.Responses.Status(code) != nil { - return code - } +// canned turns the response bodies into apitest responses. +func canned() map[string]apitest.Response { + out := make(map[string]apitest.Response, len(responses)) + for id, body := range responses { + out[id] = apitest.Response{Body: body} } - return 200 + return out } func TestContract(t *testing.T) { t.Parallel() - cs, srv := newContractServer(t) + srv := apitest.NewServer(t, canned()) c, err := New(Options{BaseURL: srv.URL, Tokens: StaticToken("t"), HTTPClient: srv.Client()}) if err != nil { t.Fatal(err) @@ -244,7 +146,7 @@ func TestContract(t *testing.T) { // Every vendored operation is exercised, so the allowlist carries // nothing the CLI does not use. for id := range responses { - if !slices.Contains(cs.seen, id) { + if !slices.Contains(srv.Seen(), id) { t.Errorf("operation %s was never called", id) } } diff --git a/internal/cli/auth_test.go b/internal/cli/auth_test.go index c1abc59..9cc9c3a 100644 --- a/internal/cli/auth_test.go +++ b/internal/cli/auth_test.go @@ -21,6 +21,7 @@ import ( "github.com/devlyft/cli/internal/cmdutil" "github.com/devlyft/cli/internal/exitcode" "github.com/devlyft/cli/internal/iostreams" + "github.com/devlyft/cli/internal/testutil/golden" ) //nolint:gosec // fake tokens @@ -307,7 +308,7 @@ func TestAuthGolden(t *testing.T) { } got := fmt.Sprintf("$ devlyft %s\n--- exit %d\n--- stdout\n%s--- stderr\n%s", strings.Join(tt.args, " "), code, stdout.String(), stderr.String()) - assertGolden(t, filepath.Join("testdata", tt.name+".golden"), got) + golden.Assert(t, filepath.Join("testdata", tt.name+".golden"), got) }) } } diff --git a/internal/cli/golden_test.go b/internal/cli/golden_test.go new file mode 100644 index 0000000..520982c --- /dev/null +++ b/internal/cli/golden_test.go @@ -0,0 +1,259 @@ +package cli_test + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/google/uuid" + "github.com/spf13/cobra" + "go.uber.org/mock/gomock" + + "github.com/devlyft/cli/internal/api" + "github.com/devlyft/cli/internal/api/mock" + "github.com/devlyft/cli/internal/clierr" + "github.com/devlyft/cli/internal/cmdutil" + "github.com/devlyft/cli/internal/output" + "github.com/devlyft/cli/internal/prompt" + "github.com/devlyft/cli/internal/testutil/clitest" + "github.com/devlyft/cli/internal/testutil/golden" +) + +type widget struct { + ID string `json:"id"` + Name string `json:"name"` + Labels map[string]string `json:"labels,omitempty"` +} + +var widgets = []widget{ + {ID: "0b1c", Name: "web", Labels: map[string]string{"tier": "frontend"}}, + {ID: "7f2e", Name: "worker\twith tab"}, +} + +const orgID = "0b4e1a3c-6c2f-4d0a-8c51-5f0f1e2d3c4b" + +var ( + fixtureTime = time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + fixtureProjects = []api.Project{ + {Id: uuid.MustParse("7d9f2a10-3b4c-4e5f-8a6b-1c2d3e4f5a6b"), OrganizationId: uuid.MustParse(orgID), Name: "web", CreatedAt: fixtureTime, UpdatedAt: fixtureTime}, + {Id: uuid.MustParse("9e8d7c6b-5a49-4382-a1b0-c9d8e7f6a5b4"), OrganizationId: uuid.MustParse(orgID), Name: "worker", CreatedAt: fixtureTime, UpdatedAt: fixtureTime}, + } +) + +// fixtureCommands are stand-ins for real commands, exercising the printer, +// the error mapping and confirmation. +func fixtureCommands(f *cmdutil.Factory) []*cobra.Command { + list := &cobra.Command{ + Use: "widgets", + RunE: func(*cobra.Command, []string) error { + t := &output.Table{Headers: []string{"id", "name"}} + ids := make([]string, 0, len(widgets)) + for _, w := range widgets { + t.Rows = append(t.Rows, []string{w.ID, w.Name}) + ids = append(ids, w.ID) + } + return f.Printer.Print(output.Result{Data: widgets, Table: t, IDs: ids}) + }, + } + describe := &cobra.Command{ + Use: "widget", + RunE: func(*cobra.Command, []string) error { + return f.Printer.Print(output.Result{Data: widgets[0], IDs: []string{widgets[0].ID}}) + }, + } + var status int + var code, requestID string + fail := &cobra.Command{ + Use: "fail", + Args: cobra.NoArgs, + RunE: func(*cobra.Command, []string) error { + switch { + case status != 0: + return clierr.FromHTTP(status, "server said no", requestID) + case code == "plain": + return errors.New("dial tcp: connection refused") + case code == "deadline": + return fmt.Errorf("waiting: %w", context.DeadlineExceeded) + case code == "canceled": + return context.Canceled + } + e := clierr.New(clierr.Code(code), "failed with %s", code) + e.Hint = "try again" + e.Details = map[string]any{"deploy": "web"} + return e + }, + } + fail.Flags().IntVar(&status, "status", 0, "") + fail.Flags().StringVar(&code, "code", "", "") + fail.Flags().StringVar(&requestID, "request-id", "", "") + + var yes bool + del := &cobra.Command{ + Use: "delete", + RunE: func(cmd *cobra.Command, _ []string) error { + ok, err := prompt.Confirm(f.IO, "Delete widget web?", yes) + if err != nil { + return err + } + if ok { + _, err = fmt.Fprintln(cmd.OutOrStdout(), "deleted") + } + return err + }, + } + del.Flags().BoolVar(&yes, "yes", false, cmdutil.YesFlagUsage) + + // projects goes through the factory's API client, as real commands do. + projects := &cobra.Command{ + Use: "projects ORG", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := f.Client() + if err != nil { + return err + } + ps, err := c.ListProjects(cmd.Context(), args[0]) + if err != nil { + return err + } + t := &output.Table{Headers: []string{"id", "name"}} + ids := make([]string, 0, len(ps)) + for i := range ps { + t.Rows = append(t.Rows, []string{ps[i].Id.String(), ps[i].Name}) + ids = append(ids, ps[i].Id.String()) + } + return f.Printer.Print(output.Result{Data: ps, Table: t, IDs: ids}) + }, + } + + return []*cobra.Command{list, describe, fail, del, projects} +} + +type goldenCase struct { + name string + args []string + tty bool + stdin string + ctx func(context.Context) context.Context + // mock sets expectations on the API client; nil runs without one. + mock func(*mock.MockClient) +} + +// client returns a gomock API client with the case's expectations, or nil. +func (c *goldenCase) client(t *testing.T) api.Client { + t.Helper() + if c.mock == nil { + return nil + } + m := mock.NewMockClient(gomock.NewController(t)) + c.mock(m) + return m +} + +//nolint:paralleltest // uses t.Setenv +func TestGolden(t *testing.T) { + t.Setenv("NO_COLOR", "") + t.Setenv("DEVLYFT_DEBUG", "") + t.Setenv("DEVLYFT_API_TOKEN", "") + + cancelled := func(ctx context.Context) context.Context { + ctx, cancel := context.WithCancel(ctx) + cancel() + return ctx + } + + tests := []goldenCase{ + // Formats. + {name: "list_default_pipe", args: []string{"widgets"}}, + {name: "list_default_tty", args: []string{"widgets"}, tty: true}, + {name: "list_default_tty_no_color", args: []string{"widgets", "--no-color"}, tty: true}, + {name: "list_table", args: []string{"widgets", "--format", "table"}}, + {name: "list_tsv", args: []string{"widgets", "--format=tsv"}, tty: true}, + {name: "list_json", args: []string{"widgets", "--format", "json"}}, + {name: "list_yaml", args: []string{"widgets", "--format", "yaml"}}, + {name: "list_jq_strings", args: []string{"widgets", "--jq", ".[].name"}}, + {name: "list_jq_objects", args: []string{"widgets", "--format", "json", "--jq", ".[] | {id}"}}, + {name: "list_quiet", args: []string{"widgets", "-q"}}, + {name: "describe_default", args: []string{"widget"}, tty: true}, + {name: "describe_json", args: []string{"widget", "--format", "json"}}, + + // Usage errors: exit 2. + {name: "usage_unknown_command", args: []string{"nope"}}, + {name: "usage_unknown_flag", args: []string{"widgets", "--bogus"}}, + {name: "usage_unknown_flag_json", args: []string{"widgets", "--bogus", "--format", "json"}}, + {name: "usage_extra_args", args: []string{"fail", "extra"}}, + {name: "usage_bad_format", args: []string{"widgets", "--format", "xml"}}, + {name: "usage_jq_with_yaml", args: []string{"widgets", "--format", "yaml", "--jq", "."}}, + {name: "usage_bad_jq", args: []string{"widgets", "--jq", ".["}}, + {name: "usage_quiet_with_json", args: []string{"widgets", "-q", "--format", "json"}}, + {name: "usage_bad_verbosity", args: []string{"widgets", "--verbosity", "loud"}}, + + // Each error code, human and JSON. + {name: "err_auth_required", args: []string{"fail", "--code", "auth_required"}}, + {name: "err_auth_required_json", args: []string{"fail", "--code", "auth_required", "--format", "json"}}, + {name: "err_permission_denied", args: []string{"fail", "--code", "permission_denied"}}, + {name: "err_not_found", args: []string{"fail", "--code", "not_found"}}, + {name: "err_conflict", args: []string{"fail", "--code", "conflict"}}, + {name: "err_invalid_argument", args: []string{"fail", "--code", "invalid_argument"}}, + {name: "err_timeout", args: []string{"fail", "--code", "timeout"}}, + {name: "err_placement_pending", args: []string{"fail", "--code", "placement_pending"}}, + {name: "err_rollout_failed", args: []string{"fail", "--code", "rollout_failed"}}, + {name: "err_unavailable", args: []string{"fail", "--code", "unavailable"}}, + {name: "err_internal", args: []string{"fail", "--code", "internal"}}, + {name: "err_plain", args: []string{"fail", "--code", "plain"}}, + {name: "err_deadline", args: []string{"fail", "--code", "deadline"}}, + + // API errors. + {name: "http_401", args: []string{"fail", "--status", "401"}}, + {name: "http_403_json", args: []string{"fail", "--status", "403", "--request-id", "req-123", "--jq", "."}}, + {name: "http_404", args: []string{"fail", "--status", "404", "--request-id", "req-123"}}, + {name: "http_409", args: []string{"fail", "--status", "409"}}, + {name: "http_412", args: []string{"fail", "--status", "412"}}, + {name: "http_422", args: []string{"fail", "--status", "422"}}, + {name: "http_429_json", args: []string{"fail", "--status", "429", "--format", "json"}}, + {name: "http_503", args: []string{"fail", "--status", "503"}}, + + // Interrupt: exit 130. + {name: "interrupted", args: []string{"fail", "--code", "canceled"}, ctx: cancelled}, + {name: "interrupted_json", args: []string{"fail", "--code", "canceled", "--format", "json"}, ctx: cancelled}, + + // Confirmation. + {name: "confirm_yes_flag", args: []string{"delete", "--yes"}}, + {name: "confirm_no_tty", args: []string{"delete"}}, + {name: "confirm_tty_yes", args: []string{"delete"}, tty: true, stdin: "y\n"}, + {name: "confirm_tty_no", args: []string{"delete"}, tty: true, stdin: "\n"}, + + // API client injected through the factory (gomock). + {name: "api_list", args: []string{"projects", orgID}, mock: func(m *mock.MockClient) { + m.EXPECT().ListProjects(gomock.Any(), orgID).Return(fixtureProjects, nil) + }}, + {name: "api_list_json", args: []string{"projects", orgID, "--format", "json"}, mock: func(m *mock.MockClient) { + m.EXPECT().ListProjects(gomock.Any(), orgID).Return(fixtureProjects, nil) + }}, + {name: "api_not_found", args: []string{"projects", orgID}, mock: func(m *mock.MockClient) { + m.EXPECT().ListProjects(gomock.Any(), orgID).Return(nil, clierr.FromHTTP(404, "organization not found", "req-9")) + }}, + {name: "api_not_configured", args: []string{"projects", orgID}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := t.Context() + if tt.ctx != nil { + ctx = tt.ctx(ctx) + } + res := clitest.Run(t, &clitest.Options{ + Args: tt.args, + TTY: tt.tty, + Stdin: tt.stdin, + API: tt.client(t), + Commands: fixtureCommands, + Context: ctx, + }) + golden.Assert(t, filepath.Join("testdata", tt.name+".golden"), res.Transcript()) + }) + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 3e13105..6a0d00f 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -150,6 +150,15 @@ func Execute(ctx context.Context, ios *iostreams.IOStreams, args []string) int { return code } +// ExecuteWith runs the CLI in-process with a caller-supplied factory and +// any extra commands attached to the root, without the update check. Tests +// use it through internal/testutil/clitest. +func ExecuteWith(ctx context.Context, f *cmdutil.Factory, args []string, extra ...*cobra.Command) int { + root := NewRootCmd(f) + root.AddCommand(extra...) + return execute(ctx, f, root, args) +} + func execute(ctx context.Context, f *cmdutil.Factory, root *cobra.Command, args []string) int { ios := f.IO root.SetArgs(args) diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index be67e07..4692079 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -1,228 +1,12 @@ package cli import ( - "context" - "errors" - "flag" - "fmt" - "os" - "path/filepath" "strings" "testing" - "github.com/spf13/cobra" - - "github.com/devlyft/cli/internal/clierr" - "github.com/devlyft/cli/internal/cmdutil" "github.com/devlyft/cli/internal/iostreams" - "github.com/devlyft/cli/internal/output" - "github.com/devlyft/cli/internal/prompt" ) -var update = flag.Bool("update", false, "rewrite golden files") - -type widget struct { - ID string `json:"id"` - Name string `json:"name"` - Labels map[string]string `json:"labels,omitempty"` -} - -var widgets = []widget{ - {ID: "0b1c", Name: "web", Labels: map[string]string{"tier": "frontend"}}, - {ID: "7f2e", Name: "worker\twith tab"}, -} - -// fixtureCommands are stand-ins for real commands, exercising the printer, -// the error mapping and confirmation. -func fixtureCommands(f *cmdutil.Factory) []*cobra.Command { - list := &cobra.Command{ - Use: "widgets", - RunE: func(*cobra.Command, []string) error { - t := &output.Table{Headers: []string{"id", "name"}} - ids := make([]string, 0, len(widgets)) - for _, w := range widgets { - t.Rows = append(t.Rows, []string{w.ID, w.Name}) - ids = append(ids, w.ID) - } - return f.Printer.Print(output.Result{Data: widgets, Table: t, IDs: ids}) - }, - } - describe := &cobra.Command{ - Use: "widget", - RunE: func(*cobra.Command, []string) error { - return f.Printer.Print(output.Result{Data: widgets[0], IDs: []string{widgets[0].ID}}) - }, - } - var status int - var code, requestID string - fail := &cobra.Command{ - Use: "fail", - Args: cobra.NoArgs, - RunE: func(*cobra.Command, []string) error { - switch { - case status != 0: - return clierr.FromHTTP(status, "server said no", requestID) - case code == "plain": - return errors.New("dial tcp: connection refused") - case code == "deadline": - return fmt.Errorf("waiting: %w", context.DeadlineExceeded) - case code == "canceled": - return context.Canceled - } - e := clierr.New(clierr.Code(code), "failed with %s", code) - e.Hint = "try again" - e.Details = map[string]any{"deploy": "web"} - return e - }, - } - fail.Flags().IntVar(&status, "status", 0, "") - fail.Flags().StringVar(&code, "code", "", "") - fail.Flags().StringVar(&requestID, "request-id", "", "") - - var yes bool - del := &cobra.Command{ - Use: "delete", - RunE: func(cmd *cobra.Command, _ []string) error { - ok, err := prompt.Confirm(f.IO, "Delete widget web?", yes) - if err != nil { - return err - } - if ok { - _, err = fmt.Fprintln(cmd.OutOrStdout(), "deleted") - } - return err - }, - } - del.Flags().BoolVar(&yes, "yes", false, cmdutil.YesFlagUsage) - - return []*cobra.Command{list, describe, fail, del} -} - -type golden struct { - name string - args []string - tty bool - stdin string - ctx func(context.Context) context.Context -} - -//nolint:paralleltest // uses t.Setenv -func TestGolden(t *testing.T) { - t.Setenv("NO_COLOR", "") - t.Setenv("DEVLYFT_DEBUG", "") - t.Setenv("DEVLYFT_API_TOKEN", "") - - cancelled := func(ctx context.Context) context.Context { - ctx, cancel := context.WithCancel(ctx) - cancel() - return ctx - } - - tests := []golden{ - // Formats. - {name: "list_default_pipe", args: []string{"widgets"}}, - {name: "list_default_tty", args: []string{"widgets"}, tty: true}, - {name: "list_default_tty_no_color", args: []string{"widgets", "--no-color"}, tty: true}, - {name: "list_table", args: []string{"widgets", "--format", "table"}}, - {name: "list_tsv", args: []string{"widgets", "--format=tsv"}, tty: true}, - {name: "list_json", args: []string{"widgets", "--format", "json"}}, - {name: "list_yaml", args: []string{"widgets", "--format", "yaml"}}, - {name: "list_jq_strings", args: []string{"widgets", "--jq", ".[].name"}}, - {name: "list_jq_objects", args: []string{"widgets", "--format", "json", "--jq", ".[] | {id}"}}, - {name: "list_quiet", args: []string{"widgets", "-q"}}, - {name: "describe_default", args: []string{"widget"}, tty: true}, - {name: "describe_json", args: []string{"widget", "--format", "json"}}, - - // Usage errors: exit 2. - {name: "usage_unknown_command", args: []string{"nope"}}, - {name: "usage_unknown_flag", args: []string{"widgets", "--bogus"}}, - {name: "usage_unknown_flag_json", args: []string{"widgets", "--bogus", "--format", "json"}}, - {name: "usage_extra_args", args: []string{"fail", "extra"}}, - {name: "usage_bad_format", args: []string{"widgets", "--format", "xml"}}, - {name: "usage_jq_with_yaml", args: []string{"widgets", "--format", "yaml", "--jq", "."}}, - {name: "usage_bad_jq", args: []string{"widgets", "--jq", ".["}}, - {name: "usage_quiet_with_json", args: []string{"widgets", "-q", "--format", "json"}}, - {name: "usage_bad_verbosity", args: []string{"widgets", "--verbosity", "loud"}}, - - // Each error code, human and JSON. - {name: "err_auth_required", args: []string{"fail", "--code", "auth_required"}}, - {name: "err_auth_required_json", args: []string{"fail", "--code", "auth_required", "--format", "json"}}, - {name: "err_permission_denied", args: []string{"fail", "--code", "permission_denied"}}, - {name: "err_not_found", args: []string{"fail", "--code", "not_found"}}, - {name: "err_conflict", args: []string{"fail", "--code", "conflict"}}, - {name: "err_invalid_argument", args: []string{"fail", "--code", "invalid_argument"}}, - {name: "err_timeout", args: []string{"fail", "--code", "timeout"}}, - {name: "err_placement_pending", args: []string{"fail", "--code", "placement_pending"}}, - {name: "err_rollout_failed", args: []string{"fail", "--code", "rollout_failed"}}, - {name: "err_unavailable", args: []string{"fail", "--code", "unavailable"}}, - {name: "err_internal", args: []string{"fail", "--code", "internal"}}, - {name: "err_plain", args: []string{"fail", "--code", "plain"}}, - {name: "err_deadline", args: []string{"fail", "--code", "deadline"}}, - - // API errors. - {name: "http_401", args: []string{"fail", "--status", "401"}}, - {name: "http_403_json", args: []string{"fail", "--status", "403", "--request-id", "req-123", "--jq", "."}}, - {name: "http_404", args: []string{"fail", "--status", "404", "--request-id", "req-123"}}, - {name: "http_409", args: []string{"fail", "--status", "409"}}, - {name: "http_412", args: []string{"fail", "--status", "412"}}, - {name: "http_422", args: []string{"fail", "--status", "422"}}, - {name: "http_429_json", args: []string{"fail", "--status", "429", "--format", "json"}}, - {name: "http_503", args: []string{"fail", "--status", "503"}}, - - // Interrupt: exit 130. - {name: "interrupted", args: []string{"fail", "--code", "canceled"}, ctx: cancelled}, - {name: "interrupted_json", args: []string{"fail", "--code", "canceled", "--format", "json"}, ctx: cancelled}, - - // Confirmation. - {name: "confirm_yes_flag", args: []string{"delete", "--yes"}}, - {name: "confirm_no_tty", args: []string{"delete"}}, - {name: "confirm_tty_yes", args: []string{"delete"}, tty: true, stdin: "y\n"}, - {name: "confirm_tty_no", args: []string{"delete"}, tty: true, stdin: "\n"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ios, in, stdout, stderr := iostreams.Test() - ios.SetTTY(tt.tty) - in.WriteString(tt.stdin) - - f := &cmdutil.Factory{IO: ios} - root := NewRootCmd(f) - root.AddCommand(fixtureCommands(f)...) - - ctx := t.Context() - if tt.ctx != nil { - ctx = tt.ctx(ctx) - } - code := execute(ctx, f, root, tt.args) - - got := fmt.Sprintf("$ devlyft %s\n--- exit %d\n--- stdout\n%s--- stderr\n%s", - strings.Join(tt.args, " "), code, stdout.String(), stderr.String()) - assertGolden(t, filepath.Join("testdata", tt.name+".golden"), got) - }) - } -} - -func assertGolden(t *testing.T, path, got string) { - t.Helper() - if *update { - if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(path, []byte(got), 0o600); err != nil { - t.Fatal(err) - } - return - } - want, err := os.ReadFile(path) //nolint:gosec // test fixture path - if err != nil { - t.Fatalf("reading golden file (run with -update to create): %v", err) - } - if got != string(want) { - t.Errorf("output mismatch for %s (run with -update to accept)\n--- got\n%s\n--- want\n%s", path, got, want) - } -} - func TestExecuteVersion(t *testing.T) { t.Parallel() diff --git a/internal/cli/script_test.go b/internal/cli/script_test.go new file mode 100644 index 0000000..2e64c56 --- /dev/null +++ b/internal/cli/script_test.go @@ -0,0 +1,137 @@ +package cli_test + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/rogpeppe/go-internal/testscript" + + "github.com/devlyft/cli/internal/cli" + "github.com/devlyft/cli/internal/iostreams" +) + +// TestMain lets testscript run this test binary as the `devlyft` command, +// so scripts exercise the real entry point: process exit codes, real +// stdout and stderr, and environment variables. +func TestMain(m *testing.M) { + testscript.Main(m, map[string]func(){ + "devlyft": func() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + code := cli.Execute(ctx, iostreams.System(), os.Args[1:]) + stop() + os.Exit(code) + }, + }) +} + +// TestScripts runs testdata/script/*.txtar. Besides the standard commands, +// scripts can use: +// +// exitcode N prog [args...] run prog and require exit code N; its +// stdout and stderr are then checked as usual +// fakeapi serve $WORK/api as a fake API and point +// DEVLYFT_API_URL at it: GET /v1/x is answered +// with $WORK/api/GET/v1/x.json, 404 otherwise +func TestScripts(t *testing.T) { + t.Parallel() + testscript.Run(t, testscript.Params{ + Dir: filepath.Join("testdata", "script"), + Setup: func(env *testscript.Env) error { + // The update notifier is off without a terminal anyway; be explicit. + env.Setenv("DEVLYFT_NO_UPDATE_NOTIFIER", "1") + env.Setenv("NO_COLOR", "") + return nil + }, + Cmds: map[string]func(ts *testscript.TestScript, neg bool, args []string){ + "exitcode": cmdExitCode, + "fakeapi": cmdFakeAPI, + }, + }) +} + +func cmdExitCode(ts *testscript.TestScript, neg bool, args []string) { + if neg || len(args) < 2 { + ts.Fatalf("usage: exitcode N prog [args...]") + } + want, err := strconv.Atoi(args[0]) + if err != nil { + ts.Fatalf("exitcode: invalid code %q", args[0]) + } + got := 0 + if err := ts.Exec(args[1], args[2:]...); err != nil { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + ts.Fatalf("exitcode: %v", err) + } + got = exitErr.ExitCode() + } + if got != want { + ts.Fatalf("%s exited with %d, want %d", strings.Join(args[1:], " "), got, want) + } +} + +func cmdFakeAPI(ts *testscript.TestScript, neg bool, args []string) { + if neg || len(args) != 0 { + ts.Fatalf("usage: fakeapi") + } + srv := httptest.NewServer(fakeAPI(ts.MkAbs("api"))) + ts.Defer(srv.Close) + ts.Setenv("DEVLYFT_API_URL", srv.URL) +} + +// fakeAPI answers METHOD /path with root/METHOD/path.json. +func fakeAPI(root string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := filepath.Join(root, r.Method, filepath.FromSlash(strings.TrimPrefix(r.URL.Path, "/"))+".json") + body, err := os.ReadFile(path) //nolint:gosec // test fixture under $WORK + if err != nil { + http.Error(w, `{"error":"not found"}`, http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) //nolint:gosec // test fixture, not user input + }) +} + +func TestFakeAPI(t *testing.T) { + t.Parallel() + + root := t.TempDir() + dir := filepath.Join(root, "GET", "v1") + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "organizations.json"), []byte(`{"organizations":[]}`), 0o600); err != nil { + t.Fatal(err) + } + h := fakeAPI(root) + + tests := []struct { + method, path string + status int + body string + }{ + {http.MethodGet, "/v1/organizations", http.StatusOK, `{"organizations":[]}`}, + {http.MethodPost, "/v1/organizations", http.StatusNotFound, ""}, + {http.MethodGet, "/v1/projects", http.StatusNotFound, ""}, + } + for _, tt := range tests { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequestWithContext(t.Context(), tt.method, tt.path, http.NoBody)) + if rec.Code != tt.status { + t.Errorf("%s %s: status %d, want %d", tt.method, tt.path, rec.Code, tt.status) + } + if tt.body != "" && rec.Body.String() != tt.body { + t.Errorf("%s %s: body %q, want %q", tt.method, tt.path, rec.Body.String(), tt.body) + } + } +} diff --git a/internal/cli/testdata/api_list.golden b/internal/cli/testdata/api_list.golden new file mode 100644 index 0000000..72bca88 --- /dev/null +++ b/internal/cli/testdata/api_list.golden @@ -0,0 +1,6 @@ +$ devlyft projects 0b4e1a3c-6c2f-4d0a-8c51-5f0f1e2d3c4b +--- exit 0 +--- stdout +7d9f2a10-3b4c-4e5f-8a6b-1c2d3e4f5a6b web +9e8d7c6b-5a49-4382-a1b0-c9d8e7f6a5b4 worker +--- stderr diff --git a/internal/cli/testdata/api_list_json.golden b/internal/cli/testdata/api_list_json.golden new file mode 100644 index 0000000..55c8a2b --- /dev/null +++ b/internal/cli/testdata/api_list_json.golden @@ -0,0 +1,20 @@ +$ devlyft projects 0b4e1a3c-6c2f-4d0a-8c51-5f0f1e2d3c4b --format json +--- exit 0 +--- stdout +[ + { + "created_at": "2026-10-01T00:00:00Z", + "id": "7d9f2a10-3b4c-4e5f-8a6b-1c2d3e4f5a6b", + "name": "web", + "organization_id": "0b4e1a3c-6c2f-4d0a-8c51-5f0f1e2d3c4b", + "updated_at": "2026-10-01T00:00:00Z" + }, + { + "created_at": "2026-10-01T00:00:00Z", + "id": "9e8d7c6b-5a49-4382-a1b0-c9d8e7f6a5b4", + "name": "worker", + "organization_id": "0b4e1a3c-6c2f-4d0a-8c51-5f0f1e2d3c4b", + "updated_at": "2026-10-01T00:00:00Z" + } +] +--- stderr diff --git a/internal/cli/testdata/api_not_configured.golden b/internal/cli/testdata/api_not_configured.golden new file mode 100644 index 0000000..3e8f5f5 --- /dev/null +++ b/internal/cli/testdata/api_not_configured.golden @@ -0,0 +1,6 @@ +$ devlyft projects 0b4e1a3c-6c2f-4d0a-8c51-5f0f1e2d3c4b +--- exit 3 +--- stdout +--- stderr +Error: not logged in to api.devlyft.io +Hint: Run `devlyft auth login --with-token`, or set DEVLYFT_API_TOKEN. diff --git a/internal/cli/testdata/api_not_found.golden b/internal/cli/testdata/api_not_found.golden new file mode 100644 index 0000000..ab755c7 --- /dev/null +++ b/internal/cli/testdata/api_not_found.golden @@ -0,0 +1,6 @@ +$ devlyft projects 0b4e1a3c-6c2f-4d0a-8c51-5f0f1e2d3c4b +--- exit 4 +--- stdout +--- stderr +Error: organization not found +Request ID: req-9 diff --git a/internal/cli/testdata/script/completion.txtar b/internal/cli/testdata/script/completion.txtar new file mode 100644 index 0000000..7022aac --- /dev/null +++ b/internal/cli/testdata/script/completion.txtar @@ -0,0 +1,24 @@ +# Every supported shell gets a script on stdout that registers devlyft. +exec devlyft completion bash +stdout 'devlyft' +! stderr . + +exec devlyft completion zsh +stdout '#compdef devlyft' +! stderr . + +exec devlyft completion fish +stdout 'complete -c devlyft' +! stderr . + +exec devlyft completion powershell +stdout 'Register-ArgumentCompleter' +! stderr . + +# Unknown or missing shells are usage errors. +exitcode 2 devlyft completion tcsh +! stdout . +stderr 'invalid argument "tcsh"' + +exitcode 2 devlyft completion +! stdout . diff --git a/internal/cli/testdata/script/help.txtar b/internal/cli/testdata/script/help.txtar new file mode 100644 index 0000000..933650d --- /dev/null +++ b/internal/cli/testdata/script/help.txtar @@ -0,0 +1,12 @@ +# Help goes to stdout and exits 0. +exec devlyft --help +stdout '^devlyft is the official command-line interface for DevLyft\.' +stdout '^\s+completion\s' +stdout '^\s+version\s' +stdout '--format string' +stdout '--jq string' +! stderr . + +exec devlyft help version +stdout 'Print the devlyft version' +! stderr . diff --git a/internal/cli/testdata/script/usage_errors.txtar b/internal/cli/testdata/script/usage_errors.txtar new file mode 100644 index 0000000..82f05df --- /dev/null +++ b/internal/cli/testdata/script/usage_errors.txtar @@ -0,0 +1,30 @@ +# Mistakes caught locally exit 2, print one error and a hint to stderr, and +# nothing to stdout. + +exitcode 2 devlyft nope +! stdout . +stderr '^Error: unknown command "nope" for "devlyft"$' +stderr '^Hint: Run ''devlyft --help'' for usage\.$' + +exitcode 2 devlyft version --bogus +! stdout . +stderr '^Error: unknown flag: --bogus$' + +exitcode 2 devlyft version --format xml +! stdout . +stderr 'invalid --format "xml": must be one of table, tsv, json, yaml' + +exitcode 2 devlyft version --format yaml --jq . +stderr '--jq cannot be combined with --format yaml' + +exitcode 2 devlyft version --verbosity loud +stderr 'invalid --verbosity "loud"' + +# Under --format json the error is a single JSON object on stderr, even when +# the flags themselves are wrong. +exitcode 2 devlyft version --bogus --format json +! stdout . +stderr '^\{"code":"invalid_argument","message":"unknown flag: --bogus","http_status":null,"request_id":null,"details":null,"retryable":false\}$' + +exitcode 2 devlyft nope --jq . +stderr '^\{"code":"invalid_argument",' diff --git a/internal/cli/testdata/script/version.txtar b/internal/cli/testdata/script/version.txtar new file mode 100644 index 0000000..1e160c6 --- /dev/null +++ b/internal/cli/testdata/script/version.txtar @@ -0,0 +1,10 @@ +# version prints one line to stdout and nothing to stderr. +exec devlyft version +stdout '^devlyft \S+ .*go1\.\S+ \S+/\S+\n$' +! stderr . + +# Arguments are a usage error: exit 2 with a hint. +exitcode 2 devlyft version extra +! stdout . +stderr '^Error: unknown command "extra" for "devlyft version"' +stderr '^Hint: Run ''devlyft version --help'' for usage\.' diff --git a/internal/cmdutil/factory.go b/internal/cmdutil/factory.go index 877cb9f..74167ba 100644 --- a/internal/cmdutil/factory.go +++ b/internal/cmdutil/factory.go @@ -4,6 +4,7 @@ package cmdutil import ( + "errors" "log/slog" "github.com/devlyft/cli/internal/api" @@ -30,6 +31,10 @@ type Factory struct { Credentials func() (credstore.Store, error) // NewAPIClient returns an API client for APIURL using tokens. NewAPIClient func(tokens api.TokenSource) (api.Client, error) + + // API, when set, overrides Client. Tests use it to inject the gomock + // client from internal/api/mock. + API func() (api.Client, error) } // Host is the API host credentials are stored under. @@ -48,3 +53,18 @@ func (f *Factory) APIClient() (api.Client, error) { // YesFlagUsage is the help text for the --yes flag on destructive commands. const YesFlagUsage = "Skip the confirmation prompt (required when not running interactively)" + +// ErrNoAPI is returned by Client when no API client has been configured. +var ErrNoAPI = errors.New("no API client configured") + +// Client returns the API client commands should use: API when a test has +// set it, otherwise one authenticated with the resolved credential. +func (f *Factory) Client() (api.Client, error) { + if f.API != nil { + return f.API() + } + if f.NewAPIClient == nil { + return nil, ErrNoAPI + } + return f.APIClient() +} diff --git a/internal/testutil/apitest/apitest.go b/internal/testutil/apitest/apitest.go new file mode 100644 index 0000000..eb437e5 --- /dev/null +++ b/internal/testutil/apitest/apitest.go @@ -0,0 +1,180 @@ +// Package apitest provides a fake DevLyft API for tests. Every request it +// receives must match an operation in the vendored OpenAPI specs +// (internal/api/specs), and every canned success response must match the +// spec too, so a spec update that breaks the CLI fails the tests that use it. +package apitest + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "runtime" + "slices" + "strings" + "sync" + "testing" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/getkin/kin-openapi/openapi3filter" + "github.com/getkin/kin-openapi/routers" + "github.com/getkin/kin-openapi/routers/legacy" +) + +// Services are the vendored specs, by file name under internal/api/specs. +var Services = []string{"deploys", "org", "logs", "metrics"} + +// Response is a canned reply for one operation. +type Response struct { + // Status defaults to the operation's documented 2xx status. + Status int + // Body is JSON; empty sends no body. + Body string + // Header is added to the response. + Header http.Header +} + +// Server is a contract-checking fake API. +type Server struct { + *httptest.Server + + t testing.TB + routers []routers.Router + + mu sync.Mutex + responses map[string]Response + seen []string +} + +// NewServer starts a fake API answering each operation ID in responses. +// Requests for other operations, requests that do not match the spec and +// 2xx responses that do not match the spec fail t. The server is closed +// when the test ends. +func NewServer(tb testing.TB, responses map[string]Response) *Server { + tb.Helper() + if responses == nil { + responses = map[string]Response{} + } + s := &Server{t: tb, responses: responses} + s.Server = httptest.NewServer(s) + tb.Cleanup(s.Close) + + for _, name := range Services { + loader := openapi3.NewLoader() + doc, err := loader.LoadFromFile(SpecPath(name)) + if err != nil { + tb.Fatalf("loading %s spec: %v", name, err) + } + if err := doc.Validate(loader.Context); err != nil { + tb.Fatalf("%s spec is invalid: %v", name, err) + } + doc.Servers = openapi3.Servers{{URL: s.URL}} + r, err := legacy.NewRouter(doc) + if err != nil { + tb.Fatal(err) + } + s.routers = append(s.routers, r) + } + return s +} + +// SpecPath returns the path of a vendored spec, wherever the test runs. +func SpecPath(service string) string { + _, file, _, _ := runtime.Caller(0) + return filepath.Join(filepath.Dir(file), "..", "..", "api", "specs", service+".yaml") +} + +// Respond sets or replaces the response for an operation. +func (s *Server) Respond(operationID string, r Response) { + s.mu.Lock() + defer s.mu.Unlock() + s.responses[operationID] = r +} + +// Seen returns the operation IDs called so far, in order. +func (s *Server) Seen() []string { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.seen) +} + +// ServeHTTP validates the request, then replies with the canned response. +func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { + // Server-side URLs have no scheme or host; the router matches them + // against the spec's server URL. + req.URL.Scheme, req.URL.Host = "http", req.Host + var route *routers.Route + var params map[string]string + for _, r := range s.routers { + if rt, p, err := r.FindRoute(req); err == nil { + route, params = rt, p + break + } + } + if route == nil { + s.t.Errorf("%s %s matches no vendored operation", req.Method, req.URL.Path) + http.Error(w, "no route", http.StatusNotFound) + return + } + id := route.Operation.OperationID + + s.mu.Lock() + s.seen = append(s.seen, id) + resp, ok := s.responses[id] + s.mu.Unlock() + + ctx := context.Background() //nolint:usetesting // validation outlives no test + in := &openapi3filter.RequestValidationInput{ + Request: req, + PathParams: params, + Route: route, + Options: &openapi3filter.Options{MultiError: true}, + } + if err := openapi3filter.ValidateRequest(ctx, in); err != nil { + s.t.Errorf("%s: request does not match the spec: %v", id, err) + } + if !ok { + s.t.Errorf("%s: no canned response", id) + http.Error(w, "no canned response", http.StatusNotImplemented) + return + } + + status := resp.Status + if status == 0 { + status = SuccessStatus(route.Operation) + } + if status < http.StatusMultipleChoices { + out := &openapi3filter.ResponseValidationInput{ + RequestValidationInput: in, + Status: status, + Header: http.Header{"Content-Type": {"application/json"}}, + Body: io.NopCloser(strings.NewReader(resp.Body)), + Options: &openapi3filter.Options{MultiError: true, IncludeResponseStatus: true}, + } + if err := openapi3filter.ValidateResponse(ctx, out); err != nil { + s.t.Errorf("%s: canned response does not match the spec: %v", id, err) + } + } + + for k, vs := range resp.Header { + for _, v := range vs { + w.Header().Add(k, v) + } + } + if resp.Body != "" { + w.Header().Set("Content-Type", "application/json") + } + w.WriteHeader(status) + _, _ = io.WriteString(w, resp.Body) +} + +// SuccessStatus is the operation's documented 2xx response code. +func SuccessStatus(op *openapi3.Operation) int { + for _, code := range []int{http.StatusOK, http.StatusCreated, http.StatusNoContent} { + if op.Responses.Status(code) != nil { + return code + } + } + return http.StatusOK +} diff --git a/internal/testutil/clitest/clitest.go b/internal/testutil/clitest/clitest.go new file mode 100644 index 0000000..1fcd7db --- /dev/null +++ b/internal/testutil/clitest/clitest.go @@ -0,0 +1,104 @@ +// Package clitest runs the CLI in-process for tests: buffered streams, an +// optional terminal, and a gomock API client injected through the factory. +// +// ctrl := gomock.NewController(t) +// client := mock.NewMockClient(ctrl) +// client.EXPECT().ListProjects(gomock.Any(), orgID).Return(projects, nil) +// +// res := clitest.Run(t, &clitest.Options{Args: []string{"projects", "list"}, API: client}) +// golden.Assert(t, "testdata/projects_list.golden", res.Transcript()) +// +// Tests in package cli itself cannot import clitest (it imports cli); they +// live in package cli_test instead. +package clitest + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/spf13/cobra" + + "github.com/devlyft/cli/internal/api" + "github.com/devlyft/cli/internal/auth/credstore" + "github.com/devlyft/cli/internal/cli" + "github.com/devlyft/cli/internal/cmdutil" + "github.com/devlyft/cli/internal/iostreams" +) + +// Options describe one invocation. +type Options struct { + Args []string + // TTY makes stdin, stdout and stderr look like a terminal. + TTY bool + // Stdin is what the command reads, e.g. answers to prompts. + Stdin string + // Env is set for the duration of the test with t.Setenv, so a test that + // uses it cannot call t.Parallel. + Env map[string]string + // API is returned by the factory's API function, typically a + // *mock.MockClient. Nil leaves the factory without a client. + API api.Client + // Commands adds extra commands to the root, such as fixtures. + Commands func(*cmdutil.Factory) []*cobra.Command + // Context defaults to t.Context(). + Context context.Context +} + +// Result is what an invocation produced. +type Result struct { + Args []string + Exit int + Stdout string + Stderr string +} + +// Transcript formats the result for golden files. +func (r Result) Transcript() string { + return fmt.Sprintf("$ devlyft %s\n--- exit %d\n--- stdout\n%s--- stderr\n%s", + strings.Join(r.Args, " "), r.Exit, r.Stdout, r.Stderr) +} + +// Run executes the CLI with opts and returns its output and exit code. +func Run(tb testing.TB, opts *Options) Result { + tb.Helper() + for k, v := range opts.Env { + tb.Setenv(k, v) + } + + ios, in, stdout, stderr := iostreams.Test() + ios.SetTTY(opts.TTY) + in.WriteString(opts.Stdin) + + f := &cmdutil.Factory{IO: ios, Credentials: tempCredentials(tb)} + if opts.API != nil { + client := opts.API + f.API = func() (api.Client, error) { return client, nil } + } + var extra []*cobra.Command + if opts.Commands != nil { + extra = opts.Commands(f) + } + + ctx := opts.Context + if ctx == nil { + ctx = tb.Context() + } + code := cli.ExecuteWith(ctx, f, opts.Args, extra...) + return Result{Args: opts.Args, Exit: code, Stdout: stdout.String(), Stderr: stderr.String()} +} + +// tempCredentials returns an empty file-only credential store in a +// temporary directory, so tests never read or write the developer's own +// credentials or keyring. +func tempCredentials(tb testing.TB) func() (credstore.Store, error) { + tb.Helper() + dir := tb.TempDir() + env := credstore.Env{ + Getenv: func(k string) string { return map[string]string{"CI": "1"}[k] }, + FileExists: func(string) bool { return false }, + } + store := credstore.New(credstore.Options{Dir: dir, Env: &env}) + return func() (credstore.Store, error) { return store, nil } +} diff --git a/internal/testutil/doc.go b/internal/testutil/doc.go new file mode 100644 index 0000000..9eb08af --- /dev/null +++ b/internal/testutil/doc.go @@ -0,0 +1,11 @@ +// Package testutil is the root of the CLI's test harness. See +// docs/testing.md for how the layers fit together: +// +// - golden: golden-file assertions, regenerated with -update +// - apitest: an httptest server that validates every exchange against the +// vendored OpenAPI specs +// - clitest: runs the CLI in-process with injected streams and a gomock +// API client +// +// These packages are imported only by tests. +package testutil diff --git a/internal/testutil/golden/golden.go b/internal/testutil/golden/golden.go new file mode 100644 index 0000000..6f591d0 --- /dev/null +++ b/internal/testutil/golden/golden.go @@ -0,0 +1,36 @@ +// Package golden compares test output with files under testdata. +// +// Run `go test ./... -update` to rewrite the files from the current output, +// then review the diff before committing. +package golden + +import ( + "flag" + "os" + "path/filepath" + "testing" +) + +var update = flag.Bool("update", false, "rewrite golden files instead of comparing against them") + +// Assert fails tb if got differs from the contents of path. With -update it +// writes got to path instead. +func Assert(tb testing.TB, path, got string) { + tb.Helper() + if *update { + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + tb.Fatal(err) + } + if err := os.WriteFile(path, []byte(got), 0o600); err != nil { + tb.Fatal(err) + } + return + } + want, err := os.ReadFile(path) //nolint:gosec // test fixture path + if err != nil { + tb.Fatalf("reading golden file (run with -update to create): %v", err) + } + if got != string(want) { + tb.Errorf("output mismatch for %s (run with -update to accept)\n--- got\n%s\n--- want\n%s", path, got, want) + } +} diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go new file mode 100644 index 0000000..8ef9c61 --- /dev/null +++ b/test/smoke/smoke_test.go @@ -0,0 +1,145 @@ +//go:build smoke + +// Package smoke runs the CLI against the production API with a dedicated +// test organization. It is excluded from normal builds by the `smoke` build +// tag and runs nightly from .github/workflows/smoke.yml, never on pull +// requests. See docs/testing.md. +// +// DEVLYFT_API_TOKEN=... DEVLYFT_SMOKE_ORG= go test -tags smoke -v ./test/smoke/ +package smoke + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/devlyft/cli/internal/api" +) + +const defaultAPIURL = "https://api.devlyft.io" + +// env returns the variable or skips the test when it is unset. +func env(t *testing.T, name string) string { + t.Helper() + v := os.Getenv(name) + if v == "" { + t.Skipf("%s is not set", name) + } + return v +} + +func apiURL() string { + if u := os.Getenv("DEVLYFT_API_URL"); u != "" { + return u + } + return defaultAPIURL +} + +func newClient(t *testing.T) api.Client { + t.Helper() + c, err := api.New(api.Options{BaseURL: apiURL(), Tokens: api.StaticToken(env(t, "DEVLYFT_API_TOKEN"))}) + if err != nil { + t.Fatal(err) + } + return c +} + +// TestBinary builds the CLI and runs commands that need no API. +func TestBinary(t *testing.T) { + t.Parallel() + env(t, "DEVLYFT_API_TOKEN") + + bin := filepath.Join(t.TempDir(), "devlyft") + if runtime.GOOS == "windows" { + bin += ".exe" + } + build := exec.CommandContext(t.Context(), "go", "build", "-o", bin, "../../cmd/devlyft") //nolint:gosec // path under t.TempDir + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("go build: %v\n%s", err, out) + } + + out, err := exec.CommandContext(t.Context(), bin, "version").Output() //nolint:gosec // built above + if err != nil { + t.Fatalf("devlyft version: %v", err) + } + if !strings.HasPrefix(string(out), "devlyft ") { + t.Errorf("devlyft version printed %q", out) + } +} + +// TestRead checks the token works and the test organization is readable. +func TestRead(t *testing.T) { + t.Parallel() + c := newClient(t) + orgID := env(t, "DEVLYFT_SMOKE_ORG") + + org, err := c.GetOrganization(t.Context(), orgID) + if err != nil { + t.Fatalf("GetOrganization: %v", err) + } + t.Logf("organization %s (%s)", org.Name, org.Id) + + projects, err := c.ListProjects(t.Context(), orgID) + if err != nil { + t.Fatalf("ListProjects: %v", err) + } + t.Logf("%d projects", len(projects)) +} + +// TestDeployLifecycle creates a throwaway deploy and deletes it. It changes +// production state, so it only runs with DEVLYFT_SMOKE_WRITE=1 and the +// project, environment, location and image to use. +func TestDeployLifecycle(t *testing.T) { + t.Parallel() + if os.Getenv("DEVLYFT_SMOKE_WRITE") != "1" { + t.Skip("DEVLYFT_SMOKE_WRITE is not 1") + } + c := newClient(t) + projectID := env(t, "DEVLYFT_SMOKE_PROJECT") + envID := env(t, "DEVLYFT_SMOKE_ENV") + location := env(t, "DEVLYFT_SMOKE_LOCATION") + image := env(t, "DEVLYFT_SMOKE_IMAGE") // repo:tag + repo, tag, ok := strings.Cut(image, ":") + if !ok { + t.Fatalf("DEVLYFT_SMOKE_IMAGE %q must be repo:tag", image) + } + + name := fmt.Sprintf("smoke-%d", time.Now().Unix()) + port := 8080 + build, err := c.CreateBuild(t.Context(), projectID, &api.CreateBuildRequest{ + Name: name, Image: repo, Tag: tag, CpuRequest: "100m", MemoryRequest: "128Mi", ContainerPort: &port, + }) + if err != nil { + t.Fatalf("CreateBuild: %v", err) + } + + deploy, err := c.CreateDeploy(t.Context(), projectID, envID, &api.CreateDeployRequest{ + BuildId: build.Id, Name: name, LocationId: location, Replicas: 1, + }) + if err != nil { + t.Fatalf("CreateDeploy: %v", err) + } + deployID := deploy.Id.String() + t.Cleanup(func() { + // t.Context is already cancelled when cleanups run. + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) //nolint:usetesting // see above + defer cancel() + if err := c.DeleteDeploy(ctx, deployID); err != nil { + t.Errorf("DeleteDeploy %s: %v (delete it by hand)", deployID, err) + } + }) + + got, err := c.GetDeploy(t.Context(), deployID) + if err != nil { + t.Fatalf("GetDeploy: %v", err) + } + if got.Name != name { + t.Errorf("deploy name = %q, want %q", got.Name, name) + } +}