From 80dca1f076d42cf5260c449695ef8104d863cae6 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 18:04:38 +0300 Subject: [PATCH 01/38] feat: configurable max concurrency for flow runner CreateFlowRunner now accepts variadic functional options. Existing call sites (server flowexec session, CLI runner, tests) compile untouched and keep the CPU-derived default concurrency. WithMaxConcurrency(n) overrides the node-level parallelism cap; n <= 0 is ignored so callers can pass a config value through unconditionally. --- .../runner/flowlocalrunner/flowlocalrunner.go | 26 +++++++- .../flowlocalrunner_options_test.go | 63 +++++++++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner_options_test.go diff --git a/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner.go b/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner.go index 8fae92f62..2edf6d9b2 100644 --- a/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner.go +++ b/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner.go @@ -53,8 +53,23 @@ type FlowLocalRunner struct { var _ runner.FlowRunner = (*FlowLocalRunner)(nil) -func CreateFlowRunner(id, flowID idwrap.IDWrap, startNodeIDs []idwrap.IDWrap, flowNodeMap map[idwrap.IDWrap]node.FlowNode, edgesMap mflow.EdgesMap, timeout time.Duration, logger *slog.Logger) *FlowLocalRunner { - return &FlowLocalRunner{ +// Option customises a FlowLocalRunner at construction time. Passing no options +// yields the historical defaults, so existing call sites are unaffected. +type Option func(*FlowLocalRunner) + +// WithMaxConcurrency caps how many nodes the multi strategy executes in +// parallel. Values <= 0 are ignored so the CPU-derived default is kept. +func WithMaxConcurrency(n int) Option { + return func(r *FlowLocalRunner) { + if n <= 0 { + return + } + r.maxConcurrency = n + } +} + +func CreateFlowRunner(id, flowID idwrap.IDWrap, startNodeIDs []idwrap.IDWrap, flowNodeMap map[idwrap.IDWrap]node.FlowNode, edgesMap mflow.EdgesMap, timeout time.Duration, logger *slog.Logger, opts ...Option) *FlowLocalRunner { + r := &FlowLocalRunner{ ID: id, FlowID: flowID, FlowNodeMap: flowNodeMap, @@ -66,6 +81,13 @@ func CreateFlowRunner(id, flowID idwrap.IDWrap, startNodeIDs []idwrap.IDWrap, fl enableDataTracking: true, logger: logger, } + for _, opt := range opts { + if opt == nil { + continue + } + opt(r) + } + return r } // SetExecutionMode overrides the default Auto mode for the next run. diff --git a/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner_options_test.go b/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner_options_test.go new file mode 100644 index 000000000..918b3f92c --- /dev/null +++ b/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner_options_test.go @@ -0,0 +1,63 @@ +package flowlocalrunner + +import ( + "testing" + + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/node" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mflow" +) + +func newTestRunner(t *testing.T, opts ...Option) *FlowLocalRunner { + t.Helper() + + startID := idwrap.NewNow() + nodeMap := map[idwrap.IDWrap]node.FlowNode{} + edgesMap := mflow.EdgesMap{} + + return CreateFlowRunner(idwrap.NewNow(), idwrap.NewNow(), []idwrap.IDWrap{startID}, nodeMap, edgesMap, 0, nil, opts...) +} + +// TestCreateFlowRunnerDefaultUnchanged locks in that constructing a runner +// without options keeps the historical CPU-derived concurrency. +func TestCreateFlowRunnerDefaultUnchanged(t *testing.T) { + fr := newTestRunner(t) + + if fr.maxConcurrency != goroutineCount { + t.Fatalf("default maxConcurrency = %d, want %d (CPU-derived default)", fr.maxConcurrency, goroutineCount) + } +} + +func TestWithMaxConcurrency(t *testing.T) { + fr := newTestRunner(t, WithMaxConcurrency(3)) + + if fr.maxConcurrency != 3 { + t.Fatalf("maxConcurrency = %d, want 3", fr.maxConcurrency) + } +} + +func TestWithMaxConcurrencyNonPositiveIsNoOp(t *testing.T) { + for _, n := range []int{0, -1, -1024} { + fr := newTestRunner(t, WithMaxConcurrency(n)) + + if fr.maxConcurrency != goroutineCount { + t.Errorf("WithMaxConcurrency(%d): maxConcurrency = %d, want default %d", n, fr.maxConcurrency, goroutineCount) + } + } +} + +func TestOptionsAppliedInOrder(t *testing.T) { + fr := newTestRunner(t, WithMaxConcurrency(7), WithMaxConcurrency(2)) + + if fr.maxConcurrency != 2 { + t.Fatalf("maxConcurrency = %d, want 2 (last option wins)", fr.maxConcurrency) + } +} + +func TestNilOptionIgnored(t *testing.T) { + fr := newTestRunner(t, nil, WithMaxConcurrency(4)) + + if fr.maxConcurrency != 4 { + t.Fatalf("maxConcurrency = %d, want 4", fr.maxConcurrency) + } +} From a723134bb166caae392655d19b76a818c7432a2f Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 18:07:24 +0300 Subject: [PATCH 02/38] feat: scenariorunner VU scheduler for load profiles Adds an engine-agnostic scheduler that runs a caller-supplied iteration function across a fixed pool of virtual users. It knows nothing about flows: callers pass any callback. Bounds are Duration, MaxIterations and context cancellation; whichever comes first stops new iterations being issued, and in-flight iterations are always drained before Run returns. Iteration errors are counted in the summary and never abort the scenario. Sequence numbers are handed out exactly once, contiguously from zero. Invalid profiles (no VUs, no stop condition, nil callback) are rejected before any work starts. --- .../runner/scenariorunner/scenariorunner.go | 138 ++++++++ .../scenariorunner/scenariorunner_test.go | 335 ++++++++++++++++++ 2 files changed, 473 insertions(+) create mode 100644 packages/server/pkg/flow/runner/scenariorunner/scenariorunner.go create mode 100644 packages/server/pkg/flow/runner/scenariorunner/scenariorunner_test.go diff --git a/packages/server/pkg/flow/runner/scenariorunner/scenariorunner.go b/packages/server/pkg/flow/runner/scenariorunner/scenariorunner.go new file mode 100644 index 000000000..cd6b25603 --- /dev/null +++ b/packages/server/pkg/flow/runner/scenariorunner/scenariorunner.go @@ -0,0 +1,138 @@ +// Package scenariorunner schedules repeated executions of an arbitrary +// callback across a fixed pool of virtual users (VUs), the way a load +// generator does. +// +// It is deliberately engine-agnostic: it knows nothing about flows, HTTP or +// the rest of the runner packages. Callers supply an iteration function and a +// RunProfile; the scheduler guarantees at most RunProfile.VUs iterations are +// in flight at once and stops issuing new ones once a bound is reached. +package scenariorunner + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "time" +) + +// Configuration errors returned by Run before any work is started. +var ( + ErrInvalidVUs = errors.New("scenariorunner: VUs must be >= 1") + ErrNoStopCondition = errors.New("scenariorunner: profile must set Duration or MaxIterations") + ErrNilIteration = errors.New("scenariorunner: iteration function must not be nil") +) + +// RunProfile describes a constant-VU load profile. +type RunProfile struct { + // VUs is the number of concurrent workers. Must be >= 1. + VUs int + // Duration bounds the window during which new iterations are issued. + // Values <= 0 mean unbounded, in which case MaxIterations must be set. + Duration time.Duration + // MaxIterations bounds the total number of iterations issued. Values <= 0 + // mean unbounded, in which case Duration must be set. + MaxIterations int64 +} + +// Summary reports what a scenario actually did. +type Summary struct { + // Iterations is the number of iterations that ran to completion. + Iterations int64 + // Errors is how many of those iterations returned a non-nil error. + Errors int64 + // Elapsed is the wall-clock time from start until the last worker exited. + Elapsed time.Duration +} + +// Run executes iter repeatedly across prof.VUs workers until the profile's +// duration or iteration bound is reached, or ctx is canceled. +// +// Each call receives the zero-based index of the worker running it and a +// scenario-wide sequence number; sequence numbers are handed out exactly once, +// contiguously from zero. At most prof.VUs calls are in flight at any moment. +// +// Errors returned by iter are counted in Summary.Errors and never abort the +// scenario. Run stops issuing new iterations when a bound is hit or ctx is +// canceled, then drains the iterations already in flight before returning, so +// no worker outlives the call. +// +// Run returns ctx.Err() if the context ended the scenario early; the Summary +// still reports the work completed up to that point. Configuration problems +// are reported before any iteration runs, alongside a zero Summary. +func Run(ctx context.Context, prof RunProfile, iter func(ctx context.Context, vu int, seq int64) error) (Summary, error) { + if err := validate(prof, iter); err != nil { + return Summary{}, err + } + + var ( + next atomic.Int64 // sequence number the next iteration will claim + iterations atomic.Int64 + errCount atomic.Int64 + ) + + start := time.Now() + + var deadline time.Time + if prof.Duration > 0 { + deadline = start.Add(prof.Duration) + } + + // Iteration errors must not cancel sibling workers, so a plain WaitGroup is + // used rather than errgroup: there is no error to propagate. + var wg sync.WaitGroup + wg.Add(prof.VUs) + for vu := range prof.VUs { + go func() { + defer wg.Done() + for { + seq, ok := claim(ctx, &next, prof.MaxIterations, deadline) + if !ok { + return + } + if err := iter(ctx, vu, seq); err != nil { + errCount.Add(1) + } + iterations.Add(1) + } + }() + } + wg.Wait() + + summary := Summary{ + Iterations: iterations.Load(), + Errors: errCount.Load(), + Elapsed: time.Since(start), + } + return summary, ctx.Err() +} + +// claim reserves the next sequence number, or reports that the worker should +// stop. The stop conditions are checked before the number is reserved so that +// an iteration bound is never consumed by an iteration that does not run. +func claim(ctx context.Context, next *atomic.Int64, maxIterations int64, deadline time.Time) (int64, bool) { + if ctx.Err() != nil { + return 0, false + } + if !deadline.IsZero() && !time.Now().Before(deadline) { + return 0, false + } + seq := next.Add(1) - 1 + if maxIterations > 0 && seq >= maxIterations { + return 0, false + } + return seq, true +} + +func validate(prof RunProfile, iter func(ctx context.Context, vu int, seq int64) error) error { + if prof.VUs < 1 { + return ErrInvalidVUs + } + if prof.Duration <= 0 && prof.MaxIterations <= 0 { + return ErrNoStopCondition + } + if iter == nil { + return ErrNilIteration + } + return nil +} diff --git a/packages/server/pkg/flow/runner/scenariorunner/scenariorunner_test.go b/packages/server/pkg/flow/runner/scenariorunner/scenariorunner_test.go new file mode 100644 index 000000000..9387287eb --- /dev/null +++ b/packages/server/pkg/flow/runner/scenariorunner/scenariorunner_test.go @@ -0,0 +1,335 @@ +package scenariorunner_test + +import ( + "context" + "errors" + "runtime" + "sort" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/runner/scenariorunner" +) + +// concurrencyProbe records the high-water mark of simultaneously running +// iterations so tests can prove the VU ceiling is actually enforced. +type concurrencyProbe struct { + current atomic.Int64 + highest atomic.Int64 +} + +func (p *concurrencyProbe) enter() { + now := p.current.Add(1) + for { + high := p.highest.Load() + if now <= high || p.highest.CompareAndSwap(high, now) { + return + } + } +} + +func (p *concurrencyProbe) leave() { p.current.Add(-1) } + +func (p *concurrencyProbe) highWater() int64 { return p.highest.Load() } + +func TestRunEnforcesVUCeiling(t *testing.T) { + var probe concurrencyProbe + + summary, err := scenariorunner.Run(t.Context(), + scenariorunner.RunProfile{VUs: 5, MaxIterations: 50}, + func(context.Context, int, int64) error { + probe.enter() + defer probe.leave() + time.Sleep(20 * time.Millisecond) + return nil + }) + if err != nil { + t.Fatalf("Run() error = %v, want nil", err) + } + + if got := probe.highWater(); got > 5 { + t.Errorf("high-water concurrency = %d, want <= 5", got) + } + if got := probe.highWater(); got < 2 { + t.Errorf("high-water concurrency = %d, want >= 2 (VUs never ran in parallel)", got) + } + if summary.Iterations != 50 { + t.Errorf("Iterations = %d, want 50", summary.Iterations) + } + if summary.Errors != 0 { + t.Errorf("Errors = %d, want 0", summary.Errors) + } + if summary.Elapsed <= 0 { + t.Errorf("Elapsed = %v, want > 0", summary.Elapsed) + } +} + +func TestRunStopsIssuingAtDuration(t *testing.T) { + const ( + duration = 150 * time.Millisecond + iterCost = 20 * time.Millisecond + tolerance = 100 * time.Millisecond // generous: covers scheduler jitter on loaded CI + ) + + var ( + mu sync.Mutex + starts []time.Time + ) + + begin := time.Now() + summary, err := scenariorunner.Run(t.Context(), + scenariorunner.RunProfile{VUs: 4, Duration: duration}, + func(context.Context, int, int64) error { + mu.Lock() + starts = append(starts, time.Now()) + mu.Unlock() + time.Sleep(iterCost) + return nil + }) + if err != nil { + t.Fatalf("Run() error = %v, want nil", err) + } + + deadline := begin.Add(duration) + + mu.Lock() + defer mu.Unlock() + + if len(starts) == 0 { + t.Fatal("no iterations ran within the duration window") + } + for i, start := range starts { + if start.After(deadline.Add(tolerance)) { + t.Errorf("iteration %d started %v after the deadline, want no starts past it", + i, start.Sub(deadline)) + } + } + if summary.Iterations != int64(len(starts)) { + t.Errorf("Iterations = %d, want %d (one per observed start)", summary.Iterations, len(starts)) + } + if summary.Elapsed < duration { + t.Errorf("Elapsed = %v, want >= %v", summary.Elapsed, duration) + } +} + +func TestRunCountsErrorsWithoutAborting(t *testing.T) { + sentinel := errors.New("iteration blew up") + + summary, err := scenariorunner.Run(t.Context(), + scenariorunner.RunProfile{VUs: 4, MaxIterations: 30}, + func(_ context.Context, _ int, seq int64) error { + if seq%3 == 0 { + return sentinel + } + return nil + }) + if err != nil { + t.Fatalf("Run() error = %v, want nil (iteration errors must not abort the scenario)", err) + } + + if summary.Iterations != 30 { + t.Errorf("Iterations = %d, want 30 (errors must not stop later iterations)", summary.Iterations) + } + if summary.Errors != 10 { + t.Errorf("Errors = %d, want 10", summary.Errors) + } +} + +func TestRunHandsOutEachSequenceExactlyOnce(t *testing.T) { + const ( + vus = 4 + total = 200 + ) + + var ( + mu sync.Mutex + seqs []int64 + ) + + summary, err := scenariorunner.Run(t.Context(), + scenariorunner.RunProfile{VUs: vus, MaxIterations: total}, + func(_ context.Context, vu int, seq int64) error { + if vu < 0 || vu >= vus { + return errors.New("vu index out of range") + } + mu.Lock() + seqs = append(seqs, seq) + mu.Unlock() + return nil + }) + if err != nil { + t.Fatalf("Run() error = %v, want nil", err) + } + if summary.Errors != 0 { + t.Fatalf("Errors = %d, want 0 (vu index out of range)", summary.Errors) + } + + mu.Lock() + defer mu.Unlock() + + if len(seqs) != total { + t.Fatalf("collected %d sequence numbers, want %d", len(seqs), total) + } + sort.Slice(seqs, func(i, j int) bool { return seqs[i] < seqs[j] }) + for i, seq := range seqs { + if seq != int64(i) { + t.Fatalf("sequence numbers are not 0..%d contiguous: index %d = %d", total-1, i, seq) + } + } +} + +func TestRunStopsAtWhicheverBoundComesFirst(t *testing.T) { + summary, err := scenariorunner.Run(t.Context(), + scenariorunner.RunProfile{VUs: 2, MaxIterations: 5, Duration: time.Hour}, + func(context.Context, int, int64) error { return nil }) + if err != nil { + t.Fatalf("Run() error = %v, want nil", err) + } + if summary.Iterations != 5 { + t.Errorf("Iterations = %d, want 5 (iteration bound must win over the long duration)", summary.Iterations) + } +} + +func TestRunReturnsContextErrorAndDrains(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + var started, finished atomic.Int64 + + go func() { + time.Sleep(60 * time.Millisecond) + cancel() + }() + + summary, err := scenariorunner.Run(ctx, + scenariorunner.RunProfile{VUs: 3, Duration: time.Hour}, + func(context.Context, int, int64) error { + started.Add(1) + // Deliberately ignores ctx: a runner that abandons in-flight work + // instead of draining would return while this is still sleeping. + time.Sleep(50 * time.Millisecond) + finished.Add(1) + return nil + }) + + if !errors.Is(err, context.Canceled) { + t.Fatalf("Run() error = %v, want context.Canceled", err) + } + if s, f := started.Load(), finished.Load(); s != f { + t.Errorf("started = %d, finished = %d: in-flight iterations were not drained", s, f) + } + if summary.Iterations != finished.Load() { + t.Errorf("Iterations = %d, want %d (partial summary must report completed work)", + summary.Iterations, finished.Load()) + } + if summary.Elapsed <= 0 { + t.Errorf("Elapsed = %v, want > 0", summary.Elapsed) + } +} + +func TestRunRejectsInvalidProfiles(t *testing.T) { + okIter := func(context.Context, int, int64) error { return nil } + + tests := []struct { + name string + profile scenariorunner.RunProfile + iter func(ctx context.Context, vu int, seq int64) error + wantErr error + }{ + { + name: "zero VUs", + profile: scenariorunner.RunProfile{VUs: 0, MaxIterations: 10}, + iter: okIter, + wantErr: scenariorunner.ErrInvalidVUs, + }, + { + name: "negative VUs", + profile: scenariorunner.RunProfile{VUs: -3, Duration: time.Second}, + iter: okIter, + wantErr: scenariorunner.ErrInvalidVUs, + }, + { + name: "no stop condition", + profile: scenariorunner.RunProfile{VUs: 2}, + iter: okIter, + wantErr: scenariorunner.ErrNoStopCondition, + }, + { + name: "negative bounds are not bounds", + profile: scenariorunner.RunProfile{VUs: 2, Duration: -time.Second, MaxIterations: -5}, + iter: okIter, + wantErr: scenariorunner.ErrNoStopCondition, + }, + { + name: "nil iteration function", + profile: scenariorunner.RunProfile{VUs: 2, MaxIterations: 10}, + iter: nil, + wantErr: scenariorunner.ErrNilIteration, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var calls atomic.Int64 + iter := tt.iter + if iter != nil { + iter = func(ctx context.Context, vu int, seq int64) error { + calls.Add(1) + return tt.iter(ctx, vu, seq) + } + } + + summary, err := scenariorunner.Run(t.Context(), tt.profile, iter) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("Run() error = %v, want %v", err, tt.wantErr) + } + if calls.Load() != 0 { + t.Errorf("iteration function called %d times, want 0 (no work on config error)", calls.Load()) + } + if summary != (scenariorunner.Summary{}) { + t.Errorf("Summary = %+v, want zero value", summary) + } + }) + } +} + +func TestRunDoesNotLeakGoroutines(t *testing.T) { + // Warm up so one-shot runtime goroutines are not attributed to Run. + if _, err := scenariorunner.Run(t.Context(), + scenariorunner.RunProfile{VUs: 2, MaxIterations: 2}, + func(context.Context, int, int64) error { return nil }); err != nil { + t.Fatalf("warm-up Run() error = %v", err) + } + + before := runtime.NumGoroutine() + + if _, err := scenariorunner.Run(t.Context(), + scenariorunner.RunProfile{VUs: 16, MaxIterations: 200}, + func(context.Context, int, int64) error { + time.Sleep(time.Millisecond) + return nil + }); err != nil { + t.Fatalf("Run() error = %v", err) + } + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := scenariorunner.Run(ctx, + scenariorunner.RunProfile{VUs: 16, Duration: time.Hour}, + func(context.Context, int, int64) error { return nil }); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled Run() error = %v, want context.Canceled", err) + } + + // Goroutines exit asynchronously; give the scheduler a bounded window. + var after int + for range 100 { + after = runtime.NumGoroutine() + if after <= before+2 { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Errorf("goroutines before = %d, after = %d: scheduler leaked workers", before, after) +} From c962a3015c6ba97c8dd883a5aa3c0005e8cc6c5f Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 18:11:25 +0300 Subject: [PATCH 03/38] feat: lean execution mode drops response bodies after assertion Long load runs retain every response body in the flow variable map, so memory grows with iteration count. WithLeanMode(true) makes HTTP request nodes write a placeholder instead of the decoded body. The seam is FlowNodeRequest.LeanMode: that struct is built in exactly one place in the workspace, so the flag reaches every node without touching any node constructor or the flow builder. The body is swapped in buildResponseVar, before it can be copied into the flow output, which also folds the duplicated duration conversion out of RunSync/RunAsync. Assertions are evaluated by ResponseCreateHTTP against the raw response rather than the variable map, so they keep working on a dropped body; only downstream extraction from response.body is given up, which is the point of the mode. Off by default: no option means byte-identical output to before. --- packages/server/pkg/flow/node/node.go | 7 + .../server/pkg/flow/node/nrequest/nrequest.go | 27 ++- .../flow/node/nrequest/nrequest_lean_test.go | 209 ++++++++++++++++++ .../runner/flowlocalrunner/flowlocalrunner.go | 12 + .../flowlocalrunner_options_test.go | 52 +++++ 5 files changed, 301 insertions(+), 6 deletions(-) create mode 100644 packages/server/pkg/flow/node/nrequest/nrequest_lean_test.go diff --git a/packages/server/pkg/flow/node/node.go b/packages/server/pkg/flow/node/node.go index c7dc06894..66a7ae64b 100644 --- a/packages/server/pkg/flow/node/node.go +++ b/packages/server/pkg/flow/node/node.go @@ -61,6 +61,13 @@ type FlowNodeRequest struct { IterationContext *runner.IterationContext // For hierarchical execution naming in loops ExecutionID idwrap.IDWrap // Unique ID for this specific execution of the node Logger *slog.Logger // Optional structured logger for node diagnostics + + // LeanMode drops response bodies from node output so memory stays flat + // across long load runs. Assertions still see the full response because + // they are evaluated against it directly rather than through VarMap, but + // downstream nodes cannot extract from a body that was not retained. + // Off by default. + LeanMode bool } type LogPushFunc func(status runner.FlowNodeStatus) diff --git a/packages/server/pkg/flow/node/nrequest/nrequest.go b/packages/server/pkg/flow/node/nrequest/nrequest.go index 9751c29da..ea006b5df 100644 --- a/packages/server/pkg/flow/node/nrequest/nrequest.go +++ b/packages/server/pkg/flow/node/nrequest/nrequest.go @@ -59,6 +59,11 @@ const ( OUTPUT_REQUEST_NAME = "request" ) +// LeanBodyPlaceholder stands in for the response body in node output when the +// flow runs in lean mode, so consumers can tell a dropped body from an empty +// one. +const LeanBodyPlaceholder = "[body dropped: lean mode]" + type NodeRequestOutput struct { Request request.RequestResponseVar `json:"request"` Response httpclient.ResponseVar `json:"response"` @@ -86,6 +91,20 @@ func buildNodeRequestOutputMap(output NodeRequestOutput) map[string]any { return result } +// buildResponseVar converts a response into the shape written to the flow's +// variable map. In lean mode the decoded body is swapped for a placeholder +// before it can be copied into the flow output, which is what keeps memory flat +// across long load runs. Assertions are unaffected: they are evaluated against +// the raw response, not this value. +func buildResponseVar(resp request.RequestResponse, lean bool) httpclient.ResponseVar { + respVar := httpclient.ConvertResponseToVar(resp.HttpResp) + respVar.Duration = int32(resp.LapTime.Milliseconds()) // nolint:gosec // G115 + if lean { + respVar.Body = LeanBodyPlaceholder + } + return respVar +} + func cloneStringMapToAny(src map[string]string) map[string]any { if len(src) == 0 { return map[string]any{} @@ -186,11 +205,9 @@ func (nr *NodeRequest) RunSync(ctx context.Context, req *node.FlowNodeRequest) n } // Build output using measured duration - respVar := httpclient.ConvertResponseToVar(resp.HttpResp) - respVar.Duration = int32(resp.LapTime.Milliseconds()) // nolint:gosec // G115 output := NodeRequestOutput{ Request: request.ConvertRequestToVar(prepareOutput), - Response: respVar, + Response: buildResponseVar(*resp, req.LeanMode), } respMap := buildNodeRequestOutputMap(output) @@ -379,11 +396,9 @@ func (nr *NodeRequest) RunAsync(ctx context.Context, req *node.FlowNodeRequest, } // Build output using measured duration - respVar := httpclient.ConvertResponseToVar(resp.HttpResp) - respVar.Duration = int32(resp.LapTime.Milliseconds()) // nolint:gosec // G115 output := NodeRequestOutput{ Request: request.ConvertRequestToVar(prepareOutput), - Response: respVar, + Response: buildResponseVar(*resp, req.LeanMode), } respMap := buildNodeRequestOutputMap(output) diff --git a/packages/server/pkg/flow/node/nrequest/nrequest_lean_test.go b/packages/server/pkg/flow/node/nrequest/nrequest_lean_test.go new file mode 100644 index 000000000..d37336ee5 --- /dev/null +++ b/packages/server/pkg/flow/node/nrequest/nrequest_lean_test.go @@ -0,0 +1,209 @@ +package nrequest + +import ( + "context" + "io" + "net/http" + "strings" + "sync" + "testing" + + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/node" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mflow" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mhttp" +) + +const leanTestBody = `{"token":"s3cret","items":[1,2,3]}` + +type leanStubHTTPClient struct{} + +func (leanStubHTTPClient) Do(*http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: 201, + Body: io.NopCloser(strings.NewReader(leanTestBody)), + Header: http.Header{"Content-Type": []string{"application/json"}}, + }, nil +} + +// newLeanFixture builds a request node whose stub response carries a real JSON +// body, wired to a flow request with the given lean setting. +func newLeanFixture(t *testing.T, lean bool, asserts []mhttp.HTTPAssert) (*NodeRequest, *node.FlowNodeRequest) { + t.Helper() + + nodeID := idwrap.NewNow() + httpID := idwrap.NewNow() + + respChan := make(chan NodeRequestSideResp, 1) + startResponseConsumer(respChan) + + requestNode := New( + nodeID, + "req", + mhttp.HTTP{ID: httpID, Name: "req", Url: "https://example.dev", Method: "GET", BodyKind: mhttp.HttpBodyKindRaw}, + nil, // headers + nil, // params + &mhttp.HTTPBodyRaw{ID: idwrap.NewNow(), HttpID: httpID, RawData: []byte("{}")}, + nil, // formBody + nil, // urlBody + asserts, + leanStubHTTPClient{}, + respChan, + nil, // logger + ) + + flowReq := &node.FlowNodeRequest{ + VarMap: map[string]any{}, + ReadWriteLock: &sync.RWMutex{}, + NodeMap: map[idwrap.IDWrap]node.FlowNode{nodeID: requestNode}, + EdgeSourceMap: mflow.EdgesMap{}, + ExecutionID: idwrap.NewNow(), + LeanMode: lean, + } + + return requestNode, flowReq +} + +// responseOutput digs the "response" map out of the node's flow output. +func responseOutput(t *testing.T, flowReq *node.FlowNodeRequest) map[string]any { + t.Helper() + + nodeOut, ok := flowReq.VarMap["req"].(map[string]any) + if !ok { + t.Fatalf("VarMap[\"req\"] = %#v, want map[string]any", flowReq.VarMap["req"]) + } + respOut, ok := nodeOut[OUTPUT_RESPONSE_NAME].(map[string]any) + if !ok { + t.Fatalf("node output %q = %#v, want map[string]any", OUTPUT_RESPONSE_NAME, nodeOut[OUTPUT_RESPONSE_NAME]) + } + return respOut +} + +func assertResponseMetadataIntact(t *testing.T, respOut map[string]any) { + t.Helper() + + if got := respOut["status"]; got != float64(201) { + t.Errorf("response.status = %#v, want 201", got) + } + if _, ok := respOut["duration"].(float64); !ok { + t.Errorf("response.duration = %#v, want a float64 to survive lean mode", respOut["duration"]) + } + headers, ok := respOut["headers"].(map[string]any) + if !ok { + t.Fatalf("response.headers = %#v, want map[string]any", respOut["headers"]) + } + if headers["Content-Type"] != "application/json" { + t.Errorf("response.headers[Content-Type] = %#v, want application/json", headers["Content-Type"]) + } +} + +func TestRunSyncLeanModeDropsResponseBody(t *testing.T) { + requestNode, flowReq := newLeanFixture(t, true, nil) + + if result := requestNode.RunSync(context.Background(), flowReq); result.Err != nil { + t.Fatalf("RunSync() error = %v, want nil", result.Err) + } + + respOut := responseOutput(t, flowReq) + if got := respOut["body"]; got != LeanBodyPlaceholder { + t.Errorf("response.body = %#v, want %q", got, LeanBodyPlaceholder) + } + assertResponseMetadataIntact(t, respOut) +} + +func TestRunSyncDefaultKeepsResponseBody(t *testing.T) { + requestNode, flowReq := newLeanFixture(t, false, nil) + + if result := requestNode.RunSync(context.Background(), flowReq); result.Err != nil { + t.Fatalf("RunSync() error = %v, want nil", result.Err) + } + + respOut := responseOutput(t, flowReq) + body, ok := respOut["body"].(map[string]any) + if !ok { + t.Fatalf("response.body = %#v, want the decoded JSON body", respOut["body"]) + } + if body["token"] != "s3cret" { + t.Errorf("response.body.token = %#v, want s3cret", body["token"]) + } + assertResponseMetadataIntact(t, respOut) +} + +func TestRunAsyncLeanModeDropsResponseBody(t *testing.T) { + for _, tc := range []struct { + name string + lean bool + wantBody func(*testing.T, any) + }{ + { + name: "lean drops body", + lean: true, + wantBody: func(t *testing.T, body any) { + t.Helper() + if body != LeanBodyPlaceholder { + t.Errorf("response.body = %#v, want %q", body, LeanBodyPlaceholder) + } + }, + }, + { + name: "default keeps body", + lean: false, + wantBody: func(t *testing.T, body any) { + t.Helper() + decoded, ok := body.(map[string]any) + if !ok { + t.Fatalf("response.body = %#v, want the decoded JSON body", body) + } + if decoded["token"] != "s3cret" { + t.Errorf("response.body.token = %#v, want s3cret", decoded["token"]) + } + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + requestNode, flowReq := newLeanFixture(t, tc.lean, nil) + + resultChan := make(chan node.FlowNodeResult, 1) + go requestNode.RunAsync(context.Background(), flowReq, resultChan) + + result := <-resultChan + if result.Err != nil { + t.Fatalf("RunAsync() error = %v, want nil", result.Err) + } + + respOut := responseOutput(t, flowReq) + tc.wantBody(t, respOut["body"]) + assertResponseMetadataIntact(t, respOut) + }) + } +} + +// Assertions read the response directly rather than the flow output, so lean +// mode must not change whether they pass. +func TestLeanModeKeepsBodyAssertionsWorking(t *testing.T) { + asserts := []mhttp.HTTPAssert{ + {ID: idwrap.NewNow(), Enabled: true, Value: `response.body.token == "s3cret"`}, + {ID: idwrap.NewNow(), Enabled: true, Value: "response.status == 201"}, + } + + for _, lean := range []bool{false, true} { + requestNode, flowReq := newLeanFixture(t, lean, asserts) + + if result := requestNode.RunSync(context.Background(), flowReq); result.Err != nil { + t.Errorf("lean=%t: RunSync() error = %v, want nil (assertions must be unaffected)", lean, result.Err) + } + } +} + +// buildNodeRequestOutputMap keeps its historical body-retaining behaviour so +// existing callers and tests are unaffected. +func TestBuildNodeRequestOutputMapDefaultsToFullBody(t *testing.T) { + full := buildNodeRequestOutputMap(sampleOutput()) + respOut, ok := full[OUTPUT_RESPONSE_NAME].(map[string]any) + if !ok { + t.Fatalf("response output = %#v, want map[string]any", full[OUTPUT_RESPONSE_NAME]) + } + if _, isPlaceholder := respOut["body"].(string); isPlaceholder { + t.Errorf("response.body = %#v, want the full body by default", respOut["body"]) + } +} diff --git a/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner.go b/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner.go index 2edf6d9b2..cbccf79bb 100644 --- a/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner.go +++ b/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner.go @@ -46,6 +46,7 @@ type FlowLocalRunner struct { maxConcurrency int mode ExecutionMode selectedMode ExecutionMode + leanMode bool enableDataTracking bool logger *slog.Logger @@ -68,6 +69,16 @@ func WithMaxConcurrency(n int) Option { } } +// WithLeanMode enables lean execution: nodes drop response bodies from their +// flow output once assertions have been evaluated, keeping memory flat across +// long load runs. Downstream nodes cannot extract from a dropped body, so this +// is opt-in and off by default. +func WithLeanMode(enabled bool) Option { + return func(r *FlowLocalRunner) { + r.leanMode = enabled + } +} + func CreateFlowRunner(id, flowID idwrap.IDWrap, startNodeIDs []idwrap.IDWrap, flowNodeMap map[idwrap.IDWrap]node.FlowNode, edgesMap mflow.EdgesMap, timeout time.Duration, logger *slog.Logger, opts ...Option) *FlowLocalRunner { r := &FlowLocalRunner{ ID: id, @@ -205,6 +216,7 @@ func (r *FlowLocalRunner) RunWithEvents(ctx context.Context, channels runner.Flo PendingAtmoicMap: pendingAtmoicMap, PendingMapMu: pendingMu, Logger: r.logger, + LeanMode: r.leanMode, } mode := r.mode diff --git a/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner_options_test.go b/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner_options_test.go index 918b3f92c..68b60a59a 100644 --- a/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner_options_test.go +++ b/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner_options_test.go @@ -1,9 +1,11 @@ package flowlocalrunner import ( + "context" "testing" "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/node" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/runner" "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mflow" ) @@ -26,6 +28,56 @@ func TestCreateFlowRunnerDefaultUnchanged(t *testing.T) { if fr.maxConcurrency != goroutineCount { t.Fatalf("default maxConcurrency = %d, want %d (CPU-derived default)", fr.maxConcurrency, goroutineCount) } + if fr.leanMode { + t.Fatal("default leanMode = true, want false") + } +} + +func TestWithLeanMode(t *testing.T) { + if fr := newTestRunner(t, WithLeanMode(true)); !fr.leanMode { + t.Error("WithLeanMode(true): leanMode = false, want true") + } + if fr := newTestRunner(t, WithLeanMode(false)); fr.leanMode { + t.Error("WithLeanMode(false): leanMode = true, want false") + } +} + +// The runner must hand its lean setting to every node through the shared +// FlowNodeRequest, which is the only place nodes can read it from. +func TestLeanModeReachesNodeRequest(t *testing.T) { + for _, lean := range []bool{false, true} { + probe := &leanProbeNode{id: idwrap.NewNow()} + fr := CreateFlowRunner( + idwrap.NewNow(), idwrap.NewNow(), []idwrap.IDWrap{probe.id}, + map[idwrap.IDWrap]node.FlowNode{probe.id: probe}, + mflow.EdgesMap{}, 0, nil, WithLeanMode(lean), + ) + + if err := fr.RunWithEvents(t.Context(), runner.FlowEventChannels{}, nil); err != nil { + t.Fatalf("lean=%t: RunWithEvents() error = %v", lean, err) + } + if probe.seen != lean { + t.Errorf("lean=%t: node observed FlowNodeRequest.LeanMode = %t", lean, probe.seen) + } + } +} + +// leanProbeNode records the LeanMode it was executed with. +type leanProbeNode struct { + id idwrap.IDWrap + seen bool +} + +func (n *leanProbeNode) GetID() idwrap.IDWrap { return n.id } +func (n *leanProbeNode) GetName() string { return "lean-probe" } + +func (n *leanProbeNode) RunSync(_ context.Context, req *node.FlowNodeRequest) node.FlowNodeResult { + n.seen = req.LeanMode + return node.FlowNodeResult{} +} + +func (n *leanProbeNode) RunAsync(ctx context.Context, req *node.FlowNodeRequest, resultChan chan node.FlowNodeResult) { + resultChan <- n.RunSync(ctx, req) } func TestWithMaxConcurrency(t *testing.T) { From 38e3620e4f868227d582a4cad76ddb485a88e6b9 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 18:49:43 +0300 Subject: [PATCH 04/38] test: pin the flow runner default to the CPU-derived value The default-concurrency test compared against the package variable, which would still pass if the default were changed to a constant. Assert the variable itself is MaxParallelism() so that regression is caught. Also states in the scenariorunner docs when ctx.Err() is returned and that panics inside an iteration are not recovered. --- .../flowlocalrunner/flowlocalrunner_options_test.go | 3 +++ .../pkg/flow/runner/scenariorunner/scenariorunner.go | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner_options_test.go b/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner_options_test.go index 68b60a59a..55628eaa8 100644 --- a/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner_options_test.go +++ b/packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner_options_test.go @@ -25,6 +25,9 @@ func newTestRunner(t *testing.T, opts ...Option) *FlowLocalRunner { func TestCreateFlowRunnerDefaultUnchanged(t *testing.T) { fr := newTestRunner(t) + if goroutineCount != MaxParallelism() { + t.Fatalf("package default = %d, want the CPU-derived %d", goroutineCount, MaxParallelism()) + } if fr.maxConcurrency != goroutineCount { t.Fatalf("default maxConcurrency = %d, want %d (CPU-derived default)", fr.maxConcurrency, goroutineCount) } diff --git a/packages/server/pkg/flow/runner/scenariorunner/scenariorunner.go b/packages/server/pkg/flow/runner/scenariorunner/scenariorunner.go index cd6b25603..30f411af5 100644 --- a/packages/server/pkg/flow/runner/scenariorunner/scenariorunner.go +++ b/packages/server/pkg/flow/runner/scenariorunner/scenariorunner.go @@ -57,9 +57,13 @@ type Summary struct { // canceled, then drains the iterations already in flight before returning, so // no worker outlives the call. // -// Run returns ctx.Err() if the context ended the scenario early; the Summary -// still reports the work completed up to that point. Configuration problems -// are reported before any iteration runs, alongside a zero Summary. +// Run returns ctx.Err() if the context is done once the scenario ends, which is +// normally because cancellation stopped it early; the Summary still reports the +// work completed up to that point. Configuration problems are reported before +// any iteration runs, alongside a zero Summary. +// +// A panic inside iter is not recovered: it crashes the process, as it would +// anywhere else in the engine. func Run(ctx context.Context, prof RunProfile, iter func(ctx context.Context, vu int, seq int64) error) (Summary, error) { if err := validate(prof, iter); err != nil { return Summary{}, err From 7b8a8810f5fd88d3adec105c3adddd858cdfd376 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 18:49:53 +0300 Subject: [PATCH 05/38] test: add YAML round-trip golden corpus Characterization tests for the yamlflowsimplev2 Import -> Export pipeline, one fixture per family (request headers/assertions/templates, if/for/for_each control flow, js/wait, graphql, websocket, sub-flows, multi-flow run block, environments/credentials). Each case asserts round-trip stability (export(import(export(import(x)))) is a fixed point) and pins the stable output as a .golden snapshot. This snapshot captures today's behavior including known bugs: HTTP assertions are silently dropped on import (GraphQL assertions survive correctly), credentials and active/global environment selection are not re-exported. sub_flow_trigger is deliberately excluded from the sub-flows fixture: it has a pre-existing node-ID tracking bug (converter_node.go processSteps) where the flow node's ID is overwritten to the flow's start-node ID but the NodeSubFlowTrigger.FlowNodeID implementation record keeps the original ID, so the exporter's lookup always misses and silently drops the step. That breaks round-trip stability outright (re-import errors with "depends on unknown step"), independent of anything in this change. Not fixed here since it's outside this change's scope. --- .../translate/yamlflowsimplev2/golden_test.go | 149 ++++++++++++++++++ .../golden/control_flow_if_for_foreach.golden | 74 +++++++++ .../golden/control_flow_if_for_foreach.yaml | 49 ++++++ .../golden/environments_credentials.golden | 33 ++++ .../golden/environments_credentials.yaml | 34 ++++ .../testdata/golden/graphql_assertions.golden | 55 +++++++ .../testdata/golden/graphql_assertions.yaml | 47 ++++++ .../testdata/golden/js_wait.golden | 34 ++++ .../testdata/golden/js_wait.yaml | 24 +++ ...equest_headers_assertions_templates.golden | 59 +++++++ .../request_headers_assertions_templates.yaml | 62 ++++++++ .../testdata/golden/run_multi_flow.golden | 52 ++++++ .../testdata/golden/run_multi_flow.yaml | 37 +++++ .../testdata/golden/sub_flows.golden | 61 +++++++ .../testdata/golden/sub_flows.yaml | 41 +++++ .../testdata/golden/websocket.golden | 38 +++++ .../testdata/golden/websocket.yaml | 26 +++ 17 files changed, 875 insertions(+) create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/golden_test.go create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/control_flow_if_for_foreach.golden create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/control_flow_if_for_foreach.yaml create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/environments_credentials.golden create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/environments_credentials.yaml create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/graphql_assertions.golden create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/graphql_assertions.yaml create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/js_wait.golden create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/js_wait.yaml create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.golden create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.yaml create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_multi_flow.golden create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_multi_flow.yaml create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.golden create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.yaml create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/websocket.golden create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/websocket.yaml diff --git a/packages/server/pkg/translate/yamlflowsimplev2/golden_test.go b/packages/server/pkg/translate/yamlflowsimplev2/golden_test.go new file mode 100644 index 000000000..4c77b38b0 --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/golden_test.go @@ -0,0 +1,149 @@ +package yamlflowsimplev2 + +import ( + "bytes" + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" +) + +// update rewrites the .golden files with freshly generated output. Run with: +// +// go test ./pkg/translate/yamlflowsimplev2/ -run TestGoldenRoundTrip -update +var update = flag.Bool("update", false, "rewrite .golden files") + +const goldenDir = "testdata/golden" + +// goldenCases returns the sorted list of golden case names, one per *.yaml +// fixture under testdata/golden (the paired *.golden snapshot is excluded +// since it doesn't carry the .yaml suffix). +func goldenCases(t *testing.T) []string { + t.Helper() + + entries, err := os.ReadDir(goldenDir) + if err != nil { + t.Fatalf("failed to read %s: %v", goldenDir, err) + } + + var names []string + for _, entry := range entries { + if entry.IsDir() { + continue + } + if name, ok := strings.CutSuffix(entry.Name(), ".yaml"); ok { + names = append(names, name) + } + } + sort.Strings(names) + return names +} + +// exportAfterImport runs the full Import -> Export pipeline and returns the +// resulting YAML bytes. +func exportAfterImport(t *testing.T, in []byte) []byte { + t.Helper() + + opts := GetDefaultOptions(idwrap.NewNow()) + bundle, err := ConvertSimplifiedYAML(in, opts) + if err != nil { + t.Fatalf("ConvertSimplifiedYAML failed: %v", err) + } + + out, err := MarshalSimplifiedYAML(bundle) + if err != nil { + t.Fatalf("MarshalSimplifiedYAML failed: %v", err) + } + return out +} + +func readGoldenFile(t *testing.T, path string) []byte { + t.Helper() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read %s: %v", path, err) + } + return data +} + +func writeGoldenFile(t *testing.T, path string, data []byte) { + t.Helper() + + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatalf("failed to write %s: %v", path, err) + } +} + +// diff renders a line-oriented listing of every line that differs between +// want and got, to keep golden mismatches readable in test output. +func diff(want, got []byte) string { + wantLines := strings.Split(string(want), "\n") + gotLines := strings.Split(string(got), "\n") + + maxLines := len(wantLines) + if len(gotLines) > maxLines { + maxLines = len(gotLines) + } + + var b strings.Builder + for i := 0; i < maxLines; i++ { + var w, g string + if i < len(wantLines) { + w = wantLines[i] + } + if i < len(gotLines) { + g = gotLines[i] + } + if w != g { + fmt.Fprintf(&b, "line %d:\n want: %q\n got: %q\n", i+1, w, g) + } + } + if len(wantLines) != len(gotLines) { + fmt.Fprintf(&b, "(want has %d lines, got has %d lines)\n", len(wantLines), len(gotLines)) + } + return b.String() +} + +// TestGoldenRoundTrip locks in the current Import -> Export behavior of the +// YAML flow contract, one fixture per family under testdata/golden. This is a +// characterization test: it snapshots today's behavior, bugs included (most +// notably, HTTP assertions are silently dropped on import as of this +// snapshot). Future deliberate behavior changes update the golden via +// -update, making every change visible as a diff. +// +// Each case asserts two things: +// +// 1. Stability: re-exporting an already-exported document is a no-op. +// Import(yaml) -> Export -> yamlA; Import(yamlA) -> Export -> yamlB; +// yamlA == yamlB. +// 2. No unintentional drift: the stable output matches the committed +// .golden snapshot. +func TestGoldenRoundTrip(t *testing.T) { + for _, name := range goldenCases(t) { + t.Run(name, func(t *testing.T) { + in := readGoldenFile(t, filepath.Join(goldenDir, name+".yaml")) + + first := exportAfterImport(t, in) + second := exportAfterImport(t, first) + if !bytes.Equal(first, second) { + t.Fatalf("unstable round-trip for %s:\n%s", name, diff(first, second)) + } + + goldenPath := filepath.Join(goldenDir, name+".golden") + if *update { + writeGoldenFile(t, goldenPath, first) + } + + want := readGoldenFile(t, goldenPath) + if !bytes.Equal(first, want) { + t.Fatalf("golden mismatch for %s (run with -update after intentional changes):\n%s", name, diff(want, first)) + } + }) + } +} diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/control_flow_if_for_foreach.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/control_flow_if_for_foreach.golden new file mode 100644 index 000000000..0e339603b --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/control_flow_if_for_foreach.golden @@ -0,0 +1,74 @@ +workspace_name: Golden Control Flow +run: + - flow: ControlFlow +requests: + - name: FetchStatus + method: GET + url: https://api.example.com/status + - name: ProcessItem + method: GET + url: https://api.example.com/items +flows: + - name: ControlFlow + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - request: + name: FetchStatus + depends_on: Start + position_x: 300 + position_y: 0 + use_request: FetchStatus + - if: + name: CheckStatus + depends_on: FetchStatus + position_x: 600 + position_y: 0 + condition: FetchStatus.response.status == 200 + - js: + name: OnFailure + depends_on: CheckStatus.else + position_x: 900 + position_y: -75 + code: |- + export default function(context) { + return { ok: false }; + } + - js: + name: OnSuccess + depends_on: CheckStatus.then + position_x: 900 + position_y: 75 + code: |- + export default function(context) { + return { ok: true }; + } + - for: + name: RetryLoop + depends_on: OnFailure + position_x: 1200 + position_y: -75 + iter_count: "3" + - for_each: + name: EachItem + depends_on: OnSuccess + position_x: 1200 + position_y: 75 + items: '[1, 2, 3]' + - js: + name: RetryAttempt + depends_on: RetryLoop.loop + position_x: 1500 + position_y: -75 + code: |- + export default function(context) { + return { attempt: true }; + } + - request: + name: ProcessItem + depends_on: EachItem.loop + position_x: 1500 + position_y: 75 + use_request: ProcessItem diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/control_flow_if_for_foreach.yaml b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/control_flow_if_for_foreach.yaml new file mode 100644 index 000000000..dc374b201 --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/control_flow_if_for_foreach.yaml @@ -0,0 +1,49 @@ +workspace_name: Golden Control Flow +flows: + - name: ControlFlow + steps: + - manual_start: + name: Start + - request: + name: FetchStatus + depends_on: Start + method: GET + url: https://api.example.com/status + - if: + name: CheckStatus + depends_on: FetchStatus + condition: FetchStatus.response.status == 200 + - js: + name: OnFailure + depends_on: CheckStatus.else + code: | + export default function(context) { + return { ok: false }; + } + - js: + name: OnSuccess + depends_on: CheckStatus.then + code: | + export default function(context) { + return { ok: true }; + } + - for: + name: RetryLoop + depends_on: OnFailure + iter_count: '3' + - for_each: + name: EachItem + depends_on: OnSuccess + items: '[1, 2, 3]' + - js: + name: RetryAttempt + depends_on: RetryLoop.loop + code: | + export default function(context) { + return { attempt: true }; + } + - request: + name: ProcessItem + depends_on: EachItem.loop + method: GET + url: https://api.example.com/items diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/environments_credentials.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/environments_credentials.golden new file mode 100644 index 000000000..63c295be8 --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/environments_credentials.golden @@ -0,0 +1,33 @@ +workspace_name: Golden Environments Credentials +run: + - flow: EnvFlow +requests: + - name: PingAPI + method: GET + url: '{{ base_url }}/ping' + headers: + Authorization: Bearer {{ api_token }} +flows: + - name: EnvFlow + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - request: + name: PingAPI + depends_on: Start + position_x: 300 + position_y: 0 + use_request: PingAPI +environments: + - name: default + variables: + base_url: https://api.example.com + - name: staging + variables: + api_token: '{{ #env:STAGING_API_TOKEN }}' + base_url: https://staging.api.example.com + - name: shared + variables: + shared_flag: "true" diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/environments_credentials.yaml b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/environments_credentials.yaml new file mode 100644 index 000000000..029bf6157 --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/environments_credentials.yaml @@ -0,0 +1,34 @@ +workspace_name: Golden Environments Credentials +credentials: + - name: primary-openai + type: openai + token: '{{ #env:PRIMARY_OPENAI_TOKEN }}' + - name: secondary-anthropic + type: anthropic + api_key: '{{ #env:SECONDARY_ANTHROPIC_KEY }}' +active_environment: staging +global_environment: shared +environments: + - name: default + variables: + base_url: https://api.example.com + - name: staging + description: Staging environment + variables: + base_url: https://staging.api.example.com + api_token: '{{ #env:STAGING_API_TOKEN }}' + - name: shared + variables: + shared_flag: 'true' +flows: + - name: EnvFlow + steps: + - manual_start: + name: Start + - request: + name: PingAPI + depends_on: Start + method: GET + url: '{{ base_url }}/ping' + headers: + Authorization: 'Bearer {{ api_token }}' diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/graphql_assertions.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/graphql_assertions.golden new file mode 100644 index 000000000..ee444f52d --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/graphql_assertions.golden @@ -0,0 +1,55 @@ +workspace_name: Golden GraphQL +run: + - flow: GraphQLFlow +graphql_requests: + - name: InlineQuery + url: https://api.example.com/graphql + query: |- + query Widget($id: ID!) { + widget(id: $id) { + id + name + } + } + variables: '{"id": "1"}' + headers: + Accept: application/json + assertions: + - expression: response.status == 200 + enabled: true + - expression: response.body.data.widget.id != nil + enabled: false + - name: Widgets + url: https://api.example.com/graphql + query: |- + query { + widgets { + id + name + } + } + variables: '{}' + headers: + Accept: application/json + assertions: + - response.status == 200 + - response.body.data.widgets != nil +flows: + - name: GraphQLFlow + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - graphql: + name: Widgets + depends_on: Start + position_x: 300 + position_y: 0 + use_request: Widgets + - graphql: + name: InlineQuery + depends_on: Widgets + position_x: 600 + position_y: 0 + use_request: InlineQuery diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/graphql_assertions.yaml b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/graphql_assertions.yaml new file mode 100644 index 000000000..351f77067 --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/graphql_assertions.yaml @@ -0,0 +1,47 @@ +workspace_name: Golden GraphQL +graphql_requests: + - name: ListWidgets + url: https://api.example.com/graphql + query: |- + query { + widgets { + id + name + } + } + variables: '{}' + headers: + Accept: application/json + assertions: + - response.status == 200 + - response.body.data.widgets != nil +flows: + - name: GraphQLFlow + steps: + - manual_start: + name: Start + - graphql: + name: Widgets + depends_on: Start + use_request: ListWidgets + - graphql: + name: InlineQuery + depends_on: Widgets + url: https://api.example.com/graphql + query: |- + query Widget($id: ID!) { + widget(id: $id) { + id + name + } + } + variables: '{"id": "1"}' + headers: + - name: Accept + value: application/json + enabled: true + assertions: + - expression: response.status == 200 + enabled: true + - expression: response.body.data.widget.id != nil + enabled: false diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/js_wait.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/js_wait.golden new file mode 100644 index 000000000..831491bdf --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/js_wait.golden @@ -0,0 +1,34 @@ +workspace_name: Golden JS Wait +run: + - flow: JsWaitFlow +flows: + - name: JsWaitFlow + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - js: + name: ComputeDelay + depends_on: Start + position_x: 300 + position_y: 0 + code: |- + export default function(context) { + return { delayMs: 250 }; + } + - wait: + name: PauseBriefly + depends_on: ComputeDelay + position_x: 600 + position_y: 0 + duration_ms: "250" + - js: + name: AfterWait + depends_on: PauseBriefly + position_x: 900 + position_y: 0 + code: |- + export default function(context) { + return { done: true }; + } diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/js_wait.yaml b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/js_wait.yaml new file mode 100644 index 000000000..fecea7047 --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/js_wait.yaml @@ -0,0 +1,24 @@ +workspace_name: Golden JS Wait +flows: + - name: JsWaitFlow + steps: + - manual_start: + name: Start + - js: + name: ComputeDelay + depends_on: Start + code: | + export default function(context) { + return { delayMs: 250 }; + } + - wait: + name: PauseBriefly + depends_on: ComputeDelay + duration_ms: '250' + - js: + name: AfterWait + depends_on: PauseBriefly + code: | + export default function(context) { + return { done: true }; + } diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.golden new file mode 100644 index 000000000..53d0a5afe --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.golden @@ -0,0 +1,59 @@ +workspace_name: Golden Request Corpus +run: + - flow: RequestFlow +requests: + - name: ListFormRequest + method: POST + url: https://api.example.com/list + headers: + - name: Accept + value: application/json + enabled: true + - name: X-Trace-Id + value: '{{ MapFormRequest.response.body.traceId }}' + enabled: true + - name: X-Debug + value: "true" + enabled: false + body: '{"note":"created by ListFormRequest"}' + - name: MapFormRequest + method: GET + url: https://api.example.com/map + headers: + Accept: application/json + X-Trace-Id: golden-trace + query_params: + limit: "20" + page: "1" + - name: TemplatedRequest + method: GET + url: https://api.example.com/base + headers: + Accept: application/json + X-Client: golden-corpus + X-Extra: template-override +flows: + - name: RequestFlow + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - request: + name: MapFormRequest + depends_on: Start + position_x: 300 + position_y: 0 + use_request: MapFormRequest + - request: + name: ListFormRequest + depends_on: MapFormRequest + position_x: 600 + position_y: 0 + use_request: ListFormRequest + - request: + name: TemplatedRequest + depends_on: ListFormRequest + position_x: 900 + position_y: 0 + use_request: TemplatedRequest diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.yaml b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.yaml new file mode 100644 index 000000000..21db600ab --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.yaml @@ -0,0 +1,62 @@ +workspace_name: Golden Request Corpus +requests: + - name: BaseRequest + method: GET + url: https://api.example.com/base + headers: + Accept: application/json + X-Client: golden-corpus + assertions: + - response.status == 200 +flows: + - name: RequestFlow + steps: + - manual_start: + name: Start + - request: + name: MapFormRequest + depends_on: Start + method: GET + url: https://api.example.com/map + headers: + Accept: application/json + X-Trace-Id: golden-trace + query_params: + page: '1' + limit: '20' + assertions: + - response.status == 200 + - response.body.ok == true + - request: + name: ListFormRequest + depends_on: MapFormRequest + method: POST + url: https://api.example.com/list + headers: + - name: Accept + value: application/json + enabled: true + - name: X-Trace-Id + value: '{{ MapFormRequest.response.body.traceId }}' + description: propagated trace id + enabled: true + - name: X-Debug + value: 'true' + enabled: false + body: + type: json + json: + note: created by ListFormRequest + assertions: + - expression: response.status == 201 + enabled: true + - expression: response.body.legacyField == null + enabled: false + - request: + name: TemplatedRequest + depends_on: ListFormRequest + use_request: BaseRequest + headers: + X-Extra: template-override + assertions: + - response.body.id != nil diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_multi_flow.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_multi_flow.golden new file mode 100644 index 000000000..161d2d3cd --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_multi_flow.golden @@ -0,0 +1,52 @@ +workspace_name: Golden Run Multi Flow +run: + - flow: FlowA + - flow: FlowB + - flow: FlowC +requests: + - name: RequestA + method: GET + url: https://api.example.com/a + - name: RequestB + method: GET + url: https://api.example.com/b + - name: RequestC + method: GET + url: https://api.example.com/c +flows: + - name: FlowA + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - request: + name: RequestA + depends_on: Start + position_x: 300 + position_y: 0 + use_request: RequestA + - name: FlowB + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - request: + name: RequestB + depends_on: Start + position_x: 300 + position_y: 0 + use_request: RequestB + - name: FlowC + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - request: + name: RequestC + depends_on: Start + position_x: 300 + position_y: 0 + use_request: RequestC diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_multi_flow.yaml b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_multi_flow.yaml new file mode 100644 index 000000000..6c6f04119 --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_multi_flow.yaml @@ -0,0 +1,37 @@ +workspace_name: Golden Run Multi Flow +run: + - flow: FlowC + depends_on: + - FlowA + - FlowB + - flow: FlowA + - flow: FlowB + depends_on: FlowA +flows: + - name: FlowA + steps: + - manual_start: + name: Start + - request: + name: RequestA + depends_on: Start + method: GET + url: https://api.example.com/a + - name: FlowB + steps: + - manual_start: + name: Start + - request: + name: RequestB + depends_on: Start + method: GET + url: https://api.example.com/b + - name: FlowC + steps: + - manual_start: + name: Start + - request: + name: RequestC + depends_on: Start + method: GET + url: https://api.example.com/c diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.golden new file mode 100644 index 000000000..673f1cf01 --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.golden @@ -0,0 +1,61 @@ +workspace_name: Golden Sub Flows +run: + - flow: MainFlow + - flow: GreeterFlow +requests: + - name: FetchUser + method: GET + url: https://api.example.com/users/1 +flows: + - name: MainFlow + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - request: + name: FetchUser + depends_on: Start + position_x: 300 + position_y: 0 + use_request: FetchUser + - run_sub_flow: + name: CallGreeter + depends_on: FetchUser + position_x: 600 + position_y: 0 + flow: GreeterFlow + inputs: + userName: '{{ FetchUser.response.body.name }}' + - js: + name: UseGreeting + depends_on: CallGreeter + position_x: 900 + position_y: 0 + code: |- + export default function(context) { + return { greeting: context.CallGreeter }; + } + - name: GreeterFlow + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - js: + name: BuildGreeting + depends_on: Start + position_x: 300 + position_y: 0 + code: |- + export default function(context) { + return { greeting: "Hello " + context.userName }; + } + - sub_flow_return: + name: Return + depends_on: BuildGreeting + position_x: 600 + position_y: 0 + outputs: + - name: greeting + expression: BuildGreeting.greeting diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.yaml b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.yaml new file mode 100644 index 000000000..e9a1684a4 --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.yaml @@ -0,0 +1,41 @@ +workspace_name: Golden Sub Flows +flows: + - name: MainFlow + steps: + - manual_start: + name: Start + - request: + name: FetchUser + depends_on: Start + method: GET + url: https://api.example.com/users/1 + - run_sub_flow: + name: CallGreeter + depends_on: FetchUser + flow: GreeterFlow + inputs: + userName: '{{ FetchUser.response.body.name }}' + - js: + name: UseGreeting + depends_on: CallGreeter + code: | + export default function(context) { + return { greeting: context.CallGreeter }; + } + - name: GreeterFlow + steps: + - manual_start: + name: Start + - js: + name: BuildGreeting + depends_on: Start + code: | + export default function(context) { + return { greeting: "Hello " + context.userName }; + } + - sub_flow_return: + name: Return + depends_on: BuildGreeting + outputs: + - name: greeting + expression: BuildGreeting.greeting diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/websocket.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/websocket.golden new file mode 100644 index 000000000..4186824ed --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/websocket.golden @@ -0,0 +1,38 @@ +workspace_name: Golden WebSocket +run: + - flow: WebSocketFlow +flows: + - name: WebSocketFlow + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - ws_connection: + name: Conn + depends_on: Start + position_x: 300 + position_y: 0 + url: wss://echo.example.com/socket + headers: + Authorization: Bearer token123 + - ws_send: + name: SendPing + depends_on: Conn + position_x: 600 + position_y: 0 + ws_connection_node_name: Conn + message: '{"type":"ping"}' + - wait: + name: WaitForPong + depends_on: SendPing + position_x: 900 + position_y: 0 + duration_ms: "500" + - ws_send: + name: SendClose + depends_on: WaitForPong + position_x: 1200 + position_y: 0 + ws_connection_node_name: Conn + message: '{"type":"close"}' diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/websocket.yaml b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/websocket.yaml new file mode 100644 index 000000000..38569a36b --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/websocket.yaml @@ -0,0 +1,26 @@ +workspace_name: Golden WebSocket +flows: + - name: WebSocketFlow + steps: + - manual_start: + name: Start + - ws_connection: + name: Conn + depends_on: Start + url: wss://echo.example.com/socket + headers: + Authorization: Bearer token123 + - ws_send: + name: SendPing + depends_on: Conn + ws_connection_node_name: Conn + message: '{"type":"ping"}' + - wait: + name: WaitForPong + depends_on: SendPing + duration_ms: '500' + - ws_send: + name: SendClose + depends_on: WaitForPong + ws_connection_node_name: Conn + message: '{"type":"close"}' From cbdee90c6f9c7918cabea8649751f7bae4a14dd8 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 18:50:06 +0300 Subject: [PATCH 06/38] feat: loadmetrics HDR aggregation package Standalone packages/server/pkg/loadmetrics package for load-test metrics: HDR-histogram aggregation keyed by (step, status-class), interval frames via Aggregator.Flush, lossless Merge across frames, and ClassifyStatus for mapping HTTP status/transport errors (incl. timeout detection) to buckets. Adds github.com/HdrHistogram/hdrhistogram-go as a new packages/server dependency (1us..10min range, 3 significant figures per the frozen contract). TDD: loadmetrics_test.go written first against the not-yet-existing API (confirmed RED via compile failure), then loadmetrics.go implemented to green. Covers percentile correctness against a known uniform distribution, merge equivalence across split aggregators, status classification incl. timeout detection, concurrent Record under -race, and RPS math. --- packages/server/go.mod | 1 + packages/server/go.sum | 2 + .../server/pkg/loadmetrics/loadmetrics.go | 288 ++++++++++++++++++ .../pkg/loadmetrics/loadmetrics_test.go | 207 +++++++++++++ 4 files changed, 498 insertions(+) create mode 100644 packages/server/pkg/loadmetrics/loadmetrics.go create mode 100644 packages/server/pkg/loadmetrics/loadmetrics_test.go diff --git a/packages/server/go.mod b/packages/server/go.mod index 66fc84919..2d5c24bb7 100644 --- a/packages/server/go.mod +++ b/packages/server/go.mod @@ -40,6 +40,7 @@ require ( cloud.google.com/go/iam v1.5.2 // indirect cloud.google.com/go/longrunning v0.6.7 // indirect cloud.google.com/go/vertexai v0.12.0 // indirect + github.com/HdrHistogram/hdrhistogram-go v1.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/packages/server/go.sum b/packages/server/go.sum index e2242f8d8..b570e9400 100644 --- a/packages/server/go.sum +++ b/packages/server/go.sum @@ -21,6 +21,8 @@ cloud.google.com/go/vertexai v0.12.0/go.mod h1:8u+d0TsvBfAAd2x5R6GMgbYhsLgo3J7lm connectrpc.com/connect v1.19.1 h1:R5M57z05+90EfEvCY1b7hBxDVOUl45PrtXtAV2fOC14= connectrpc.com/connect v1.19.1/go.mod h1:tN20fjdGlewnSFeZxLKb0xwIZ6ozc3OQs2hTXy4du9w= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/HdrHistogram/hdrhistogram-go v1.3.0 h1:NBGs5RJ6Q7lDFhszi5AHovwDrSzJAF1ElZy2g0suRTg= +github.com/HdrHistogram/hdrhistogram-go v1.3.0/go.mod h1:CiIeGiHSd06zjX+FypuEJ5EQ07KKtxZ+8J6hszwVQig= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= diff --git a/packages/server/pkg/loadmetrics/loadmetrics.go b/packages/server/pkg/loadmetrics/loadmetrics.go new file mode 100644 index 000000000..0fe66ffa9 --- /dev/null +++ b/packages/server/pkg/loadmetrics/loadmetrics.go @@ -0,0 +1,288 @@ +// Package loadmetrics aggregates load-test execution results into interval +// frames using HDR histograms, keyed by (step, status-class). +// +// The usage shape is: an Aggregator collects Records concurrently during an +// interval, Flush drains that interval into an immutable Frame, and Merge +// lossily-free combines any number of Frames - whether successive flushes +// from one Aggregator, or one flush each from many concurrent Aggregators - +// into a single Report with derived percentiles. +// +// This package has no dependency on the rest of the server; its exported +// surface is a frozen contract consumed by the load-test ingest pipeline. +package loadmetrics + +import ( + "context" + "errors" + "os" + "sync" + "time" + + hdrhistogram "github.com/HdrHistogram/hdrhistogram-go" +) + +// StatusClass buckets a recorded outcome for aggregation. Values are the +// literal strings carried on the wire (see the LoadMetricEntry TypeSpec +// model) - do not change them without updating that mapping. +type StatusClass string + +const ( + StatusClass2xx StatusClass = "2xx" + StatusClass3xx StatusClass = "3xx" + StatusClass4xx StatusClass = "4xx" + StatusClass5xx StatusClass = "5xx" + StatusClassError StatusClass = "error" + StatusClassTimeout StatusClass = "timeout" +) + +// ClassifyStatus buckets an HTTP-ish status code and/or transport error into +// a StatusClass. A non-nil err always wins over code (whatever code happens +// to be at that point, e.g. zero, is irrelevant once the request failed at +// the transport level). Timeouts - deadline-exceeded context errors, or any +// error the standard library recognizes via os.IsTimeout - classify as +// StatusClassTimeout rather than the more generic StatusClassError. +func ClassifyStatus(code int, err error) StatusClass { + if err != nil { + if errors.Is(err, context.DeadlineExceeded) || os.IsTimeout(err) { + return StatusClassTimeout + } + return StatusClassError + } + + switch { + case code >= 200 && code < 300: + return StatusClass2xx + case code >= 300 && code < 400: + return StatusClass3xx + case code >= 400 && code < 500: + return StatusClass4xx + case code >= 500 && code < 600: + return StatusClass5xx + default: + return StatusClassError + } +} + +// Key identifies one aggregation bucket: a load-test step at a particular +// outcome class. +type Key struct { + Step string + StatusClass StatusClass +} + +// Entry is one (step, status-class) bucket's accumulated stats for a single +// interval. +type Entry struct { + Count int64 + ErrorCount int64 + Bytes int64 + Hist *hdrhistogram.Histogram // 1us..10min, 3 significant figures +} + +// Frame is one flushed interval's worth of aggregated entries. +type Frame struct { + IntervalStart time.Time + Interval time.Duration + Entries map[Key]Entry +} + +const ( + // hdrLowestDiscernibleValue and hdrHighestTrackableValue bound recorded + // latencies to 1 microsecond .. 10 minutes at 3 significant figures, per + // the frozen contract. Values are clamped into this range before being + // recorded (see clampMicros) so RecordValue can never fail on us. + hdrLowestDiscernibleValue = int64(1) + hdrSignificantFigures = 3 +) + +var hdrHighestTrackableValue = (10 * time.Minute).Microseconds() + +func newHistogram() *hdrhistogram.Histogram { + return hdrhistogram.New(hdrLowestDiscernibleValue, hdrHighestTrackableValue, hdrSignificantFigures) +} + +func clampMicros(us int64) int64 { + switch { + case us < hdrLowestDiscernibleValue: + return hdrLowestDiscernibleValue + case us > hdrHighestTrackableValue: + return hdrHighestTrackableValue + default: + return us + } +} + +// Aggregator accumulates Records into per-key HDR histograms for the current +// interval. It is safe for concurrent use. +type Aggregator struct { + mu sync.Mutex + intervalStart time.Time + entries map[Key]*Entry +} + +// NewAggregator creates an Aggregator. interval documents the caller's +// intended flush cadence; it is not baked into Frame.Interval, which instead +// reflects the actual elapsed time between flushes (see Flush) so reported +// RPS stays correct even if the caller flushes early, late, or irregularly. +func NewAggregator(interval time.Duration) *Aggregator { + _ = interval + return &Aggregator{ + intervalStart: time.Now(), + entries: make(map[Key]*Entry), + } +} + +// Record adds one observation to the bucket identified by k. It is +// goroutine-safe. +func (a *Aggregator) Record(k Key, latency time.Duration, bytes int64, isErr bool) { + us := clampMicros(latency.Microseconds()) + + a.mu.Lock() + defer a.mu.Unlock() + + e, ok := a.entries[k] + if !ok { + e = &Entry{Hist: newHistogram()} + a.entries[k] = e + } + + e.Count++ + if isErr { + e.ErrorCount++ + } + e.Bytes += bytes + + // us is always clamped into the histogram's configured range above, so + // RecordValue cannot fail here. + _ = e.Hist.RecordValue(us) +} + +// Flush drains the current interval into a Frame and starts a fresh interval +// for subsequent Records. now becomes the next interval's start, so +// successive Flush calls describe contiguous, non-overlapping time ranges. +func (a *Aggregator) Flush(now time.Time) Frame { + a.mu.Lock() + defer a.mu.Unlock() + + frame := Frame{ + IntervalStart: a.intervalStart, + Interval: now.Sub(a.intervalStart), + Entries: make(map[Key]Entry, len(a.entries)), + } + for k, e := range a.entries { + frame.Entries[k] = *e + } + + a.entries = make(map[Key]*Entry) + a.intervalStart = now + + return frame +} + +// Stats is a set of derived, percentile-level statistics for a bucket, or +// for the whole run in Report.Total. +type Stats struct { + Count, ErrorCount, Bytes int64 + P50, P90, P95, P99, Max time.Duration + RPS float64 // Count / covered wall time +} + +// Report is the fully-merged result of one or more Frames: overall Stats +// plus a per-(step,status-class) breakdown. +type Report struct { + Total Stats + PerStep map[Key]Stats +} + +// counts accumulates the plain (non-histogram) fields of an Entry while +// merging; kept separate from Entry so a nil/absent Hist is never implied. +type counts struct { + Count, ErrorCount, Bytes int64 +} + +// Merge combines any number of Frames - from one Aggregator's successive +// flushes, or from many concurrent Aggregators - into a single Report. +// Merging is lossless: percentiles on the merged histograms are equivalent +// (within HDR's significant-figure precision) to what a single Aggregator +// fed all the same observations would have produced. +// +// The wall time used for RPS is the union of the supplied frames' time +// ranges (earliest IntervalStart to latest IntervalStart+Interval), not the +// sum of their durations - this keeps RPS correct both for a single +// Aggregator's successive contiguous flushes and for many Aggregators +// flushing over the same overlapping window. +func Merge(frames []Frame) Report { + perStepCounts := make(map[Key]*counts) + perStepHist := make(map[Key]*hdrhistogram.Histogram) + + var start, end time.Time + for i, f := range frames { + if i == 0 || f.IntervalStart.Before(start) { + start = f.IntervalStart + } + if fEnd := f.IntervalStart.Add(f.Interval); i == 0 || fEnd.After(end) { + end = fEnd + } + + for k, e := range f.Entries { + c, ok := perStepCounts[k] + if !ok { + c = &counts{} + perStepCounts[k] = c + } + c.Count += e.Count + c.ErrorCount += e.ErrorCount + c.Bytes += e.Bytes + + h, ok := perStepHist[k] + if !ok { + h = newHistogram() + perStepHist[k] = h + } + if e.Hist != nil { + h.Merge(e.Hist) + } + } + } + + wallSeconds := end.Sub(start).Seconds() + + totalHist := newHistogram() + var total counts + perStep := make(map[Key]Stats, len(perStepCounts)) + for k, c := range perStepCounts { + h := perStepHist[k] + perStep[k] = statsFrom(*c, h, wallSeconds) + + total.Count += c.Count + total.ErrorCount += c.ErrorCount + total.Bytes += c.Bytes + totalHist.Merge(h) + } + + return Report{ + Total: statsFrom(total, totalHist, wallSeconds), + PerStep: perStep, + } +} + +func statsFrom(c counts, h *hdrhistogram.Histogram, wallSeconds float64) Stats { + stats := Stats{ + Count: c.Count, + ErrorCount: c.ErrorCount, + Bytes: c.Bytes, + P50: microseconds(h.ValueAtPercentile(50)), + P90: microseconds(h.ValueAtPercentile(90)), + P95: microseconds(h.ValueAtPercentile(95)), + P99: microseconds(h.ValueAtPercentile(99)), + Max: microseconds(h.Max()), + } + if wallSeconds > 0 { + stats.RPS = float64(c.Count) / wallSeconds + } + return stats +} + +func microseconds(us int64) time.Duration { + return time.Duration(us) * time.Microsecond +} diff --git a/packages/server/pkg/loadmetrics/loadmetrics_test.go b/packages/server/pkg/loadmetrics/loadmetrics_test.go new file mode 100644 index 000000000..9d39a0023 --- /dev/null +++ b/packages/server/pkg/loadmetrics/loadmetrics_test.go @@ -0,0 +1,207 @@ +package loadmetrics + +import ( + "context" + "errors" + "fmt" + "os" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestClassifyStatus(t *testing.T) { + tests := []struct { + name string + code int + err error + want StatusClass + }{ + {"200 OK classifies as 2xx", 200, nil, StatusClass2xx}, + {"299 upper boundary classifies as 2xx", 299, nil, StatusClass2xx}, + {"301 redirect classifies as 3xx", 301, nil, StatusClass3xx}, + {"404 not found classifies as 4xx", 404, nil, StatusClass4xx}, + {"500 server error classifies as 5xx", 500, nil, StatusClass5xx}, + {"599 upper boundary classifies as 5xx", 599, nil, StatusClass5xx}, + {"transport error with no status classifies as error", 0, errors.New("connection reset by peer"), StatusClassError}, + {"context deadline exceeded classifies as timeout", 0, context.DeadlineExceeded, StatusClassTimeout}, + {"wrapped context deadline exceeded classifies as timeout", 0, fmt.Errorf("dial: %w", context.DeadlineExceeded), StatusClassTimeout}, + {"os deadline exceeded classifies as timeout", 0, os.ErrDeadlineExceeded, StatusClassTimeout}, + {"error takes precedence over a successful code", 200, errors.New("boom"), StatusClassError}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ClassifyStatus(tt.code, tt.err) + assert.Equal(t, tt.want, got) + }) + } +} + +// TestAggregatorPercentileCorrectness records a uniform 1..1000ms distribution +// and checks the HDR-derived P50/P99 land within 1% of the true values, +// which is well outside HDR's 3-significant-figure quantization error at +// this magnitude. +func TestAggregatorPercentileCorrectness(t *testing.T) { + agg := NewAggregator(time.Minute) + k := Key{Step: "load-step", StatusClass: StatusClass2xx} + + for ms := 1; ms <= 1000; ms++ { + agg.Record(k, time.Duration(ms)*time.Millisecond, 0, false) + } + + frame := agg.Flush(time.Now()) + entry, ok := frame.Entries[k] + require.True(t, ok, "expected an entry for the recorded key") + require.NotNil(t, entry.Hist) + require.Equal(t, int64(1000), entry.Count) + + wantP50 := 500 * time.Millisecond + wantP99 := 990 * time.Millisecond + + gotP50 := time.Duration(entry.Hist.ValueAtPercentile(50)) * time.Microsecond + gotP99 := time.Duration(entry.Hist.ValueAtPercentile(99)) * time.Microsecond + + assert.InDelta(t, wantP50, gotP50, float64(wantP50)*0.01, "P50 should be within 1%% of 500ms") + assert.InDelta(t, wantP99, gotP99, float64(wantP99)*0.01, "P99 should be within 1%% of 990ms") +} + +// TestMergeEquivalence feeds the same 1000-sample distribution into a single +// Aggregator and, separately, into two Aggregators fed disjoint (even/odd) +// halves. Merging the two half-frames must reproduce the same counts and +// percentiles (within HDR tolerance) as the single full aggregator - proving +// Merge genuinely combines independent histograms rather than trivially +// matching a self-merge. +func TestMergeEquivalence(t *testing.T) { + k := Key{Step: "load-step", StatusClass: StatusClass2xx} + + full := NewAggregator(time.Minute) + evens := NewAggregator(time.Minute) + odds := NewAggregator(time.Minute) + + for ms := 1; ms <= 1000; ms++ { + latency := time.Duration(ms) * time.Millisecond + isErr := ms%13 == 0 + size := int64(ms * 7) + + full.Record(k, latency, size, isErr) + if ms%2 == 0 { + evens.Record(k, latency, size, isErr) + } else { + odds.Record(k, latency, size, isErr) + } + } + + now := time.Now() + fullFrame := full.Flush(now) + evensFrame := evens.Flush(now) + oddsFrame := odds.Flush(now) + + // Sanity: the two halves are genuinely disjoint subsets, not duplicates + // of the full data - otherwise this test would pass trivially. + evensOnly := Merge([]Frame{evensFrame}).PerStep[k] + require.Equal(t, int64(500), evensOnly.Count) + + fullStats := Merge([]Frame{fullFrame}).PerStep[k] + splitStats := Merge([]Frame{evensFrame, oddsFrame}).PerStep[k] + + require.Equal(t, int64(1000), fullStats.Count) + assert.Equal(t, fullStats.Count, splitStats.Count) + assert.Equal(t, fullStats.ErrorCount, splitStats.ErrorCount) + assert.Equal(t, fullStats.Bytes, splitStats.Bytes) + assert.Equal(t, fullStats.Max, splitStats.Max) + + tolerance := func(want time.Duration) float64 { return float64(want) * 0.01 } + assert.InDelta(t, fullStats.P50, splitStats.P50, tolerance(fullStats.P50)) + assert.InDelta(t, fullStats.P90, splitStats.P90, tolerance(fullStats.P90)) + assert.InDelta(t, fullStats.P95, splitStats.P95, tolerance(fullStats.P95)) + assert.InDelta(t, fullStats.P99, splitStats.P99, tolerance(fullStats.P99)) +} + +// TestMergeRPSMath constructs a Frame directly (bypassing wall-clock timing +// entirely) so the RPS = Count / covered-wall-time formula can be checked +// exactly: 100 records covering a 5s frame must yield 20.0 RPS. +func TestMergeRPSMath(t *testing.T) { + k := Key{Step: "load-step", StatusClass: StatusClass2xx} + + hist := newHistogram() + for i := 0; i < 100; i++ { + require.NoError(t, hist.RecordValue(10_000)) // 10ms, arbitrary + } + + frame := Frame{ + IntervalStart: time.Unix(1_700_000_000, 0), + Interval: 5 * time.Second, + Entries: map[Key]Entry{ + k: {Count: 100, ErrorCount: 3, Bytes: 12_345, Hist: hist}, + }, + } + + report := Merge([]Frame{frame}) + + assert.Equal(t, int64(100), report.Total.Count) + assert.Equal(t, int64(3), report.Total.ErrorCount) + assert.Equal(t, int64(12_345), report.Total.Bytes) + assert.InDelta(t, 20.0, report.Total.RPS, 1e-9) + assert.InDelta(t, 20.0, report.PerStep[k].RPS, 1e-9) +} + +// TestMergeEmptyFrames guards the RPS division-by-zero edge case: merging no +// frames must not produce NaN/Inf. +func TestMergeEmptyFrames(t *testing.T) { + report := Merge(nil) + + assert.Equal(t, int64(0), report.Total.Count) + assert.InDelta(t, 0.0, report.Total.RPS, 1e-9) + assert.Empty(t, report.PerStep) +} + +// TestAggregatorFlushDrainsAndResets checks that Flush both returns the +// interval's data and starts a fresh interval for subsequent Records. +func TestAggregatorFlushDrainsAndResets(t *testing.T) { + agg := NewAggregator(time.Second) + k := Key{Step: "step", StatusClass: StatusClass2xx} + + agg.Record(k, 10*time.Millisecond, 100, false) + + first := agg.Flush(time.Now()) + require.Len(t, first.Entries, 1) + assert.Equal(t, int64(1), first.Entries[k].Count) + + second := agg.Flush(time.Now()) + assert.Empty(t, second.Entries) +} + +// TestAggregatorRecordConcurrentRace exercises Record from many goroutines +// concurrently; run with -race to prove there's no data race, and assert the +// final counts to also catch lost-update bugs. +func TestAggregatorRecordConcurrentRace(t *testing.T) { + agg := NewAggregator(time.Second) + const goroutines = 8 + const perGoroutine = 10_000 + + var wg sync.WaitGroup + wg.Add(goroutines) + for g := range goroutines { + go func(g int) { + defer wg.Done() + k := Key{Step: fmt.Sprintf("step-%d", g%3), StatusClass: StatusClass2xx} + for i := range perGoroutine { + latency := time.Duration(i%1000+1) * time.Microsecond + agg.Record(k, latency, int64(i%256), i%97 == 0) + } + }(g) + } + wg.Wait() + + frame := agg.Flush(time.Now()) + + var total int64 + for _, e := range frame.Entries { + total += e.Count + } + assert.Equal(t, int64(goroutines*perGoroutine), total) +} From 74a3b41bb6a6b9e2a7321569fed8f3b622c9c1bc Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 18:52:56 +0300 Subject: [PATCH 07/38] feat: version the yamlflow schema Add a version field to the yamlflow document: absent (0) is treated as the current version for backward compatibility, and versions newer than this build supports are rejected with a clear error naming the offending value and the supported ceiling. Export always stamps the current version as the first key of the document so the schema version is visible without parsing the rest of the file. Golden corpus updated (-update): every fixture gains exactly one "version: 2" line, nothing else changes. --- .../translate/yamlflowsimplev2/exporter.go | 1 + .../golden/control_flow_if_for_foreach.golden | 1 + .../golden/environments_credentials.golden | 1 + .../testdata/golden/graphql_assertions.golden | 1 + .../testdata/golden/js_wait.golden | 1 + ...equest_headers_assertions_templates.golden | 1 + .../testdata/golden/run_multi_flow.golden | 1 + .../testdata/golden/sub_flows.golden | 1 + .../testdata/golden/websocket.golden | 1 + .../pkg/translate/yamlflowsimplev2/types.go | 15 ++++ .../yamlflowsimplev2/version_test.go | 69 +++++++++++++++++++ 11 files changed, 93 insertions(+) create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/version_test.go diff --git a/packages/server/pkg/translate/yamlflowsimplev2/exporter.go b/packages/server/pkg/translate/yamlflowsimplev2/exporter.go index 13f9b4a7a..e6f7532f0 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/exporter.go +++ b/packages/server/pkg/translate/yamlflowsimplev2/exporter.go @@ -201,6 +201,7 @@ func MarshalSimplifiedYAML(data *ioworkspace.WorkspaceBundle) ([]byte, error) { } yamlFormat := YamlFlowFormatV2{ + Version: CurrentYamlFlowVersion, WorkspaceName: wsName, Flows: make([]YamlFlowFlowV2, 0), } diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/control_flow_if_for_foreach.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/control_flow_if_for_foreach.golden index 0e339603b..b3492631d 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/control_flow_if_for_foreach.golden +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/control_flow_if_for_foreach.golden @@ -1,3 +1,4 @@ +version: 2 workspace_name: Golden Control Flow run: - flow: ControlFlow diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/environments_credentials.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/environments_credentials.golden index 63c295be8..1f2885db4 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/environments_credentials.golden +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/environments_credentials.golden @@ -1,3 +1,4 @@ +version: 2 workspace_name: Golden Environments Credentials run: - flow: EnvFlow diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/graphql_assertions.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/graphql_assertions.golden index ee444f52d..ca19acb6f 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/graphql_assertions.golden +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/graphql_assertions.golden @@ -1,3 +1,4 @@ +version: 2 workspace_name: Golden GraphQL run: - flow: GraphQLFlow diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/js_wait.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/js_wait.golden index 831491bdf..020e4fd61 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/js_wait.golden +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/js_wait.golden @@ -1,3 +1,4 @@ +version: 2 workspace_name: Golden JS Wait run: - flow: JsWaitFlow diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.golden index 53d0a5afe..21d376de2 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.golden +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.golden @@ -1,3 +1,4 @@ +version: 2 workspace_name: Golden Request Corpus run: - flow: RequestFlow diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_multi_flow.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_multi_flow.golden index 161d2d3cd..aa2bd0c3e 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_multi_flow.golden +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_multi_flow.golden @@ -1,3 +1,4 @@ +version: 2 workspace_name: Golden Run Multi Flow run: - flow: FlowA diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.golden index 673f1cf01..186685709 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.golden +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.golden @@ -1,3 +1,4 @@ +version: 2 workspace_name: Golden Sub Flows run: - flow: MainFlow diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/websocket.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/websocket.golden index 4186824ed..1010ff6dc 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/websocket.golden +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/websocket.golden @@ -1,3 +1,4 @@ +version: 2 workspace_name: Golden WebSocket run: - flow: WebSocketFlow diff --git a/packages/server/pkg/translate/yamlflowsimplev2/types.go b/packages/server/pkg/translate/yamlflowsimplev2/types.go index 8cd1f7988..876b76029 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/types.go +++ b/packages/server/pkg/translate/yamlflowsimplev2/types.go @@ -12,8 +12,16 @@ import ( "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mgraphql" ) +// CurrentYamlFlowVersion is the yamlflow schema version this build writes on +// export and the highest version it accepts on import. An absent `version` +// key (zero value) is treated as this version for backward compatibility +// with documents written before the field existed. Bump this when making a +// breaking change to the YAML flow contract. +const CurrentYamlFlowVersion = 2 + // YamlFlowFormatV2 represents the modern YAML structure for simplified workflows type YamlFlowFormatV2 struct { + Version int `yaml:"version,omitempty"` WorkspaceName string `yaml:"workspace_name"` ActiveEnvironment string `yaml:"active_environment,omitempty"` GlobalEnvironment string `yaml:"global_environment,omitempty"` @@ -558,6 +566,13 @@ func (opts ConvertOptionsV2) Validate() error { } func (yf YamlFlowFormatV2) Validate() error { + if yf.Version > CurrentYamlFlowVersion { + return NewYamlFlowErrorV2( + fmt.Sprintf("unsupported yamlflow version %d (this build supports up to %d)", yf.Version, CurrentYamlFlowVersion), + "version", yf.Version, + ) + } + if yf.WorkspaceName == "" { return NewYamlFlowErrorV2("workspace_name is required", "workspace_name", nil) } diff --git a/packages/server/pkg/translate/yamlflowsimplev2/version_test.go b/packages/server/pkg/translate/yamlflowsimplev2/version_test.go new file mode 100644 index 000000000..91aaae0f8 --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/version_test.go @@ -0,0 +1,69 @@ +package yamlflowsimplev2 + +import ( + "strings" + "testing" + + "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" +) + +const minimalVersionTestYAML = ` +workspace_name: Version Test +flows: + - name: VersionFlow + steps: + - manual_start: + name: Start +` + +// TestVersionField locks in the yamlflow schema versioning contract: an +// explicit supported version imports cleanly, an absent version defaults to +// the current version, an unsupported (too new) version is rejected with a +// specific error, and export always stamps the current version as the first +// key of the document. +func TestVersionField(t *testing.T) { + t.Run("version 2 succeeds", func(t *testing.T) { + yamlData := "version: 2\n" + minimalVersionTestYAML + opts := GetDefaultOptions(idwrap.NewNow()) + if _, err := ConvertSimplifiedYAML([]byte(yamlData), opts); err != nil { + t.Fatalf("expected version 2 to succeed, got error: %v", err) + } + }) + + t.Run("absent version succeeds", func(t *testing.T) { + opts := GetDefaultOptions(idwrap.NewNow()) + if _, err := ConvertSimplifiedYAML([]byte(minimalVersionTestYAML), opts); err != nil { + t.Fatalf("expected absent version to succeed, got error: %v", err) + } + }) + + t.Run("version 3 errors", func(t *testing.T) { + yamlData := "version: 3\n" + minimalVersionTestYAML + opts := GetDefaultOptions(idwrap.NewNow()) + _, err := ConvertSimplifiedYAML([]byte(yamlData), opts) + if err == nil { + t.Fatal("expected version 3 to fail, got nil error") + } + const wantSubstr = "unsupported yamlflow version 3 (this build supports up to 2)" + if !strings.Contains(err.Error(), wantSubstr) { + t.Fatalf("expected error to contain %q, got: %v", wantSubstr, err) + } + }) + + t.Run("export emits version as first key", func(t *testing.T) { + opts := GetDefaultOptions(idwrap.NewNow()) + bundle, err := ConvertSimplifiedYAML([]byte(minimalVersionTestYAML), opts) + if err != nil { + t.Fatalf("failed to convert: %v", err) + } + out, err := MarshalSimplifiedYAML(bundle) + if err != nil { + t.Fatalf("failed to marshal: %v", err) + } + const wantFirstLine = "version: 2" + firstLine := strings.SplitN(string(out), "\n", 2)[0] + if firstLine != wantFirstLine { + t.Fatalf("expected first line to be %q, got %q", wantFirstLine, firstLine) + } + }) +} From daad88f6e7770129aab76584ac3051aa8a6ad799 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 18:56:41 +0300 Subject: [PATCH 08/38] fix: import HTTP assertions from yamlflow (were silently dropped) HTTP request assertions were parsed from YAML and correctly merged through templates (mergeHTTPRequestDataStruct), but processRequestStep never converted them into mhttp.HTTPAssert records, so WorkspaceBundle.HTTPAsserts stayed empty for every HTTP request on import. They always exported fine (the exporter reads straight from HTTPAsserts), which is why the bug was invisible round-trip: assertions just quietly vanished on the way in. Mirrors the GraphQL assertion conversion in processGraphQLStructStep, which never had this bug: HTTPAssociatedData gains an Asserts field, processRequestStep converts finalReq.Assertions into HTTPAssert records bound to the HttpID of the request, and mergeAssociatedData folds them into the bundle. Golden corpus updated (-update): only the fixture exercising HTTP request assertions changes, and only by having its assertions reappear (including the use_request template-merge case, which now correctly shows both the assertions from the template and the ones added by the step itself). --- .../yamlflowsimplev2/assertions_test.go | 75 +++++++++++++++++++ .../yamlflowsimplev2/converter_flow.go | 2 + .../yamlflowsimplev2/converter_node.go | 1 + .../yamlflowsimplev2/converter_template.go | 19 +++++ ...equest_headers_assertions_templates.golden | 11 +++ 5 files changed, 108 insertions(+) create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/assertions_test.go diff --git a/packages/server/pkg/translate/yamlflowsimplev2/assertions_test.go b/packages/server/pkg/translate/yamlflowsimplev2/assertions_test.go new file mode 100644 index 000000000..7cdbfb8ea --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/assertions_test.go @@ -0,0 +1,75 @@ +package yamlflowsimplev2 + +import ( + "testing" + + "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mhttp" +) + +// TestHTTPAssertionsImported locks in the fix for a bug where HTTP request +// assertions were parsed from YAML but never converted into mhttp.HTTPAssert +// records, so they silently vanished on import (they always exported fine, +// since WorkspaceBundle.HTTPAsserts simply stayed empty). GraphQL assertions +// never had this bug (see processGraphQLStructStep in converter_node.go). +func TestHTTPAssertionsImported(t *testing.T) { + workspaceID := idwrap.NewNow() + + yamlData := ` +workspace_name: Assertion Test +flows: + - name: AssertFlow + steps: + - manual_start: + name: Start + - request: + name: CheckStatus + depends_on: Start + method: GET + url: https://api.example.com/status + assertions: + - expression: response.status == 200 + enabled: true + - expression: response.body.ok == true + enabled: false +` + + opts := GetDefaultOptions(workspaceID) + result, err := ConvertSimplifiedYAML([]byte(yamlData), opts) + if err != nil { + t.Fatalf("failed to convert: %v", err) + } + + if len(result.HTTPRequests) != 1 { + t.Fatalf("expected 1 HTTP request, got %d", len(result.HTTPRequests)) + } + httpID := result.HTTPRequests[0].ID + + if len(result.HTTPAsserts) != 2 { + t.Fatalf("expected 2 HTTP asserts bound to the request, got %d", len(result.HTTPAsserts)) + } + + byValue := make(map[string]mhttp.HTTPAssert, len(result.HTTPAsserts)) + for _, a := range result.HTTPAsserts { + if a.HttpID.Compare(httpID) != 0 { + t.Errorf("assert %q bound to wrong HttpID: got %s, want %s", a.Value, a.HttpID.String(), httpID.String()) + } + byValue[a.Value] = a + } + + enabledAssert, ok := byValue["response.status == 200"] + if !ok { + t.Fatal("missing assertion 'response.status == 200'") + } + if !enabledAssert.Enabled { + t.Error("expected 'response.status == 200' assertion to be enabled") + } + + disabledAssert, ok := byValue["response.body.ok == true"] + if !ok { + t.Fatal("missing assertion 'response.body.ok == true'") + } + if disabledAssert.Enabled { + t.Error("expected 'response.body.ok == true' assertion to be disabled") + } +} diff --git a/packages/server/pkg/translate/yamlflowsimplev2/converter_flow.go b/packages/server/pkg/translate/yamlflowsimplev2/converter_flow.go index b1148c7ee..062f83295 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/converter_flow.go +++ b/packages/server/pkg/translate/yamlflowsimplev2/converter_flow.go @@ -129,6 +129,7 @@ type HTTPAssociatedData struct { BodyRaw mhttp.HTTPBodyRaw BodyForms []mhttp.HTTPBodyForm BodyUrlencoded []mhttp.HTTPBodyUrlencoded + Asserts []mhttp.HTTPAssert FlowNode *mflow.Node RequestNode *mflow.NodeRequest } @@ -296,6 +297,7 @@ func mergeAssociatedData(result *ioworkspace.WorkspaceBundle, assoc *HTTPAssocia } result.HTTPBodyForms = append(result.HTTPBodyForms, assoc.BodyForms...) result.HTTPBodyUrlencoded = append(result.HTTPBodyUrlencoded, assoc.BodyUrlencoded...) + result.HTTPAsserts = append(result.HTTPAsserts, assoc.Asserts...) if assoc.FlowNode != nil { result.FlowNodes = append(result.FlowNodes, *assoc.FlowNode) diff --git a/packages/server/pkg/translate/yamlflowsimplev2/converter_node.go b/packages/server/pkg/translate/yamlflowsimplev2/converter_node.go index dd5704c49..1769e058a 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/converter_node.go +++ b/packages/server/pkg/translate/yamlflowsimplev2/converter_node.go @@ -357,6 +357,7 @@ func processRequestStep(nodeName string, nodeID, flowID idwrap.IDWrap, step *Yam associated := &HTTPAssociatedData{ Headers: convertToHTTPHeaders(finalReq.Headers, httpID), SearchParams: convertToHTTPSearchParams(finalReq.QueryParams, httpID), + Asserts: convertToHTTPAsserts(finalReq.Assertions, httpID, now), FlowNode: &flowNode, RequestNode: &requestNode, } diff --git a/packages/server/pkg/translate/yamlflowsimplev2/converter_template.go b/packages/server/pkg/translate/yamlflowsimplev2/converter_template.go index 865e552ef..d6aea2cd7 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/converter_template.go +++ b/packages/server/pkg/translate/yamlflowsimplev2/converter_template.go @@ -67,6 +67,25 @@ func convertToHTTPSearchParams(yamlParams []YamlNameValuePairV2, httpID idwrap.I return params } +// convertToHTTPAsserts converts parsed YAML assertions into mhttp.HTTPAssert +// records bound to httpID. Mirrors the GraphQL assertion conversion in +// processGraphQLStructStep (converter_node.go). +func convertToHTTPAsserts(yamlAssertions []YamlAssertionV2, httpID idwrap.IDWrap, now int64) []mhttp.HTTPAssert { + var asserts []mhttp.HTTPAssert + for i, a := range yamlAssertions { + asserts = append(asserts, mhttp.HTTPAssert{ + ID: idwrap.NewNow(), + HttpID: httpID, + Value: a.Expression, + Enabled: a.Enabled, + DisplayOrder: float32(i), + CreatedAt: now, + UpdatedAt: now, + }) + } + return asserts +} + func convertBodyStruct(body *YamlBodyUnion, httpID idwrap.IDWrap, opts ConvertOptionsV2) (mhttp.HTTPBodyRaw, []mhttp.HTTPBodyForm, []mhttp.HTTPBodyUrlencoded, mhttp.HttpBodyKind) { bodyRaw := mhttp.HTTPBodyRaw{ ID: idwrap.NewNow(), diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.golden index 21d376de2..79b92d573 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.golden +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/request_headers_assertions_templates.golden @@ -17,6 +17,11 @@ requests: value: "true" enabled: false body: '{"note":"created by ListFormRequest"}' + assertions: + - expression: response.status == 201 + enabled: true + - expression: response.body.legacyField == null + enabled: false - name: MapFormRequest method: GET url: https://api.example.com/map @@ -26,6 +31,9 @@ requests: query_params: limit: "20" page: "1" + assertions: + - response.status == 200 + - response.body.ok == true - name: TemplatedRequest method: GET url: https://api.example.com/base @@ -33,6 +41,9 @@ requests: Accept: application/json X-Client: golden-corpus X-Extra: template-override + assertions: + - response.status == 200 + - response.body.id != nil flows: - name: RequestFlow steps: From 696d9ed96131fc65e78de901c85255bd8c4c83b5 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 18:58:49 +0300 Subject: [PATCH 09/38] feat: load metrics envelope in TypeSpec Adds the four load-testing metrics/artifact models to the spec, following the existing per-domain .tsp conventions (plain namespace, camelCase fields, Protobuf.WellKnown.Timestamp/Map, small nested helper models): - LoadMetricEntry: one (step, status-class) bucket for a reporting interval - count/errorCount/bytes, hdrHistogram bytes, p50/p90/p95/p99/maxUs - LoadMetricFrame: intervalStart/intervalMs + LoadMetricEntry[] - LoadRunReport: total (LoadStats) + perStep (LoadRunStepStats[], which spreads LoadStats alongside its step/statusClass key - mirrors how CommonTableFields is spread into per-domain child models) - LoadFailureArtifact: step/vu/iteration, request{method,url,headers, bodySample}, optional response{status,headers,bodySample} (absent on transport-level failures), resolvedVariables map, error, capturedAt LoadStatusClass enum members (TwoXx/ThreeXx/FourXx/FiveXx/Error/Timeout) mirror packages/server/pkg/loadmetrics.StatusClass's wire values; header key/value shape mirrors HttpResponseHeader/GraphQLResponseHeader. packages/spec/dist is gitignored and not committed anywhere in this repo (confirmed via `git log -- packages/spec/dist`), so generated output is not included here - regenerate with `pnpm nx run spec:build`. Verified the generated .proto/Go/TS appear correctly and packages/server, packages/db, packages/spec, packages/auth-lib all still `go build ./...` cleanly. Also runs `go mod tidy` in packages/server now that the workspace builds, correcting hdrhistogram-go's require-block placement (it was marked `// indirect` in the previous commit because go mod tidy could not resolve the whole module until spec:build produced packages/spec/dist). --- packages/server/go.mod | 5 +- packages/spec/api/load-metrics.tsp | 96 ++++++++++++++++++++++++++++++ packages/spec/api/main.tsp | 1 + 3 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 packages/spec/api/load-metrics.tsp diff --git a/packages/server/go.mod b/packages/server/go.mod index 2d5c24bb7..2eb66be8d 100644 --- a/packages/server/go.mod +++ b/packages/server/go.mod @@ -4,10 +4,12 @@ go 1.25 require ( connectrpc.com/connect v1.19.1 + github.com/HdrHistogram/hdrhistogram-go v1.3.0 github.com/Microsoft/go-winio v0.6.2 github.com/andybalholm/brotli v1.2.0 github.com/coder/websocket v1.8.14 github.com/expr-lang/expr v1.17.7 + github.com/go-faker/faker/v4 v4.7.0 github.com/goccy/go-json v0.10.5 github.com/golang-jwt/jwt/v5 v5.3.0 github.com/google/uuid v1.6.0 @@ -40,12 +42,10 @@ require ( cloud.google.com/go/iam v1.5.2 // indirect cloud.google.com/go/longrunning v0.6.7 // indirect cloud.google.com/go/vertexai v0.12.0 // indirect - github.com/HdrHistogram/hdrhistogram-go v1.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-faker/faker/v4 v4.7.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/generative-ai-go v0.20.1 // indirect @@ -58,7 +58,6 @@ require ( github.com/pkoukk/tiktoken-go v0.1.6 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/stretchr/objx v0.5.2 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect diff --git a/packages/spec/api/load-metrics.tsp b/packages/spec/api/load-metrics.tsp new file mode 100644 index 000000000..1de0f60fc --- /dev/null +++ b/packages/spec/api/load-metrics.tsp @@ -0,0 +1,96 @@ +using DevTools; + +namespace Api.LoadMetrics; + +@doc("Outcome bucket for one recorded load-test request. Mirrors packages/server/pkg/loadmetrics.StatusClass.") +enum LoadStatusClass { + TwoXx, + ThreeXx, + FourXx, + FiveXx, + Error, + Timeout, +} + +@doc("One (step, status-class) bucket's aggregated stats for a single reporting interval.") +model LoadMetricEntry { + @doc("Name of the load-test step this bucket belongs to") step: string; + statusClass: LoadStatusClass; + count: int64; + errorCount: int64; + bytes: int64; + + @doc("HDR histogram of latencies for this bucket, compressed-encoded (see hdrhistogram-go Histogram.Encode)") + hdrHistogram: bytes; + + @doc("50th percentile latency, in microseconds") p50Us: int64; + @doc("90th percentile latency, in microseconds") p90Us: int64; + @doc("95th percentile latency, in microseconds") p95Us: int64; + @doc("99th percentile latency, in microseconds") p99Us: int64; + @doc("Maximum observed latency, in microseconds") maxUs: int64; +} + +@doc("One flushed interval's worth of aggregated load-test entries.") +model LoadMetricFrame { + intervalStart: Protobuf.WellKnown.Timestamp; + @doc("Duration of the interval covered by this frame, in milliseconds") intervalMs: int64; + entries: LoadMetricEntry[]; +} + +@doc("Derived, percentile-level statistics - either the whole run's total, or one step/status-class breakdown.") +model LoadStats { + count: int64; + errorCount: int64; + bytes: int64; + p50Us: int64; + p90Us: int64; + p95Us: int64; + p99Us: int64; + maxUs: int64; + @doc("Requests per second, i.e. count divided by covered wall time") rps: float32; +} + +model LoadRunStepStats { + step: string; + statusClass: LoadStatusClass; + ...LoadStats; +} + +@doc("Fully-merged report for a completed (or in-progress) load-test run.") +model LoadRunReport { + total: LoadStats; + perStep: LoadRunStepStats[]; +} + +model LoadFailureHeader { + key: string; + value: string; +} + +model LoadFailureRequest { + method: string; + url: string; + headers: LoadFailureHeader[]; + @doc("Truncated preview of the request body, not the full payload") bodySample: string; +} + +model LoadFailureResponse { + status: int32; + headers: LoadFailureHeader[]; + @doc("Truncated preview of the response body, not the full payload") bodySample: string; +} + +@doc("Captured detail for one failed load-test request, for post-run debugging.") +model LoadFailureArtifact { + step: string; + @doc("Virtual user index that produced this failure") vu: int32; + @doc("Iteration number within the virtual user's loop") iteration: int32; + request: LoadFailureRequest; + + @doc("Absent when the failure occurred before a response was received (e.g. transport error, timeout)") + response?: LoadFailureResponse; + + @doc("Variable values resolved into the request at the time it was built") resolvedVariables: Protobuf.Map; + error: string; + capturedAt: Protobuf.WellKnown.Timestamp; +} diff --git a/packages/spec/api/main.tsp b/packages/spec/api/main.tsp index 9fcb49703..417bd8051 100644 --- a/packages/spec/api/main.tsp +++ b/packages/spec/api/main.tsp @@ -12,6 +12,7 @@ import "./graphql.tsp"; import "./health.tsp"; import "./http.tsp"; import "./import.tsp"; +import "./load-metrics.tsp"; import "./private/auth-adapter.tsp"; import "./private/node-js-executor.tsp"; import "./log.tsp"; From a250f1996f67946b595a85eeb6f24a9031c08887 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 19:02:13 +0300 Subject: [PATCH 10/38] feat: run-flows GitHub Action Composite action that downloads the released devtoolscli binary for the runner OS/arch, runs a .yamlflow.yaml file, and publishes a job summary plus JSON/JUnit reports as outputs (json-report, junit-report, success). Replaces hand-rolled pnpm/nx CLI builds in consumer CI: "latest" resolves to the highest cli@* release tag via git ls-remote (the repo also cuts desktop@/web@ releases on the same tracker), the exact asset is verified with curl -fsI before downloading, and errors name the tried URL. Linux/macOS only; every step is shell: bash. --- actions/run-flows/README.md | 116 ++++++++++++++++++ actions/run-flows/action.yml | 78 ++++++++++++ actions/run-flows/scripts/download-cli.sh | 92 ++++++++++++++ actions/run-flows/scripts/finalize.sh | 32 +++++ actions/run-flows/scripts/run-flow.sh | 33 +++++ actions/run-flows/scripts/write-summary.sh | 49 ++++++++ .../run-flows/testdata/smoke.yamlflow.yaml | 38 ++++++ 7 files changed, 438 insertions(+) create mode 100644 actions/run-flows/README.md create mode 100644 actions/run-flows/action.yml create mode 100755 actions/run-flows/scripts/download-cli.sh create mode 100755 actions/run-flows/scripts/finalize.sh create mode 100755 actions/run-flows/scripts/run-flow.sh create mode 100755 actions/run-flows/scripts/write-summary.sh create mode 100644 actions/run-flows/testdata/smoke.yamlflow.yaml diff --git a/actions/run-flows/README.md b/actions/run-flows/README.md new file mode 100644 index 000000000..bbf22d1e3 --- /dev/null +++ b/actions/run-flows/README.md @@ -0,0 +1,116 @@ +# run-flows + +Composite GitHub Action that runs a DevTools `.yamlflow.yaml` file with the +released `devtoolscli` binary, publishes a job summary, and produces JSON / +JUnit reports as step outputs. + +It downloads the `devtoolscli` release binary for the runner's OS/arch itself +— the consuming workflow only needs to check out its own repo (the one +containing the `.yamlflow.yaml` file), not this one: + +```yaml +- uses: the-dev-tools/dev-tools/actions/run-flows@main + with: + file: flows/smoke.yamlflow.yaml +``` + +Pin `@main` to a commit SHA (or a `cli@` tag, which is a normal git +tag on this monorepo) if you want the action's own behavior to stay fixed +independently of the `version` input. + +## Supported runners + +Linux and macOS only (`ubuntu-*`, `macos-*` runners), `x64` and `arm64`. +Windows is out of scope for this action: `devtoolscli` does publish Windows +release assets (see `.github/workflows/release-go.yaml`), but this action +does not resolve or invoke them, and every step is `shell: bash`. Windows CI +should use the manual install steps in [`docs/cli.md`](../../docs/cli.md) +instead. + +## Inputs + +| Name | Required | Default | Description | +| --------------- | -------- | ------------------- | ----------------------------------------------------------------------------------------------------------- | +| `file` | yes | — | Path to the `.yamlflow.yaml` file to run. | +| `flow` | no | _(unset)_ | Single flow name to run. Defaults to the file's top-level `run:` block. | +| `version` | no | `latest` | `devtoolscli` release to install: `latest`, a release tag (`cli@1.0.3`), or a bare version (`1.0.3`). | +| `report-dir` | no | `.devtools-reports` | Directory to write the JSON and JUnit reports into. | +| `fail-on-error` | no | `true` | Fail this step if any flow fails. Set to `'false'` to always exit 0 and check the `success` output instead. | + +## Outputs + +| Name | Description | +| -------------- | ----------------------------------------------------------------- | +| `json-report` | Path to the JSON report (empty string if none was produced). | +| `junit-report` | Path to the JUnit XML report (empty string if none was produced). | +| `success` | `'true'` if every flow in the run succeeded, `'false'` otherwise. | + +A job summary table (flow name, ✅/❌ status, duration) is always published to +the job's summary, even when `fail-on-error: 'false'` or the run fails — +useful for `if: always()` follow-up steps. + +## Examples + +### PR check + +```yaml +name: API flows +on: + pull_request: +jobs: + flows: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: the-dev-tools/dev-tools/actions/run-flows@main + with: + file: flows/smoke.yamlflow.yaml +``` + +### Nightly cron against staging + +```yaml +name: Nightly flow check +on: + schedule: + - cron: '0 6 * * *' +jobs: + flows: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: the-dev-tools/dev-tools/actions/run-flows@main + id: flows + with: + file: flows/nightly.yamlflow.yaml + version: cli@1.0.3 + fail-on-error: 'false' + env: + LOGIN_EMAIL: ${{ secrets.LOGIN_EMAIL }} + LOGIN_PASSWORD: ${{ secrets.LOGIN_PASSWORD }} + - name: Notify on failure + if: steps.flows.outputs.success != 'true' + run: echo "Nightly flows failed — see the job summary for details" # replace with a real notification step +``` + +`env:` overrides for the YAML file's `#env:NAME` placeholders work the same +way as in the manual CLI usage documented in +[`docs/cli.md`](../../docs/cli.md#environment-variable-overrides). + +## How it works + +1. Resolves `version` to a release tag (`cli@`) and downloads the + matching `devtools-cli---` asset from this repo's + GitHub Releases into `$RUNNER_TEMP/devtools/bin`. `latest` resolves to the + highest `cli@*` tag via `git ls-remote` (the repo also cuts `desktop@`/ + `web@`/etc. releases, so a plain "latest release" API lookup would not be + specific enough). +2. Runs `devtoolscli flow run [flow] --report console --report json:/report.json --report junit:/junit.xml`. +3. Publishes a job summary table from the JSON report, whether or not the run + succeeded. +4. Sets the `json-report` / `junit-report` / `success` outputs, then fails + the step if `fail-on-error` is `'true'` (the default) and any flow failed. + +See `apps/cli/internal/reporter/reporter.go` for the JSON report schema and +[`docs/cli.md`](../../docs/cli.md) for the YAML flow format and the +`{{ ... }}` interpolation syntax used in `testdata/smoke.yamlflow.yaml`. diff --git a/actions/run-flows/action.yml b/actions/run-flows/action.yml new file mode 100644 index 000000000..d4844b879 --- /dev/null +++ b/actions/run-flows/action.yml @@ -0,0 +1,78 @@ +name: 'DevTools Run Flows' +description: 'Run a DevTools YAML flow with the released devtoolscli CLI and publish JSON/JUnit reports plus a job summary.' +author: 'the-dev-tools' + +inputs: + file: + description: 'Path to the .yamlflow.yaml file to run.' + required: true + flow: + description: "Single flow name to run. Defaults to the file's top-level run: block." + required: false + version: + description: "devtoolscli release to install: 'latest' or a release tag, e.g. cli@1.0.3." + required: false + default: 'latest' + report-dir: + description: 'Directory to write the JSON and JUnit reports into.' + required: false + default: '.devtools-reports' + fail-on-error: + description: "Fail this step if any flow fails. Set to 'false' to always exit 0 and inspect the success output instead." + required: false + default: 'true' + +outputs: + json-report: + description: 'Path to the JSON report (empty string if none was produced).' + value: ${{ steps.finalize.outputs.json-report }} + junit-report: + description: 'Path to the JUnit XML report (empty string if none was produced).' + value: ${{ steps.finalize.outputs.junit-report }} + success: + description: "'true' if every flow in the run succeeded, 'false' otherwise." + value: ${{ steps.finalize.outputs.success }} + +runs: + using: composite + steps: + # linux/macOS only — devtoolscli release-go.yaml does not publish a + # Windows-friendly layout for this action to consume; see README.md. + - name: Download devtoolscli + id: download + shell: bash + run: '"$GITHUB_ACTION_PATH/scripts/download-cli.sh"' + env: + VERSION: ${{ inputs.version }} + + # continue-on-error: a failing flow run must not skip the summary/output + # steps below — they need to run via if: always() regardless. + - name: Run flow + id: run + shell: bash + continue-on-error: true + run: '"$GITHUB_ACTION_PATH/scripts/run-flow.sh"' + env: + CLI_BIN: ${{ steps.download.outputs.bin }} + FILE: ${{ inputs.file }} + FLOW: ${{ inputs.flow }} + REPORT_DIR: ${{ inputs.report-dir }} + + - name: Write job summary + id: summary + if: always() + shell: bash + run: '"$GITHUB_ACTION_PATH/scripts/write-summary.sh"' + env: + REPORT_DIR: ${{ inputs.report-dir }} + RUN_OUTCOME: ${{ steps.run.outcome }} + + - name: Set outputs + id: finalize + if: always() + shell: bash + run: '"$GITHUB_ACTION_PATH/scripts/finalize.sh"' + env: + REPORT_DIR: ${{ inputs.report-dir }} + RUN_OUTCOME: ${{ steps.run.outcome }} + FAIL_ON_ERROR: ${{ inputs.fail-on-error }} diff --git a/actions/run-flows/scripts/download-cli.sh b/actions/run-flows/scripts/download-cli.sh new file mode 100755 index 000000000..036376e75 --- /dev/null +++ b/actions/run-flows/scripts/download-cli.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Resolves the devtoolscli release for this runner's OS/arch and downloads it +# into $RUNNER_TEMP/devtools/bin. Reused by actions/run-flows/action.yml. +# +# Env in: +# VERSION - "latest" or a release tag, e.g. "cli@1.0.3" (also accepts a +# bare version like "1.0.3") +# RUNNER_OS - set by GitHub Actions ("Linux" / "macOS" / "Windows") +# RUNNER_ARCH - set by GitHub Actions ("X64" / "ARM64" / ...) +# RUNNER_TEMP - set by GitHub Actions; falls back to /tmp for local runs +# GITHUB_PATH - GitHub Actions path file (appended to) +# GITHUB_OUTPUT - GitHub Actions output file (appended to) +# +# Outputs (via $GITHUB_OUTPUT): +# bin - absolute path to the installed devtoolscli binary +# version - resolved version number (without the "cli@" prefix) +set -euo pipefail + +REPO_OWNER='the-dev-tools' +REPO_NAME='dev-tools' +REPO_URL="https://github.com/${REPO_OWNER}/${REPO_NAME}" + +version_input="${VERSION:-latest}" + +case "${RUNNER_OS:-}" in + Linux) os='linux' ;; + macOS) os='darwin' ;; + *) + echo "::error::run-flows only supports Linux and macOS runners (got RUNNER_OS='${RUNNER_OS:-}'). Windows is out of scope — see actions/run-flows/README.md." >&2 + exit 1 + ;; +esac + +case "${RUNNER_ARCH:-}" in + X64) arch='x64' ;; + ARM64) arch='arm64' ;; + *) + echo "::error::run-flows only supports X64 and ARM64 runners (got RUNNER_ARCH='${RUNNER_ARCH:-}')." >&2 + exit 1 + ;; +esac + +platform="${os}-${arch}" + +# Resolve "latest"/bare-version/tag input into a concrete release tag. Releases +# for the CLI are tagged "cli@" (see tools/gha-scripts/src/cli.ts and +# .github/workflows/release-go.yaml); the repo also cuts desktop@/web@ releases +# on the same tracker, so "latest release" APIs can't be used as-is. +if [[ "$version_input" == 'latest' ]]; then + set +e + tag=$(git ls-remote --tags --refs "${REPO_URL}.git" 'cli@*' 2>/dev/null | sed 's#.*refs/tags/##' | sort -V | tail -n1) + set -e + if [[ -z "$tag" ]]; then + echo "::error::Could not resolve the latest devtoolscli release: no cli@* tags found on ${REPO_URL} (or the network request failed)." >&2 + exit 1 + fi +elif [[ "$version_input" == cli@* ]]; then + tag="$version_input" +else + tag="cli@${version_input}" +fi +version_number="${tag#cli@}" + +asset_name="devtools-cli-${version_number}-${platform}" +download_url="${REPO_URL}/releases/download/${tag}/${asset_name}" + +echo "Resolving devtoolscli ${tag} for ${platform}..." + +if ! curl -fsSI -o /dev/null "$download_url"; then + echo "::error::devtoolscli release asset not found: ${download_url}" >&2 + echo "::error::Checked tag '${tag}' (from version input '${version_input}'). Confirm it exists at ${REPO_URL}/releases/tag/${tag} and that asset naming still matches 'devtools-cli---' (see .github/workflows/release-go.yaml)." >&2 + exit 1 +fi + +bin_dir="${RUNNER_TEMP:-/tmp}/devtools/bin" +bin_path="${bin_dir}/devtoolscli" +mkdir -p "$bin_dir" + +if ! curl -fsSL -o "$bin_path" "$download_url"; then + echo "::error::Failed to download ${download_url}" >&2 + exit 1 +fi +chmod +x "$bin_path" + +echo "Installed devtoolscli ${version_number} -> ${bin_path}" +"$bin_path" version + +echo "$bin_dir" >> "$GITHUB_PATH" +{ + echo "bin=${bin_path}" + echo "version=${version_number}" +} >> "$GITHUB_OUTPUT" diff --git a/actions/run-flows/scripts/finalize.sh b/actions/run-flows/scripts/finalize.sh new file mode 100755 index 000000000..0a7c417e0 --- /dev/null +++ b/actions/run-flows/scripts/finalize.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Sets the action's outputs (json-report, junit-report, success) and enforces +# the fail-on-error input. Runs with `if: always()` in action.yml so outputs +# are always set, even when the flow run failed. +# +# Env in: +# REPORT_DIR - directory containing report.json/junit.xml (inputs.report-dir) +# RUN_OUTCOME - outcome of the "Run flow" step ("success"/"failure"/"") +# FAIL_ON_ERROR - inputs.fail-on-error ("true"/"false") +set -uo pipefail + +json_report="${REPORT_DIR:-.devtools-reports}/report.json" +junit_report="${REPORT_DIR:-.devtools-reports}/junit.xml" + +[[ -f "$json_report" ]] || json_report='' +[[ -f "$junit_report" ]] || junit_report='' + +success='false' +[[ "${RUN_OUTCOME:-}" == 'success' ]] && success='true' + +{ + echo "json-report=${json_report}" + echo "junit-report=${junit_report}" + echo "success=${success}" +} >> "$GITHUB_OUTPUT" + +if [[ "$success" == 'false' && "${FAIL_ON_ERROR:-true}" == 'true' ]]; then + echo "::error::devtoolscli flow run did not complete successfully (run step outcome: ${RUN_OUTCOME:-unknown}) and fail-on-error is 'true'." >&2 + exit 1 +fi + +exit 0 diff --git a/actions/run-flows/scripts/run-flow.sh b/actions/run-flows/scripts/run-flow.sh new file mode 100755 index 000000000..2edfb2110 --- /dev/null +++ b/actions/run-flows/scripts/run-flow.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Runs the resolved devtoolscli binary against the given yamlflow file. +# Exits with devtoolscli's own exit code (0 on success, non-zero if any flow +# failed) — the calling step uses continue-on-error so a failing run doesn't +# skip the summary/output steps that follow it. +# +# Env in: +# CLI_BIN - absolute path to the devtoolscli binary (from download-cli.sh) +# FILE - path to the .yamlflow.yaml file (inputs.file) +# FLOW - optional single flow name (inputs.flow) +# REPORT_DIR - directory to write json/junit reports into (inputs.report-dir) +set -uo pipefail + +if [[ -z "${CLI_BIN:-}" || ! -x "$CLI_BIN" ]]; then + echo "::error::run-flow.sh: CLI_BIN ('${CLI_BIN:-}') is not an executable file." >&2 + exit 1 +fi + +if [[ -z "${FILE:-}" ]]; then + echo "::error::run-flow.sh: the 'file' input is required." >&2 + exit 1 +fi + +report_dir="${REPORT_DIR:-.devtools-reports}" + +args=(flow run "$FILE") +if [[ -n "${FLOW:-}" ]]; then + args+=("$FLOW") +fi +args+=(--report console --report "json:${report_dir}/report.json" --report "junit:${report_dir}/junit.xml") + +echo "+ devtoolscli ${args[*]}" +"$CLI_BIN" "${args[@]}" diff --git a/actions/run-flows/scripts/write-summary.sh b/actions/run-flows/scripts/write-summary.sh new file mode 100755 index 000000000..64164442c --- /dev/null +++ b/actions/run-flows/scripts/write-summary.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Appends a flows/status/duration table to $GITHUB_STEP_SUMMARY from the JSON +# report. Runs with `if: always()` in action.yml so it still produces a +# summary when the flow run step failed. Deliberately does not use `set -e`: +# a malformed report should degrade the summary, not abort the composite +# action before outputs get set by finalize.sh. +# +# Env in: +# REPORT_DIR - directory containing report.json (inputs.report-dir) +# RUN_OUTCOME - outcome of the "Run flow" step ("success"/"failure"/"" ) +set -uo pipefail + +report_json="${REPORT_DIR:-.devtools-reports}/report.json" +summary_file="${GITHUB_STEP_SUMMARY:-/dev/stdout}" + +run_outcome="${RUN_OUTCOME:-unknown}" +if [[ "$run_outcome" == 'success' ]]; then + heading='DevTools flow run — success' +else + heading='DevTools flow run — failed' +fi + +{ + echo "### ${heading}" + echo + + if [[ ! -f "$report_json" ]]; then + echo "_No JSON report was produced at \`${report_json}\`. The \"Run flow\" step outcome was **${run_outcome}** — check its logs above._" + exit 0 + fi + + if ! total=$(jq 'length' "$report_json" 2>/dev/null); then + echo "_Could not parse \`${report_json}\` as JSON. The \"Run flow\" step outcome was **${run_outcome}**._" + exit 0 + fi + passed=$(jq '[.[] | select(.status == "success")] | length' "$report_json") + + echo "**Flows:** ${passed}/${total} passed" + echo + echo '| Flow | Status | Duration |' + echo '| --- | --- | --- |' + jq -r ' + .[] + | "| " + .flow_name + + " | " + (if .status == "success" then "✅ Success" else "❌ Failed" end) + + " | " + (((.duration / 1000000000 * 100 | round) / 100 | tostring) + "s") + + " |" + ' "$report_json" +} >> "$summary_file" diff --git a/actions/run-flows/testdata/smoke.yamlflow.yaml b/actions/run-flows/testdata/smoke.yamlflow.yaml new file mode 100644 index 000000000..1dd85a8dc --- /dev/null +++ b/actions/run-flows/testdata/smoke.yamlflow.yaml @@ -0,0 +1,38 @@ +workspace_name: run-flows action smoke test +# Minimal fixture for actions/run-flows CI validation (action-test.yaml). +# Plain HTTP requests only (no js/websocket nodes) so it needs nothing but +# network access to https://jsonplaceholder.typicode.com. Uses the CLI's +# real interpolation syntax: {{ node_name.field }} — see docs/cli.md and +# apps/cli/test/yamlflow/ws_run_example.yaml. Do NOT copy the ${var} syntax +# used by other fixtures under apps/cli/test/yamlflow/ — it is stale and not +# implemented by the current interpolation engine (packages/server/pkg/expression). +run: + - flow: fetch-user + - flow: create-post + depends_on: fetch-user +flows: + - name: fetch-user + steps: + - request: + name: get_user + method: GET + url: https://jsonplaceholder.typicode.com/users/1 + - request: + name: get_user_posts + method: GET + url: https://jsonplaceholder.typicode.com/posts + query_params: + userId: '{{ get_user.response.body.id }}' + depends_on: get_user + - name: create-post + steps: + - request: + name: create_post + method: POST + url: https://jsonplaceholder.typicode.com/posts + headers: + Content-Type: application/json + body: + title: 'run-flows smoke test' + body: 'Created by actions/run-flows/testdata/smoke.yamlflow.yaml' + userId: 1 From db34e1188a0835249983f4d1acc7255b126786af Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 19:02:19 +0300 Subject: [PATCH 11/38] ci: add run-flows action test workflow Exercises the composite action end to end on ubuntu-latest and macos-latest: downloads the latest published cli@ release, runs the smoke fixture, and asserts the json-report/junit-report files and job summary were produced. Triggered on pull_request for actions/** changes plus workflow_dispatch. --- .github/workflows/action-test.yaml | 47 ++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/action-test.yaml diff --git a/.github/workflows/action-test.yaml b/.github/workflows/action-test.yaml new file mode 100644 index 000000000..e7dedbb02 --- /dev/null +++ b/.github/workflows/action-test.yaml @@ -0,0 +1,47 @@ +name: Action / run-flows + +on: + pull_request: + paths: + - 'actions/**' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Test run-flows (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + + - name: Run smoke flow + id: run-flows + uses: ./actions/run-flows + with: + file: actions/run-flows/testdata/smoke.yamlflow.yaml + version: latest + + - name: Assert outputs and summary + shell: bash + run: | + set -euo pipefail + + echo "json-report=${{ steps.run-flows.outputs.json-report }}" + echo "junit-report=${{ steps.run-flows.outputs.junit-report }}" + echo "success=${{ steps.run-flows.outputs.success }}" + + test -f "${{ steps.run-flows.outputs.json-report }}" + test -f "${{ steps.run-flows.outputs.junit-report }}" + + [ "${{ steps.run-flows.outputs.success }}" = "true" ] + + grep -q "DevTools flow run" "$GITHUB_STEP_SUMMARY" + grep -q "fetch-user" "$GITHUB_STEP_SUMMARY" + grep -q "create-post" "$GITHUB_STEP_SUMMARY" From f48468e6670a4af54fea4c19e6113a22f4eff139 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 19:02:27 +0300 Subject: [PATCH 12/38] docs: point cli.md CI guidance at the run-flows action Replace the DIY GitHub Actions snippet (which built via pnpm nx run cli:build - a server-only target missing the cli tag, so the resulting binary could not actually run flow commands) with the actions/run-flows composite action. Keep a corrected manual alternative using install.sh for Windows/air-gapped cases, invoked as `devtools` (the name install.sh actually installs it as, not `devtoolscli`). --- docs/cli.md | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index da2b0755b..ef00c6f60 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -77,7 +77,13 @@ You can specify the flag multiple times. When writing JSON or JUnit reports, the 3. If your flows depend on external APIs, run them against staging environments or mock servers to keep CI stable. 4. Consider adding `devtoolscli version` to your pipeline logs so you can diagnose regressions quickly. -A minimal GitHub Actions job looks like: +### GitHub Actions + +The bundled composite action downloads a released `devtoolscli` binary for +the runner's OS/arch, runs the flow, and publishes a job summary plus +JSON/JUnit reports as outputs — no repo checkout of DevTools itself or Nix/pnpm +toolchain needed. See [`actions/run-flows/README.md`](../actions/run-flows/README.md) +for the full inputs/outputs reference: ```yaml jobs: @@ -85,12 +91,29 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v3 + - uses: the-dev-tools/dev-tools/actions/run-flows@main with: - version: 9 - - run: pnpm install --frozen-lockfile - - run: pnpm nx run cli:build - - run: ./apps/cli/dist/devtoolscli flow run flows.yamlflow.yaml + file: flows.yamlflow.yaml + env: + LOGIN_EMAIL: ${{ secrets.LOGIN_EMAIL }} + LOGIN_PASSWORD: ${{ secrets.LOGIN_PASSWORD }} +``` + +#### Manual alternative + +Windows runners, air-gapped environments, or anything else the action doesn't +cover can install the CLI directly (see [Installation](#installation) above) +and call `flow run` themselves. Note the installed binary is named `devtools`, +not `devtoolscli`: + +```yaml +jobs: + flow-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: curl -fsSL https://raw.githubusercontent.com/the-dev-tools/dev-tools/main/apps/cli/install.sh | bash + - run: devtools flow run flows.yamlflow.yaml --report console --report junit:report.xml env: LOGIN_EMAIL: ${{ secrets.LOGIN_EMAIL }} LOGIN_PASSWORD: ${{ secrets.LOGIN_PASSWORD }} From f7e83a3c391a54fe298a9266ba5c514e623a4566 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 19:02:37 +0300 Subject: [PATCH 13/38] style: format load-metrics.tsp with prettier Wraps the resolvedVariables map type per the repo prettier config (checked by root:lint:format); no semantic change - verified spec:build output is byte-identical before and after. --- packages/spec/api/load-metrics.tsp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/spec/api/load-metrics.tsp b/packages/spec/api/load-metrics.tsp index 1de0f60fc..8f65ee904 100644 --- a/packages/spec/api/load-metrics.tsp +++ b/packages/spec/api/load-metrics.tsp @@ -90,7 +90,10 @@ model LoadFailureArtifact { @doc("Absent when the failure occurred before a response was received (e.g. transport error, timeout)") response?: LoadFailureResponse; - @doc("Variable values resolved into the request at the time it was built") resolvedVariables: Protobuf.Map; + @doc("Variable values resolved into the request at the time it was built") resolvedVariables: Protobuf.Map< + string, + string + >; error: string; capturedAt: Protobuf.WellKnown.Timestamp; } From fbd11a092fdcc70fe33b1a4945ee8381d53be83d Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 19:04:06 +0300 Subject: [PATCH 14/38] docs: correct how scenariorunner enforces the iteration bound The claim() comment said all stop conditions are checked before the sequence number is reserved. Only cancellation and the deadline are: the iteration bound is enforced after the atomic add, by discarding an over-limit claim, so next overruns MaxIterations by up to VUs. That distinction matters. A maintainer trusting the old comment could delete the post-add discard believing a pre-check covered the bound, which would silently break the exact iteration count. Moving the check before the add would be worse still: several workers could read the same pre-add value and all conclude they were under the limit. Comment only; no behaviour change. --- .../flow/runner/scenariorunner/scenariorunner.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/server/pkg/flow/runner/scenariorunner/scenariorunner.go b/packages/server/pkg/flow/runner/scenariorunner/scenariorunner.go index 30f411af5..794128c9b 100644 --- a/packages/server/pkg/flow/runner/scenariorunner/scenariorunner.go +++ b/packages/server/pkg/flow/runner/scenariorunner/scenariorunner.go @@ -112,8 +112,18 @@ func Run(ctx context.Context, prof RunProfile, iter func(ctx context.Context, vu } // claim reserves the next sequence number, or reports that the worker should -// stop. The stop conditions are checked before the number is reserved so that -// an iteration bound is never consumed by an iteration that does not run. +// stop. +// +// Cancellation and the duration deadline are checked before the reservation, so +// no sequence number is burned once either has tripped. The iteration bound is +// enforced after it instead: an over-limit claim is discarded rather than run. +// next therefore overruns MaxIterations by up to VUs, which is harmless because +// Summary counts iterations that executed, not claims that were made. +// +// The post-reservation discard is what makes Iterations == MaxIterations exact. +// Do not drop it on the assumption that a pre-check covers the bound; checking +// the bound before the atomic add would let several workers read the same value +// and overshoot. func claim(ctx context.Context, next *atomic.Int64, maxIterations int64, deadline time.Time) (int64, bool) { if ctx.Err() != nil { return 0, false From b5c78d4950d24c1ae32d1b3d4bd715c1b1bcd344 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 19:14:18 +0300 Subject: [PATCH 15/38] fix: run block executes in dependency order and rejects unknown deps RunMultipleFlows previously ran run: block entries in file declaration order and only checked a dependency if that dependency happened to have already run earlier in that same order, so an unknown or forward declared dependency name was silently ignored, and a failed dependency made the whole function return immediately with no record of what happened to the flows that never got a chance to run. Replace the ad-hoc map[string]interface{} re-parse of the run: block with the typed yamlflowsimplev2 structs (gets the depends_on scalar-or-list form for free), then topologically sort entries with Kahn algorithm before executing anything, breaking ties by original list order for determinism. Unknown dependencies and dependency cycles are now hard errors naming the offending value: unknown dependency "Missing" in run block (known flows: A, B) dependency cycle in run block: A -> B -> A Flows still execute sequentially. A flow gated by a failed or skipped dependency is now recorded and reported as skipped (status plus reason, through the logger and any configured reporters) instead of silently never attempted; flows with no such gate still run even if an unrelated earlier flow failed. The overall call still returns a non-nil error whenever any flow failed or was skipped, preserving the existing failure-gate contract. --- apps/cli/internal/runner/run_order.go | 197 ++++++++++++++++++ apps/cli/internal/runner/runner.go | 145 ++++++------- apps/cli/internal/runner/runner_test.go | 259 +++++++++++++++++++++++- 3 files changed, 522 insertions(+), 79 deletions(-) create mode 100644 apps/cli/internal/runner/run_order.go diff --git a/apps/cli/internal/runner/run_order.go b/apps/cli/internal/runner/run_order.go new file mode 100644 index 000000000..51d6a3651 --- /dev/null +++ b/apps/cli/internal/runner/run_order.go @@ -0,0 +1,197 @@ +package runner + +import ( + "fmt" + "sort" + "strings" + + yamlflowsimplev2 "github.com/the-dev-tools/dev-tools/packages/server/pkg/translate/yamlflowsimplev2" + + "gopkg.in/yaml.v3" +) + +// runEntry is a parsed run: block entry: a flow name plus the names of the +// flows (also in the run: block) that must complete successfully first. +type runEntry struct { + flowName string + dependsOn []string +} + +// parseRunEntries extracts the run: block from the workflow file using the +// typed yamlflowsimplev2 structs, instead of re-parsing it by hand as +// map[string]interface{}. This gets depends_on's scalar-or-list form +// (StringOrSlice) for free and matches exactly how the rest of the yamlflow +// contract is parsed elsewhere. +func parseRunEntries(fileData []byte) ([]runEntry, error) { + var doc struct { + Run []yamlflowsimplev2.YamlRunEntryV2 `yaml:"run"` + } + if err := yaml.Unmarshal(fileData, &doc); err != nil { + return nil, fmt.Errorf("failed to unmarshal YAML: %w", err) + } + + entries := make([]runEntry, 0, len(doc.Run)) + for _, re := range doc.Run { + if re.Flow == "" { + continue + } + entries = append(entries, runEntry{ + flowName: re.Flow, + dependsOn: []string(re.DependsOn), + }) + } + + if len(entries) == 0 { + return nil, fmt.Errorf("no run field found in workflow") + } + + return entries, nil +} + +// topoSortRunEntries orders run: block entries so that every flow appears +// after all of its dependencies, using Kahn's algorithm. Ties (multiple +// flows simultaneously ready to run) are broken by original run: block +// order, so the result is deterministic for a given file. +// +// Returns an error naming the offending dependency if depends_on references +// a flow that is not itself part of the run: block, or naming an example +// cycle if the dependency graph is not a DAG. +func topoSortRunEntries(entries []runEntry) ([]runEntry, error) { + byName := make(map[string]runEntry, len(entries)) + declOrder := make(map[string]int, len(entries)) + names := make([]string, 0, len(entries)) + for i, e := range entries { + byName[e.flowName] = e + declOrder[e.flowName] = i + names = append(names, e.flowName) + } + + for _, e := range entries { + for _, dep := range e.dependsOn { + if _, ok := byName[dep]; !ok { + return nil, fmt.Errorf("unknown dependency %q in run block (known flows: %s)", dep, strings.Join(names, ", ")) + } + } + } + + // dependents[X] = flows that declare a dependency on X. + inDegree := make(map[string]int, len(entries)) + dependents := make(map[string][]string, len(entries)) + for _, e := range entries { + inDegree[e.flowName] = len(e.dependsOn) + for _, dep := range e.dependsOn { + dependents[dep] = append(dependents[dep], e.flowName) + } + } + + byDeclOrder := func(s []string) { + sort.SliceStable(s, func(i, j int) bool { return declOrder[s[i]] < declOrder[s[j]] }) + } + + var ready []string + for _, name := range names { + if inDegree[name] == 0 { + ready = append(ready, name) + } + } + byDeclOrder(ready) + + sorted := make([]runEntry, 0, len(entries)) + for len(ready) > 0 { + name := ready[0] + ready = ready[1:] + sorted = append(sorted, byName[name]) + + newlyReady := append([]string(nil), dependents[name]...) + byDeclOrder(newlyReady) + for _, dependent := range newlyReady { + inDegree[dependent]-- + if inDegree[dependent] == 0 { + ready = append(ready, dependent) + byDeclOrder(ready) + } + } + } + + if len(sorted) != len(entries) { + remaining := make(map[string]bool) + for _, name := range names { + if inDegree[name] > 0 { + remaining[name] = true + } + } + cycle := findCycle(byName, remaining) + return nil, fmt.Errorf("dependency cycle in run block: %s", strings.Join(cycle, " → ")) + } + + return sorted, nil +} + +// findCycle returns one dependency cycle among the given remaining flows, as +// a path starting and ending on the same flow name (e.g. [A B A]). remaining +// is guaranteed non-empty and every entry in it sits on at least one cycle +// (Kahn's algorithm only leaves nodes behind when they are part of a cycle, +// or depend - transitively - on one). The starting node is chosen +// deterministically (smallest name) so the reported cycle is stable. +func findCycle(byName map[string]runEntry, remaining map[string]bool) []string { + startNames := make([]string, 0, len(remaining)) + for name := range remaining { + startNames = append(startNames, name) + } + sort.Strings(startNames) + + visited := make(map[string]bool) + onPath := make(map[string]int) + var path []string + + var visit func(name string) []string + visit = func(name string) []string { + if idx, ok := onPath[name]; ok { + cycle := append([]string(nil), path[idx:]...) + return append(cycle, name) + } + if visited[name] { + return nil + } + visited[name] = true + onPath[name] = len(path) + path = append(path, name) + + deps := append([]string(nil), byName[name].dependsOn...) + sort.Strings(deps) + for _, dep := range deps { + if !remaining[dep] { + continue + } + if cycle := visit(dep); cycle != nil { + return cycle + } + } + + path = path[:len(path)-1] + delete(onPath, name) + return nil + } + + for _, name := range startNames { + if cycle := visit(name); cycle != nil { + return cycle + } + } + + // Unreachable: Kahn's algorithm guarantees every remaining node is part + // of (or feeds into) a cycle, so visit() above always finds one. + return startNames +} + +// firstUnsuccessfulDependency returns the name of the first dependency (in +// declared order) that is missing a successful result, if any. "Missing" and +// "skipped" both count so that a skip cascades to transitive dependents. +func firstUnsuccessfulDependency(entry runEntry, statusByFlow map[string]string) (string, bool) { + for _, dep := range entry.dependsOn { + if status, ok := statusByFlow[dep]; !ok || !strings.EqualFold(status, "success") { + return dep, true + } + } + return "", false +} diff --git a/apps/cli/internal/runner/runner.go b/apps/cli/internal/runner/runner.go index 325da9dc9..ceeb11c5e 100644 --- a/apps/cli/internal/runner/runner.go +++ b/apps/cli/internal/runner/runner.go @@ -27,7 +27,6 @@ import ( "github.com/the-dev-tools/dev-tools/packages/server/pkg/service/sflow" "connectrpc.com/connect" - "gopkg.in/yaml.v3" ) type RunnerServices struct { @@ -38,93 +37,73 @@ type RunnerServices struct { JSClient node_js_executorv1connect.NodeJsExecutorServiceClient } -// RunMultipleFlows executes multiple flows based on the run field configuration +// RunMultipleFlows executes multiple flows based on the run field configuration. +// Flows run sequentially in dependency order (a topological sort of the +// run: block, ties broken by original list order) rather than run: block +// declaration order. A flow whose dependency failed, or was itself skipped, +// is reported as skipped instead of attempted; flows with no such gate still +// run even if an unrelated earlier flow failed. func RunMultipleFlows(ctx context.Context, fileData []byte, allFlows []mflow.Flow, services RunnerServices, logger *slog.Logger, reporters *reporter.ReporterGroup) error { - // Parse the run field to get flow order and dependencies - var rawYAML map[string]interface{} - if err := yaml.Unmarshal(fileData, &rawYAML); err != nil { - return fmt.Errorf("failed to unmarshal YAML: %w", err) - } - - runField, ok := rawYAML["run"].([]interface{}) - if !ok || len(runField) == 0 { - return fmt.Errorf("no run field found in workflow") + entries, err := parseRunEntries(fileData) + if err != nil { + return err } - // Parse run entries - type runEntry struct { - flowName string - dependsOn []string + // Create flow map for easy lookup, and fail fast if the run: block names + // a flow that does not exist, before running anything. + flowMap := make(map[string]*mflow.Flow, len(allFlows)) + for i := range allFlows { + flowMap[allFlows[i].Name] = &allFlows[i] } - - var runEntries []runEntry - for _, entry := range runField { - entryMap, ok := entry.(map[string]interface{}) - if !ok { - continue - } - - flowName, ok := entryMap["flow"].(string) - if !ok || flowName == "" { - continue - } - - re := runEntry{flowName: flowName} - - // Parse dependencies - if deps, ok := entryMap["depends_on"]; ok { - switch v := deps.(type) { - case string: - re.dependsOn = []string{v} - case []interface{}: - for _, dep := range v { - if depStr, ok := dep.(string); ok { - re.dependsOn = append(re.dependsOn, depStr) - } - } - } + for _, entry := range entries { + if _, exists := flowMap[entry.flowName]; !exists { + return fmt.Errorf("flow '%s' not found in workflow", entry.flowName) } - - runEntries = append(runEntries, re) } - // Create flow map for easy lookup - flowMap := make(map[string]*mflow.Flow) - for i := range allFlows { - flowMap[allFlows[i].Name] = &allFlows[i] + sorted, err := topoSortRunEntries(entries) + if err != nil { + return err } // Track execution results - executionResults := make(map[string]model.FlowRunResult) + executionResults := make(map[string]model.FlowRunResult, len(sorted)) + statusByFlow := make(map[string]string, len(sorted)) consoleEnabled := reporters != nil && reporters.HasConsole() - // Execute flows in order + // Execute flows in dependency order if consoleEnabled { fmt.Println("\n=== Multi-Flow Execution Starting ===") - fmt.Printf("Flows to execute: %d\n", len(runEntries)) + fmt.Printf("Flows to execute: %d\n", len(sorted)) } overallStartTime := time.Now() - for i, entry := range runEntries { - flow, exists := flowMap[entry.flowName] - if !exists { - return fmt.Errorf("flow '%s' not found in workflow", entry.flowName) - } + for i, entry := range sorted { + if failedDep, gated := firstUnsuccessfulDependency(entry, statusByFlow); gated { + reason := fmt.Sprintf("dependency %q failed", failedDep) + result := model.FlowRunResult{ + FlowName: entry.flowName, + Status: "skipped", + Error: reason, + } + executionResults[entry.flowName] = result + statusByFlow[entry.flowName] = result.Status - // Check dependencies - for _, dep := range entry.dependsOn { - // Check if dependency is a flow - if depResult, ok := executionResults[dep]; ok { - if !strings.EqualFold(depResult.Status, "success") { - return fmt.Errorf("flow '%s' depends on '%s' which failed", entry.flowName, dep) - } + logger.Warn("flow skipped", "flow", entry.flowName, "reason", reason) + if reporters != nil { + reporters.HandleFlowResult(result) } - // Note: We could also check for node dependencies here in the future + if consoleEnabled { + fmt.Printf("\n[%d/%d] Skipping flow: %s (%s)\n", i+1, len(sorted), entry.flowName, reason) + } + continue } + flow := flowMap[entry.flowName] + if consoleEnabled { - fmt.Printf("\n[%d/%d] Executing flow: %s\n", i+1, len(runEntries), entry.flowName) + fmt.Printf("\n[%d/%d] Executing flow: %s\n", i+1, len(sorted), entry.flowName) if len(entry.dependsOn) > 0 { fmt.Printf(" Dependencies: %v\n", entry.dependsOn) } @@ -132,6 +111,7 @@ func RunMultipleFlows(ctx context.Context, fileData []byte, allFlows []mflow.Flo result, err := RunFlow(ctx, flow, services, reporters) executionResults[entry.flowName] = result + statusByFlow[entry.flowName] = result.Status if err != nil { if consoleEnabled { @@ -150,27 +130,36 @@ func RunMultipleFlows(ctx context.Context, fileData []byte, allFlows []mflow.Flo fmt.Println("\nFlow Results:") successCount := 0 - for _, entry := range runEntries { + for _, entry := range sorted { result := executionResults[entry.flowName] - status := "✅ Success" - if !strings.EqualFold(result.Status, "success") { - status = "❌ Failed" - } else { + status := "❌ Failed" + switch { + case strings.EqualFold(result.Status, "success"): + status = "✅ Success" successCount++ + case strings.EqualFold(result.Status, "skipped"): + status = "⏭️ Skipped" } - fmt.Printf(" %-20s %s (Duration: %s)\n", result.FlowName, status, reporter.FormatDuration(result.Duration)) + fmt.Printf(" %-20s %s (Duration: %s)\n", entry.flowName, status, reporter.FormatDuration(result.Duration)) } - fmt.Printf("\nFlows completed: %d/%d\n", successCount, len(runEntries)) + fmt.Printf("\nFlows completed: %d/%d\n", successCount, len(sorted)) } - for _, result := range executionResults { - if !strings.EqualFold(result.Status, "success") { - if result.Error != "" { - return fmt.Errorf("multi-flow execution failed: %s", result.Error) - } - return fmt.Errorf("multi-flow execution failed: one or more flows failed") + var problems []string + for _, entry := range sorted { + result := executionResults[entry.flowName] + if strings.EqualFold(result.Status, "success") { + continue } + detail := result.Error + if detail == "" { + detail = "no result recorded" + } + problems = append(problems, fmt.Sprintf("%s: %s (%s)", entry.flowName, detail, result.Status)) + } + if len(problems) > 0 { + return fmt.Errorf("multi-flow execution failed: %s", strings.Join(problems, "; ")) } return nil diff --git a/apps/cli/internal/runner/runner_test.go b/apps/cli/internal/runner/runner_test.go index 790ef216f..400fe5182 100644 --- a/apps/cli/internal/runner/runner_test.go +++ b/apps/cli/internal/runner/runner_test.go @@ -10,9 +10,11 @@ import ( "net/http/httptest" "os" "strings" + "sync" "testing" "time" + "github.com/coder/websocket" "github.com/the-dev-tools/dev-tools/apps/cli/internal/common" "github.com/the-dev-tools/dev-tools/apps/cli/internal/runner" "github.com/the-dev-tools/dev-tools/packages/db/pkg/sqlc/gen" @@ -30,7 +32,6 @@ import ( "github.com/the-dev-tools/dev-tools/packages/server/pkg/service/sworkspace" yamlflowsimplev2 "github.com/the-dev-tools/dev-tools/packages/server/pkg/translate/yamlflowsimplev2" "github.com/the-dev-tools/dev-tools/packages/spec/dist/buf/go/api/private/node_js_executor/v1/node_js_executorv1connect" - "github.com/coder/websocket" ) // flowTestFixture provides a common test environment for flow execution tests @@ -885,3 +886,259 @@ flows: t.Error("WS send node 'SendHello' was not executed") } } + +// setupMultiFlowFixture converts, imports, and returns the flows for a +// run:-block test. It is a thin helper shared by the RunMultipleFlows +// ordering/validation tests below. +func setupMultiFlowFixture(t *testing.T, fixture *flowTestFixture, yamlContent string) []mflow.Flow { + t.Helper() + + fileData := []byte(yamlContent) + resolved, err := yamlflowsimplev2.ConvertSimplifiedYAML(fileData, yamlflowsimplev2.ConvertOptionsV2{ + WorkspaceID: fixture.workspaceID, + }) + if err != nil { + t.Fatalf("failed to convert YAML: %v", err) + } + + fixture.importWorkspaceBundle(resolved) + + flows, err := fixture.services.Flow.GetFlowsByWorkspaceID(fixture.ctx, fixture.workspaceID) + if err != nil { + t.Fatalf("failed to get flows: %v", err) + } + return flows +} + +// TestRunMultipleFlows_ExecutesInDependencyOrder verifies that RunMultipleFlows +// executes flows in dependency order rather than run: block declaration +// order. The run: block deliberately lists B (which depends on A) before A. +func TestRunMultipleFlows_ExecutesInDependencyOrder(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + fixture := newFlowTestFixture(t) + + var mu sync.Mutex + var requestOrder []string + fixture.mockServer.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + requestOrder = append(requestOrder, r.URL.Path) + mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]interface{}{"status": "ok"}) + }) + + yamlContent := fmt.Sprintf(`workspace_name: Order Test +run: + - flow: B + depends_on: A + - flow: A +flows: + - name: A + steps: + - manual_start: + name: Start + - request: + name: RequestA + method: GET + url: %s/a + depends_on: Start + - name: B + steps: + - manual_start: + name: Start + - request: + name: RequestB + method: GET + url: %s/b + depends_on: Start +`, fixture.mockServer.URL, fixture.mockServer.URL) + + flows := setupMultiFlowFixture(t, fixture, yamlContent) + + ctx, cancel := context.WithTimeout(fixture.ctx, 15*time.Second) + defer cancel() + + if err := runner.RunMultipleFlows(ctx, []byte(yamlContent), flows, fixture.getRunnerServices(nil), fixture.services.Logger, nil); err != nil { + t.Fatalf("multi-flow execution failed: %v", err) + } + + mu.Lock() + defer mu.Unlock() + if len(requestOrder) != 2 { + t.Fatalf("expected 2 requests, got %d: %v", len(requestOrder), requestOrder) + } + if requestOrder[0] != "/a" || requestOrder[1] != "/b" { + t.Errorf("expected requests in dependency order [/a, /b] (A before B), got %v", requestOrder) + } +} + +// TestRunMultipleFlows_UnknownDependencyError verifies that a depends_on +// entry naming a flow that is not part of the run: block produces a specific, +// actionable error rather than being silently ignored. +func TestRunMultipleFlows_UnknownDependencyError(t *testing.T) { + fixture := newFlowTestFixture(t) + + yamlContent := `workspace_name: Unknown Dep Test +run: + - flow: A + - flow: B + depends_on: Missing +flows: + - name: A + steps: + - manual_start: + name: Start + - name: B + steps: + - manual_start: + name: Start +` + flows := setupMultiFlowFixture(t, fixture, yamlContent) + + err := runner.RunMultipleFlows(fixture.ctx, []byte(yamlContent), flows, fixture.getRunnerServices(nil), fixture.services.Logger, nil) + if err == nil { + t.Fatal("expected an error for an unknown dependency, got nil") + } + + const want = `unknown dependency "Missing" in run block (known flows: A, B)` + if err.Error() != want { + t.Errorf("unexpected error message:\n got: %q\n want: %q", err.Error(), want) + } +} + +// TestRunMultipleFlows_CyclicDependencyError verifies that a dependency cycle +// in the run: block is rejected with a message naming the cycle, instead of +// hanging or silently running flows in an arbitrary order. +func TestRunMultipleFlows_CyclicDependencyError(t *testing.T) { + fixture := newFlowTestFixture(t) + + yamlContent := `workspace_name: Cycle Test +run: + - flow: A + depends_on: B + - flow: B + depends_on: A +flows: + - name: A + steps: + - manual_start: + name: Start + - name: B + steps: + - manual_start: + name: Start +` + flows := setupMultiFlowFixture(t, fixture, yamlContent) + + err := runner.RunMultipleFlows(fixture.ctx, []byte(yamlContent), flows, fixture.getRunnerServices(nil), fixture.services.Logger, nil) + if err == nil { + t.Fatal("expected a dependency cycle error, got nil") + } + if !strings.Contains(err.Error(), "dependency cycle in run block") { + t.Errorf("expected a dependency cycle error, got: %v", err) + } + if !strings.Contains(err.Error(), "A") || !strings.Contains(err.Error(), "B") { + t.Errorf("expected the cycle description to name both A and B, got: %v", err) + } +} + +// TestRunMultipleFlows_EmptyRunBlockErrors verifies the pre-existing behavior +// (unchanged by dependency-ordering support) that an empty run: block is +// rejected rather than silently doing nothing. +func TestRunMultipleFlows_EmptyRunBlockErrors(t *testing.T) { + fixture := newFlowTestFixture(t) + + yamlContent := `workspace_name: Empty Run Test +run: [] +flows: + - name: A + steps: + - manual_start: + name: Start +` + flows := setupMultiFlowFixture(t, fixture, yamlContent) + + err := runner.RunMultipleFlows(fixture.ctx, []byte(yamlContent), flows, fixture.getRunnerServices(nil), fixture.services.Logger, nil) + if err == nil { + t.Fatal("expected an error for an empty run block, got nil") + } +} + +// TestRunMultipleFlows_FailedDependencySkipsDependent verifies that when a +// flow fails, flows that depend on it are reported as skipped (with a reason +// naming the failed dependency) instead of being attempted, and the overall +// call still returns a non-nil error. This preserves today's failure-gate +// semantics (a dependent never runs after its dependency fails) while +// replacing the old silent early-return with an explicit, reported skip. +func TestRunMultipleFlows_FailedDependencySkipsDependent(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + fixture := newFlowTestFixture(t) + + requestCount := 0 + fixture.mockServer.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]interface{}{"status": "ok"}) + }) + + // A's request targets an address nothing listens on (port 1 is reserved), + // so the connection is refused and the flow fails fast. B depends on A + // and must never reach the mock server. + yamlContent := fmt.Sprintf(`workspace_name: Failure Gate Test +run: + - flow: A + - flow: B + depends_on: A +flows: + - name: A + steps: + - manual_start: + name: Start + - request: + name: RequestA + method: GET + url: http://127.0.0.1:1/unreachable + depends_on: Start + - name: B + steps: + - manual_start: + name: Start + - request: + name: RequestB + method: GET + url: %s/b + depends_on: Start +`, fixture.mockServer.URL) + + flows := setupMultiFlowFixture(t, fixture, yamlContent) + + ctx, cancel := context.WithTimeout(fixture.ctx, 15*time.Second) + defer cancel() + + err := runner.RunMultipleFlows(ctx, []byte(yamlContent), flows, fixture.getRunnerServices(nil), fixture.services.Logger, nil) + if err == nil { + t.Fatal("expected a non-nil error when a dependency fails, got nil") + } + if !strings.Contains(err.Error(), "B") { + t.Errorf("expected the error to mention flow B, got: %v", err) + } + if !strings.Contains(err.Error(), "skipped") { + t.Errorf("expected the error to report B as skipped, got: %v", err) + } + if !strings.Contains(err.Error(), `"A"`) { + t.Errorf("expected the error to name A as the failed dependency, got: %v", err) + } + + if requestCount != 0 { + t.Errorf("expected RequestB to never be attempted since A failed, but the mock server received %d request(s)", requestCount) + } +} From c6769b2196ee741790875e3a914af8c7ffe4ef64 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 19:14:18 +0300 Subject: [PATCH 16/38] test: strengthen loadmetrics status-class and RPS coverage Adds TestAggregatorKeysByStatusClass, tying ClassifyStatus's output directly into Record/Key bucketing (the shape task 5's ingest will actually use and asserting 5xx outcomes also count as errors. Also covers the zero-code/ nil-error edge case in TestClassifyStatus, and simplifies TestMergeRPSMath's histogram setup to a single RecordValues call. EOF ) --- .../pkg/loadmetrics/loadmetrics_test.go | 42 +++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/packages/server/pkg/loadmetrics/loadmetrics_test.go b/packages/server/pkg/loadmetrics/loadmetrics_test.go index 9d39a0023..2cd499be5 100644 --- a/packages/server/pkg/loadmetrics/loadmetrics_test.go +++ b/packages/server/pkg/loadmetrics/loadmetrics_test.go @@ -26,6 +26,7 @@ func TestClassifyStatus(t *testing.T) { {"404 not found classifies as 4xx", 404, nil, StatusClass4xx}, {"500 server error classifies as 5xx", 500, nil, StatusClass5xx}, {"599 upper boundary classifies as 5xx", 599, nil, StatusClass5xx}, + {"zero code with no error classifies as error", 0, nil, StatusClassError}, {"transport error with no status classifies as error", 0, errors.New("connection reset by peer"), StatusClassError}, {"context deadline exceeded classifies as timeout", 0, context.DeadlineExceeded, StatusClassTimeout}, {"wrapped context deadline exceeded classifies as timeout", 0, fmt.Errorf("dial: %w", context.DeadlineExceeded), StatusClassTimeout}, @@ -41,6 +42,43 @@ func TestClassifyStatus(t *testing.T) { } } +// TestAggregatorKeysByStatusClass checks that ClassifyStatus's output, used +// as the Key.StatusClass for Record, buckets outcomes for the same step into +// separate Frame entries rather than clobbering a single counter. +func TestAggregatorKeysByStatusClass(t *testing.T) { + agg := NewAggregator(time.Second) + const step = "checkout" + + outcomes := []struct { + code int + err error + }{ + {200, nil}, + {200, nil}, + {404, nil}, + {500, nil}, + {500, nil}, + {500, nil}, + {0, context.DeadlineExceeded}, + } + + for _, o := range outcomes { + class := ClassifyStatus(o.code, o.err) + agg.Record(Key{Step: step, StatusClass: class}, time.Millisecond, 0, o.err != nil || o.code >= 400) + } + + frame := agg.Flush(time.Now()) + + assert.Equal(t, int64(2), frame.Entries[Key{Step: step, StatusClass: StatusClass2xx}].Count) + assert.Equal(t, int64(1), frame.Entries[Key{Step: step, StatusClass: StatusClass4xx}].Count) + assert.Equal(t, int64(3), frame.Entries[Key{Step: step, StatusClass: StatusClass5xx}].Count) + assert.Equal(t, int64(1), frame.Entries[Key{Step: step, StatusClass: StatusClassTimeout}].Count) + assert.Len(t, frame.Entries, 4, "one bucket per distinct status class, not one bucket per step") + + fiveXx := frame.Entries[Key{Step: step, StatusClass: StatusClass5xx}] + assert.Equal(t, int64(3), fiveXx.ErrorCount, "5xx outcomes should also count as errors") +} + // TestAggregatorPercentileCorrectness records a uniform 1..1000ms distribution // and checks the HDR-derived P50/P99 land within 1% of the true values, // which is well outside HDR's 3-significant-figure quantization error at @@ -128,9 +166,7 @@ func TestMergeRPSMath(t *testing.T) { k := Key{Step: "load-step", StatusClass: StatusClass2xx} hist := newHistogram() - for i := 0; i < 100; i++ { - require.NoError(t, hist.RecordValue(10_000)) // 10ms, arbitrary - } + require.NoError(t, hist.RecordValues(10_000, 100)) // 100 samples at 10ms, arbitrary frame := Frame{ IntervalStart: time.Unix(1_700_000_000, 0), From 54811f06943148ce9de651b74b615e0146d47db9 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 19:42:53 +0300 Subject: [PATCH 17/38] test: pin run block behavior deltas as deliberate parts of change 3 Code review flagged three observable behavior changes in RunMultipleFlows beyond the brief's three permitted changes. All three are confined to run: parsing/execution and remove a silent-failure mode, so they are sub-parts of change 3 (dependency-ordered run: execution) rather than a separate, undisclosed change. Pinning each with a dedicated test instead of leaving them as incidental side effects of the rewrite: 1. TestRunMultipleFlows_UnknownFlowPreventsAnyExecution - flow not found is now a pre-flight check across the whole run block before anything executes, not a mid-loop discovery. A run block listing a known flow followed by an unknown one used to run the known flow and only then fail on the unknown one; now nothing runs, verified by the mock server never receiving the known flow request. 2. TestRunMultipleFlows_SingleFailureMessageShape - the aggregate error format changed. It used to be just the failing flow raw error text, picked via unordered map iteration once more than one flow failed. It is now the failing flow name and status wrapped around that same error text, built from the deterministic run order. Pins the prefix and suffix this package controls; deliberately does not pin the OS-level dial error text in the middle, which is not something this package produces or should assert byte for byte. 3. TestRunMultipleFlows_MalformedRunEntrySurfacesError - a run entry that is not a flow mapping used to be silently dropped by the old hand-rolled parser, via a failed type assertion that just continued past it. Parsing through the typed yamlflowsimplev2 struct means the same document now fails to unmarshal instead of quietly running fewer flows than declared. No production code changes; these tests exercise behavior already shipped in the dependency-ordering commit. --- apps/cli/internal/runner/runner_test.go | 157 ++++++++++++++++++++++++ 1 file changed, 157 insertions(+) diff --git a/apps/cli/internal/runner/runner_test.go b/apps/cli/internal/runner/runner_test.go index 400fe5182..9ac82a0f3 100644 --- a/apps/cli/internal/runner/runner_test.go +++ b/apps/cli/internal/runner/runner_test.go @@ -1142,3 +1142,160 @@ flows: t.Errorf("expected RequestB to never be attempted since A failed, but the mock server received %d request(s)", requestCount) } } + +// The three tests below pin behavior deltas identified in code review as +// unavoidable, in-scope side effects of moving run: parsing/execution onto +// the typed yamlflowsimplev2 structs and a pre-flight topological sort (see +// the "run: execution behavior deltas" section of task-1-report.md). None of +// these are new production code changes - they lock in what +// TestRunMultipleFlows_UnknownDependencyError, _CyclicDependencyError, etc. +// already exercise indirectly, made explicit and named. + +// TestRunMultipleFlows_UnknownFlowPreventsAnyExecution pins delta #1: the +// "flow not found" check now runs as a pre-flight pass over every run: +// entry before anything executes, rather than being discovered mid-loop. +// Previously `run: [A, Bogus]` would run A and only then fail on Bogus; +// now nothing runs. RequestA's hit count on the mock server is the proof. +func TestRunMultipleFlows_UnknownFlowPreventsAnyExecution(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + fixture := newFlowTestFixture(t) + + requestCount := 0 + fixture.mockServer.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]interface{}{"status": "ok"}) + }) + + // Import only flow A - a valid, self-contained document. "Bogus" is + // intentionally never defined under flows: anywhere. + importYAML := fmt.Sprintf(`workspace_name: Unknown Flow Test +flows: + - name: A + steps: + - manual_start: + name: Start + - request: + name: RequestA + method: GET + url: %s/a + depends_on: Start +`, fixture.mockServer.URL) + + flows := setupMultiFlowFixture(t, fixture, importYAML) + + // Fed directly to RunMultipleFlows (bypassing ConvertSimplifiedYAML, + // which would itself reject a run: entry naming an undefined flow) to + // isolate exactly the check under test: RunMultipleFlows's own + // cross-check of the run: block against the flows it was actually + // given. + runYAML := []byte(`workspace_name: Unknown Flow Test +run: + - flow: A + - flow: Bogus +`) + + err := runner.RunMultipleFlows(fixture.ctx, runYAML, flows, fixture.getRunnerServices(nil), fixture.services.Logger, nil) + if err == nil { + t.Fatal("expected an error for an unknown flow in the run block, got nil") + } + + if requestCount != 0 { + t.Errorf("expected A to never execute since Bogus is unknown, but the mock server received %d request(s)", requestCount) + } +} + +// TestRunMultipleFlows_SingleFailureMessageShape pins delta #2: the +// aggregate error format. It used to be exactly `multi-flow execution +// failed: ` (and picked *which* err via unordered map iteration when +// more than one flow failed). It is now `multi-flow execution failed: +// : ()`, built from the deterministic run order. The +// prefix/suffix produced by RunMultipleFlows are asserted exactly; the +// infix (the OS-level dial error text from the refused connection) is only +// asserted to be present, since its exact wording is not something this +// package controls or should pin. +func TestRunMultipleFlows_SingleFailureMessageShape(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + fixture := newFlowTestFixture(t) + + // Port 1 is reserved; nothing listens there, so the connection is + // refused immediately and deterministically. + yamlContent := `workspace_name: Single Failure Message Shape Test +run: + - flow: A +flows: + - name: A + steps: + - manual_start: + name: Start + - request: + name: RequestA + method: GET + url: http://127.0.0.1:1/unreachable + depends_on: Start +` + flows := setupMultiFlowFixture(t, fixture, yamlContent) + + ctx, cancel := context.WithTimeout(fixture.ctx, 15*time.Second) + defer cancel() + + err := runner.RunMultipleFlows(ctx, []byte(yamlContent), flows, fixture.getRunnerServices(nil), fixture.services.Logger, nil) + if err == nil { + t.Fatal("expected a non-nil error when the only flow fails, got nil") + } + + const wantPrefix = `multi-flow execution failed: A: ` + const wantSuffix = ` (failed)` + got := err.Error() + if !strings.HasPrefix(got, wantPrefix) { + t.Errorf("expected error to start with %q, got: %q", wantPrefix, got) + } + if !strings.HasSuffix(got, wantSuffix) { + t.Errorf("expected error to end with %q, got: %q", wantSuffix, got) + } + if !strings.Contains(got, "127.0.0.1:1") { + t.Errorf("expected error to include the underlying RunFlow error, got: %q", got) + } +} + +// TestRunMultipleFlows_MalformedRunEntrySurfacesError pins delta #3: a run: +// entry that is not a {flow: ...} mapping (a bare scalar here) used to be +// silently dropped by the old hand-rolled map[string]interface{} parser (a +// failed type assertion just `continue`d past it, so the workflow quietly +// ran fewer flows than declared). Parsing run: through the typed +// yamlflowsimplev2.YamlRunEntryV2 struct means the same document now fails +// to unmarshal, surfacing the malformed entry as an error instead of +// silently ignoring it. +func TestRunMultipleFlows_MalformedRunEntrySurfacesError(t *testing.T) { + fixture := newFlowTestFixture(t) + + importYAML := `workspace_name: Malformed Run Entry Test +flows: + - name: A + steps: + - manual_start: + name: Start +` + flows := setupMultiFlowFixture(t, fixture, importYAML) + + runYAML := []byte(`workspace_name: Malformed Run Entry Test +run: + - flow: A + - just-a-string +`) + + err := runner.RunMultipleFlows(fixture.ctx, runYAML, flows, fixture.getRunnerServices(nil), fixture.services.Logger, nil) + if err == nil { + t.Fatal("expected a YAML unmarshal error for the malformed run: entry, got nil") + } + if !strings.Contains(err.Error(), "unmarshal") { + t.Errorf("expected an unmarshal error naming the parse failure, got: %v", err) + } +} From d8680538b40627c8267f43b7eb88653f7e6922fe Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 19:43:30 +0300 Subject: [PATCH 18/38] test: document run block depends_on lost on export Code review verified a pre-existing, undisclosed gap that interacts with this task's deliverable: MarshalSimplifiedYAML synthesizes the run: block purely from flows: declaration order (the "Generate default Run configuration" step in exporter.go) and never reads depends_on at all, so any dependency graph declared in run: is silently destroyed on export. An exported-then-rerun file always degrades to plain declaration order, regardless of what the original run: block said. Not fixed here - fixing the exporter is a fourth behavior change outside this task's three permitted changes, and belongs on the hygiene backlog instead. Documented so it is visible rather than silently relying on fixture luck: - New golden fixture run_deps_lost_on_export.yaml declares flows: in an order that is deliberately NOT a valid topological sort of its own depends_on chain (C, B, A for a C-depends-on-B-depends-on-A chain, the reverse of the correct A, B, C). The committed golden shows the synthesized run: block reproducing that same wrong flat order with no depends_on at all, so the destruction is visible in the diff rather than hidden behind an order that happens to look right. - Both that fixture and sub_flows.yaml (which documents the separate sub_flow_trigger export bug) now carry a short comment naming the gap and the exact lines responsible, so a future contributor does not have to re-derive it from scratch. No production code changes. --- .../golden/run_deps_lost_on_export.golden | 53 +++++++++++++++++++ .../golden/run_deps_lost_on_export.yaml | 44 +++++++++++++++ .../testdata/golden/sub_flows.yaml | 11 ++++ 3 files changed, 108 insertions(+) create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_deps_lost_on_export.golden create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_deps_lost_on_export.yaml diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_deps_lost_on_export.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_deps_lost_on_export.golden new file mode 100644 index 000000000..31c98c6f3 --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_deps_lost_on_export.golden @@ -0,0 +1,53 @@ +version: 2 +workspace_name: Golden Run Deps Lost On Export +run: + - flow: C + - flow: B + - flow: A +requests: + - name: RequestA + method: GET + url: https://api.example.com/a + - name: RequestB + method: GET + url: https://api.example.com/b + - name: RequestC + method: GET + url: https://api.example.com/c +flows: + - name: C + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - request: + name: RequestC + depends_on: Start + position_x: 300 + position_y: 0 + use_request: RequestC + - name: B + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - request: + name: RequestB + depends_on: Start + position_x: 300 + position_y: 0 + use_request: RequestB + - name: A + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - request: + name: RequestA + depends_on: Start + position_x: 300 + position_y: 0 + use_request: RequestA diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_deps_lost_on_export.yaml b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_deps_lost_on_export.yaml new file mode 100644 index 000000000..f70d79079 --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/run_deps_lost_on_export.yaml @@ -0,0 +1,44 @@ +# Pre-existing gap (not fixed by this task, see task-1-report.md Concerns): +# MarshalSimplifiedYAML synthesizes the run: block purely from flows: order +# ("Generate default Run configuration" in exporter.go) and never reads +# depends_on at all, so every dependency declared below is silently lost on +# export - an exported-then-rerun file degrades to plain declaration order. +# flows: is deliberately declared C, B, A (the reverse of the correct +# topological order A, B, C for the depends_on chain C->B->A) so the golden +# below visibly reproduces that same wrong order in its flat run: block, +# rather than passing to look correct by accidental alphabetical luck. +workspace_name: Golden Run Deps Lost On Export +run: + - flow: C + depends_on: B + - flow: B + depends_on: A + - flow: A +flows: + - name: C + steps: + - manual_start: + name: Start + - request: + name: RequestC + depends_on: Start + method: GET + url: https://api.example.com/c + - name: B + steps: + - manual_start: + name: Start + - request: + name: RequestB + depends_on: Start + method: GET + url: https://api.example.com/b + - name: A + steps: + - manual_start: + name: Start + - request: + name: RequestA + depends_on: Start + method: GET + url: https://api.example.com/a diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.yaml b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.yaml index e9a1684a4..4491711e2 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.yaml +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/sub_flows.yaml @@ -1,3 +1,14 @@ +# Pre-existing gap (not fixed by this task, see task-1-report.md Concerns): +# sub_flow_trigger is not exercised anywhere in this corpus because it does +# not survive export. processSteps overwrites the flow node's ID with the +# flow's start-node ID (converter_node.go:235) so the trigger acts as the +# flow's entry point, but NodeSubFlowTrigger.FlowNodeID keeps the original, +# pre-overwrite ID (converter_node.go:788). The exporter's lookup map is +# keyed by that original FlowNodeID (exporter.go:145) but probed with the +# node's actual (overwritten) ID (exporter.go:661), so the lookup always +# misses and the sub_flow_trigger step is silently dropped from the +# exported YAML - which then fails to re-import, since other steps still +# depend on it by name. GreeterFlow below uses a plain manual_start instead. workspace_name: Golden Sub Flows flows: - name: MainFlow From 57499b75a0a02be67226869e411ef82217874d2f Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 19:51:12 +0300 Subject: [PATCH 19/38] test: cover Flush's real elapsed-time Interval computation Finding 1 from review: TestMergeRPSMath only exercised the RPS formula against a hand-built Frame, never the real Flush() path, so a regression reintroducing a blind echo of the nominal interval would go undetected. Adds TestAggregatorFlushIntervalReflectsElapsedTime: drives real Flush() twice with different sleep windows (20ms, 80ms) against a 5-minute nominal interval, asserts Interval tracks actual elapsed time (not the nominal constant) and differs in magnitude between flushes, and asserts IntervalStart advances. Verified the test has teeth by temporarily mutating Flush to echo the nominal interval instead of computing the real elapsed time - confirmed it fails for exactly the mutated fields, then reverted (net diff to loadmetrics.go is zero; the interval-unused design itself was independently reviewed and ruled correct, matching the design doc's flushed-at-interval- and-at-run-end semantics). --- .../pkg/loadmetrics/loadmetrics_test.go | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/packages/server/pkg/loadmetrics/loadmetrics_test.go b/packages/server/pkg/loadmetrics/loadmetrics_test.go index 2cd499be5..9eddd0e2c 100644 --- a/packages/server/pkg/loadmetrics/loadmetrics_test.go +++ b/packages/server/pkg/loadmetrics/loadmetrics_test.go @@ -211,6 +211,40 @@ func TestAggregatorFlushDrainsAndResets(t *testing.T) { assert.Empty(t, second.Entries) } +// TestAggregatorFlushIntervalReflectsElapsedTime drives the real Flush path +// (not a hand-built Frame, unlike TestMergeRPSMath) and checks Frame.Interval +// tracks actual elapsed wall-clock time between flushes, not the nominal +// interval passed to NewAggregator. NewAggregator is given a 5-minute nominal +// interval specifically so a blind echo of it would be trivially detectable +// (5m is nowhere near either sleep window below). +func TestAggregatorFlushIntervalReflectsElapsedTime(t *testing.T) { + agg := NewAggregator(5 * time.Minute) + k := Key{Step: "step", StatusClass: StatusClass2xx} + + agg.Record(k, time.Millisecond, 0, false) + time.Sleep(20 * time.Millisecond) + first := agg.Flush(time.Now()) + + assert.GreaterOrEqual(t, first.Interval, 20*time.Millisecond, "Interval should be at least the real elapsed sleep") + assert.Less(t, first.Interval, 5*time.Second, "Interval should not echo the nominal 5m interval") + + agg.Record(k, time.Millisecond, 0, false) + time.Sleep(80 * time.Millisecond) + second := agg.Flush(time.Now()) + + assert.GreaterOrEqual(t, second.Interval, 80*time.Millisecond) + assert.Less(t, second.Interval, 5*time.Second) + + // Different sleep windows must produce different Interval magnitudes - + // proves Interval is derived per-flush, not a single constant (nominal or + // otherwise) repeated every time. + assert.Greater(t, second.Interval, first.Interval, "the second flush slept longer, so its Interval should be larger") + + // Each Flush starts the next interval at `now`, so IntervalStart must + // advance between successive flushes. + assert.True(t, second.IntervalStart.After(first.IntervalStart), "IntervalStart should advance between successive flushes") +} + // TestAggregatorRecordConcurrentRace exercises Record from many goroutines // concurrently; run with -race to prove there's no data race, and assert the // final counts to also catch lost-update bugs. From c6acd2a330d0dbd43a29eac6b773f44d4f5312e1 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 19:51:43 +0300 Subject: [PATCH 20/38] feat: add threshold verdicts + environment fingerprint to LoadRunReport Finding 2 from review: LoadRunReport only carried total/perStep, omitting two of the three things the design doc assigns to it (RunReport = merged frames + threshold verdicts + environment fingerprint for baseline comparability - docs/superpowers/specs/2026-08-08-load-testing-design.md section 3.3, explicitly "the hard-to-retrofit piece designed in Phase 0"). Controller ruling was to add the fields now (optional), not defer them, since a deferred doc comment would recreate the retrofit risk Phase 0 exists to remove. Adds: - LoadThresholdVerdict: expression (string, the threshold as configured), success (boolean), observedValue (optional string, stringified since thresholds may target durations, rates, or counts). The success field name mirrors HttpResponseAssert/GraphQLResponseAssert.success - the closest existing precedent for "verdict of evaluating an expression" in this spec. - LoadEnvironmentFingerprint: workerVersion/region/machineClass, all optional strings. Modeled as its own small nested model (not inline fields on LoadRunReport) mirroring flow.tsp's Position model - a small named value group with no decorators, no primary key, referenced directly as another model's field type. - LoadRunReport gains thresholds?: LoadThresholdVerdict[] and environment?: LoadEnvironmentFingerprint, both optional (absent for exploratory runs with no thresholds, or for local/desktop runs with no cloud environment). Doc comments on both note the shape is frozen in Phase 0 but nothing populates them until thresholds/the Stresseur worker fleet ship in Phase 2. Regenerated via spec:build; verified byte-identical output across two consecutive cache-bypassed runs (idempotent), and packages/server and packages/spec still `go build ./...` cleanly. --- packages/spec/api/load-metrics.tsp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/spec/api/load-metrics.tsp b/packages/spec/api/load-metrics.tsp index 8f65ee904..b29ca9f71 100644 --- a/packages/spec/api/load-metrics.tsp +++ b/packages/spec/api/load-metrics.tsp @@ -56,10 +56,32 @@ model LoadRunStepStats { ...LoadStats; } +@doc("Verdict of evaluating one threshold (e.g. \"p95(CreateOrder) < 300ms\") against a completed run. Shape is frozen in Phase 0; nothing populates this until threshold evaluation ships in Phase 2.") +model LoadThresholdVerdict { + @doc("The threshold expression as configured") expression: string; + success: boolean; + + @doc("The observed value that was checked against the expression, stringified since thresholds may target durations, rates, or counts") + observedValue?: string; +} + +@doc("Identifies the environment a run executed in, for baseline comparability across runs. Shape is frozen in Phase 0; nothing populates this until the Stresseur cloud worker fleet ships in Phase 2 (a local/desktop run has no region or machine class).") +model LoadEnvironmentFingerprint { + workerVersion?: string; + region?: string; + machineClass?: string; +} + @doc("Fully-merged report for a completed (or in-progress) load-test run.") model LoadRunReport { total: LoadStats; perStep: LoadRunStepStats[]; + + @doc("Threshold pass/fail verdicts for this run. Absent for exploratory runs with no configured thresholds; shape frozen in Phase 0, populated starting Phase 2.") + thresholds?: LoadThresholdVerdict[]; + + @doc("Fingerprint of the environment this run executed in, for baseline comparability. Shape frozen in Phase 0, populated starting Phase 2.") + environment?: LoadEnvironmentFingerprint; } model LoadFailureHeader { From 78c43333936f7bdcf81fb56694d69a2aa6fac12d Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 20:06:15 +0300 Subject: [PATCH 21/38] docs: stresseur design spec + phase 0/1 implementation plan Ignore .superpowers SDD scratch in prettier. --- .prettierignore | 1 + .../plans/2026-08-08-stresseur-phase0-1.md | 287 +++++++++++++ .../specs/2026-08-08-load-testing-design.md | 400 ++++++++++++++++++ 3 files changed, 688 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-08-stresseur-phase0-1.md create mode 100644 docs/superpowers/specs/2026-08-08-load-testing-design.md diff --git a/.prettierignore b/.prettierignore index 2248e30ba..75ed3cb82 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,5 +1,6 @@ .ai/ .golangci.yml +.superpowers/ *.har AGENTS.md CHANGELOG.md diff --git a/docs/superpowers/plans/2026-08-08-stresseur-phase0-1.md b/docs/superpowers/plans/2026-08-08-stresseur-phase0-1.md new file mode 100644 index 000000000..87d562bc1 --- /dev/null +++ b/docs/superpowers/plans/2026-08-08-stresseur-phase0-1.md @@ -0,0 +1,287 @@ +# Stresseur Phase 0 + Phase 1 (independent items) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Cut every seam load testing needs (YAML contract, engine levers, metrics envelope, CI Action) with zero default-behavior change, per spec `docs/superpowers/specs/2026-08-08-load-testing-design.md` §4–§5. + +**Architecture:** Wave 1 = four file-disjoint lanes safe to run in parallel (Task 1 YAML contract, Task 2 engine levers, Task 3 metrics envelope, Task 4 GitHub Action). Wave 2 = dependent integration (Task 5 CLI load mode, Task 6 benchmark) — **do not dispatch Wave 2 until Wave 1 is merged and green**. Desktop charts are a separate future plan. + +**Tech Stack:** Go 1.25 (workspace), sqlc/SQLite, TypeSpec→buf codegen, cobra CLI, GitHub composite Action, `github.com/HdrHistogram/hdrhistogram-go`. + +## Global Constraints + +- **Zero default-behavior change.** No new flags/options ⇒ byte-identical output. Golden tests (Task 1 Step 1) are the enforcement mechanism. Existing tests may not be modified except where a task explicitly says so. +- **Commands need the env:** prefix everything with `direnv exec .`; for nx also `env NX_SOCKET_DIR=/tmp/nx-tmp` (e.g. `direnv exec . env NX_SOCKET_DIR=/tmp/nx-tmp pnpm nx run server:test`). +- **Go tests:** `cd packages/server && direnv exec ../.. go test ./path/ -run TestName -v -timeout 30s` for single tests; full: `direnv exec . env NX_SOCKET_DIR=/tmp/nx-tmp pnpm nx run server:test` / `cli:test`. +- **Lint before declaring done:** `direnv exec . env NX_SOCKET_DIR=/tmp/nx-tmp pnpm nx run server:lint`. Fix at root cause; **no `//nolint`, no suppressions** — including pre-existing issues your diff touches. +- **No raw SQL** (norawsql linter), **no reads inside transactions** (notxread linter). +- **Never hand-edit generated code** (`packages/spec/dist/`, `packages/db/pkg/sqlc/gen/`); regenerate via `spec:build` / `db:generate`. +- **Commits:** conventional style (`feat:`, `fix:`, `test:`), small and frequent. **Never add a `Co-Authored-By` line.** Never push. Never touch `main`. +- **Model separation:** proto ↔ model ↔ sqlc types stay distinct (`m` prefix models bridge). +- Error messages name the offending value and the valid alternatives. +- YAML export must stay deterministic (stable ordering) — the AI-fixer roadmap depends on diffable exports. + +--- + +### Task 1: YAML contract — golden corpus, `version:` field, assertion-import fix, `run:` ordering + +**Files:** + +- Create: `packages/server/pkg/translate/yamlflowsimplev2/golden_test.go` +- Create: `packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/` (corpus, ~8 files) +- Modify: `packages/server/pkg/translate/yamlflowsimplev2/types.go` (add `Version` field; top-level struct at :16-27) +- Modify: `packages/server/pkg/translate/yamlflowsimplev2/converter.go` (version validation) +- Modify: `packages/server/pkg/translate/yamlflowsimplev2/exporter.go` (emit `version: 2`; implementation entry at :23) +- Modify: `packages/server/pkg/translate/yamlflowsimplev2/converter_node.go` (assertion fix in `processRequestStep` :282-373) +- Modify: `packages/server/pkg/translate/yamlflowsimplev2/converter_flow.go` (`HTTPAssociatedData` :126-134, flow result assembly ~:263) +- Modify: `apps/cli/internal/runner/runner.go` (`RunMultipleFlows` :42-145 — topological order + explicit unknown-dep error) +- Test: `apps/cli/internal/runner/runner_test.go` (create if absent) + +**Interfaces:** + +- Consumes: nothing from other tasks. +- Produces: `YamlFlowFormatV2.Version int` (yaml key `version`, 0 = absent = treated as 2). `mhttp.HTTPAssert` records now populated in `ioworkspace.WorkspaceBundle.HTTPAsserts` on import. `RunMultipleFlows` executes flows in dependency order and returns error `unknown dependency %q in run block (known flows: %s)` for bad deps. Task 5 relies on all three. + +**Context you must read first:** `yamlflowsimplev2/README.md`, the GraphQL assertion conversion at `converter_node.go:642-653` (the working pattern to mirror), `converter_template.go:12-40` (merge semantics — assertions **append**), and testdata at `packages/server/internal/api/rimportv2/testdata/ecommerce.yaml`. + +- [ ] **Step 1: Characterization goldens (commit BEFORE any fix).** Build a corpus under `testdata/golden/`: one YAML per family — (a) request steps with map-form and list-form headers + assertions + templates via `use_request`, (b) `if`/`for`/`for_each` with `depends_on` handles (`Node.then`, `Node.loop`), (c) `js` + `wait`, (d) graphql with assertions, (e) ws_connection/ws_send, (f) sub-flows (trigger/return/run_sub_flow), (g) `run:` block multi-flow, (h) environments + credentials (with `{{ #env:X }}` placeholders). Reuse/adapt existing fixtures (`ecommerce.yaml`, `apps/cli/test/yamlflow/*.yaml` — note some use broken `${var}` syntax; write goldens with the real `{{ }}` syntax only). Test does: `Import(yaml) → Export(bundle) → yamlA; Import(yamlA) → Export → yamlB; assert yamlA == yamlB` (stability) plus snapshot `yamlA` as `.golden` file with an `-update` flag: + +```go +var update = flag.Bool("update", false, "rewrite .golden files") + +func TestGoldenRoundTrip(t *testing.T) { + for _, name := range goldenCases() { // filenames in testdata/golden + t.Run(name, func(t *testing.T) { + in := readFile(t, "testdata/golden/"+name+".yaml") + first := exportAfterImport(t, in) // ConvertSimplifiedYAML → MarshalSimplifiedYAML + second := exportAfterImport(t, first) + if !bytes.Equal(first, second) { t.Fatalf("unstable round-trip") } + golden := "testdata/golden/" + name + ".golden" + if *update { writeFile(t, golden, first) } + want := readFile(t, golden) + if !bytes.Equal(first, want) { t.Fatalf("golden mismatch (run with -update after intentional changes):\n%s", diff(want, first)) } + }) + } +} +``` + +- [ ] **Step 2: Run goldens, commit the characterization** (`test: add YAML round-trip golden corpus`). This snapshot INCLUDES today's bugs (assertions vanish) — that is the point. +- [ ] **Step 3 (version field): failing test** — `TestVersionField`: import doc with `version: 2` succeeds; absent succeeds; `version: 3` errors containing `unsupported yamlflow version 3 (this build supports up to 2)`; export output's first mapping key is `version: 2`. +- [ ] **Step 4: implement.** `Version int \`yaml:"version,omitempty"\``on`YamlFlowFormatV2`; validation in `Validate()`(types.go:560); exporter writes`version: 2`first (exporter builds an ordered structure — keep key order deterministic). Run test → PASS. Update goldens with`-update`, eyeball the diff (only `version: 2` line added), commit (`feat: version the yamlflow schema`). +- [ ] **Step 5 (assertion fix): failing test** — `TestHTTPAssertionsImported`: fixture request step with `assertions: [{expression: "response.status == 200"}, ...]` (check exact YAML assert shape in `types.go` `YamlAssertionV2` / exporter `:858` before writing) → after `ConvertSimplifiedYAML`, `result.HTTPAsserts` has the assertions bound to the request's node/example IDs. Also extend one golden case: assertions must survive `import → export` (they already export; the import side is what's broken). +- [ ] **Step 6: implement by mirroring the GraphQL path** (`converter_node.go:642-653`): add an asserts field to `HTTPAssociatedData` (converter_flow.go:126-134), populate it in `processRequestStep` from `finalReq.Assertions` (already merged by `mergeHTTPRequestDataStruct` — converter_template.go:36-38), convert to `[]mhttp.HTTPAssert` and append into the flow result where other HTTP associated data lands (~converter_flow.go:263). Run test → PASS. Update goldens (`-update`), verify diffs show assertions surviving, commit (`fix: import HTTP assertions from yamlflow (were silently dropped)`). +- [ ] **Step 7 (run: ordering): failing tests** in `apps/cli/internal/runner/runner_test.go` — construct `RunMultipleFlows` input where (a) `run:` lists `[B (depends_on A), A]` → execution order is A then B; (b) dep name `Missing` → error exactly matching `unknown dependency "Missing" in run block (known flows: A, B)`; (c) A fails → B reported skipped with reason, exit error non-nil (preserve today's failure-gate semantics, minus the silence). +- [ ] **Step 8: implement.** Parse the `run:` entries with the typed `yamlflowsimplev2` structs (replace the ad-hoc `map[string]interface{}` re-parse at runner.go:44-47), Kahn topological sort (deterministic tie-break: original list order), error on unknown/cyclic deps (`dependency cycle in run block: A → B → A`). Flows still execute sequentially (concurrency is Wave 2 territory). Run tests → PASS, commit (`fix: run block executes in dependency order and rejects unknown deps`). +- [ ] **Step 9: full gates.** `server:test`, `cli:test`, `server:lint` all green. Commit anything outstanding. + +--- + +### Task 2: Engine levers — `CreateFlowRunner` options, scenario scheduler, lean mode + +**Files:** + +- Modify: `packages/server/pkg/flow/runner/flowlocalrunner/flowlocalrunner.go` (`CreateFlowRunner` :56-69, goroutine derivation :232-241) +- Create: `packages/server/pkg/flow/runner/scenariorunner/scenariorunner.go` +- Create: `packages/server/pkg/flow/runner/scenariorunner/scenariorunner_test.go` +- Modify (lean mode, smallest viable seam): `packages/server/pkg/flow/node/nrequest/nrequest.go` — investigate first; see Step 5. +- Test: extend `flowlocalrunner` tests in-package. + +**Interfaces:** + +- Consumes: nothing from other tasks. +- Produces (Task 5 wires these): + +```go +// flowlocalrunner +type Option func(*FlowLocalRunner) // exact receiver type per file +func WithMaxConcurrency(n int) Option // n<=0 ⇒ ignore, keep default +func CreateFlowRunner(/* existing params */, opts ...Option) *FlowLocalRunner + +// scenariorunner (engine-agnostic VU scheduler; knows nothing about flows) +type RunProfile struct { + VUs int // concurrent workers, >=1 + Duration time.Duration // 0 ⇒ unbounded (use MaxIterations) + MaxIterations int64 // 0 ⇒ unbounded (use Duration); both 0 = config error +} +type Summary struct { + Iterations int64 + Errors int64 + Elapsed time.Duration +} +// iter runs ONE flow iteration; scenariorunner guarantees ≤VUs concurrent calls, +// stops issuing new iterations at Duration/MaxIterations/ctx-cancel, drains in-flight. +func Run(ctx context.Context, prof RunProfile, iter func(ctx context.Context, vu int, seq int64) error) (Summary, error) +``` + +- [ ] **Step 1 (options): failing test** — `TestCreateFlowRunnerDefaultUnchanged`: construct runner without options, assert concurrency field equals current CPU-derived value; `TestWithMaxConcurrency`: option sets it; `WithMaxConcurrency(0)` and negative are no-ops. +- [ ] **Step 2: implement** the variadic-options refactor (source-compatible: existing call sites — server `flowexec/session.go:61` and CLI `runner.go:282` — compile untouched; verify with `go build ./...` across the workspace). Run tests → PASS, commit (`feat: configurable max concurrency for flow runner`). +- [ ] **Step 3 (scheduler): failing tests** — table-driven in `scenariorunner_test.go`: + - concurrency ceiling: iter sleeps 20ms, VUs=5, MaxIterations=50 → high-water concurrent (atomic counter) ≤ 5, Summary.Iterations == 50. + - duration stop: Duration=150ms, iter takes 20ms → stops issuing after deadline, all in-flight drained, Elapsed ≥ 150ms, no iter call _starts_ after deadline. + - error counting: every 3rd iter returns error → Summary.Errors == count, run continues (errors never abort the scenario). + - cancel: ctx canceled mid-run → Run returns ctx.Err(), drains, no goroutine leak (`goleak` if already a repo dep, else assert with `runtime.NumGoroutine` delta tolerance — check go.mod first). + - config error: VUs=0 or both Duration/MaxIterations zero → error, no work done. + - run with `-race`. +- [ ] **Step 4: implement** `Run` — worker-per-VU goroutines pulling a shared atomic sequence counter, `errgroup` + context deadline, no unbounded channels. Run tests (`-race`) → PASS, commit (`feat: scenariorunner VU scheduler for load profiles`). +- [ ] **Step 5 (lean mode): investigate then implement the smallest seam.** Read `nrequest.go` fully: find where response bodies are retained in node output (`:81`, `:190` are the timing anchors; body write is nearby). Requirement: an opt-in flag (builder-level or request-node option — choose what needs the fewest call-site changes; document the choice in your report) that, after assertions/extraction complete for an iteration, drops response body retention so memory stays flat across thousands of iterations. Default MUST be current behavior; all existing tests pass unmodified. Failing test first: with lean on, node output for the request has body replaced by a truncation marker (`"[body dropped: lean mode]"`) while `response.status`/`response.duration` remain; with lean off, body intact. Commit (`feat: lean execution mode drops response bodies after assertion`). +- [ ] **Step 6: full gates** — `server:test`, `server:lint`, workspace `go build ./...`. Commit outstanding. + +--- + +### Task 3: Metrics envelope — TypeSpec models + Go HDR aggregation package + +**Files:** + +- Create: `packages/spec/src/**` new TypeSpec file for load metrics (read the existing `.tsp` layout first and follow its module/namespace conventions — find how existing domains declare models and services; do NOT invent a new pattern) +- Generated: `packages/spec/dist/**` via `spec:build` (never hand-edit) +- Create: `packages/server/pkg/loadmetrics/loadmetrics.go` +- Create: `packages/server/pkg/loadmetrics/loadmetrics_test.go` +- Modify: `packages/server/go.mod` (+ workspace `go.work.sum` as generated) — add `github.com/HdrHistogram/hdrhistogram-go` + +**Interfaces:** + +- Consumes: nothing from other tasks. +- Produces (Task 5 + Phase 2 ingest rely on these — treat as frozen once merged): + +```go +package loadmetrics + +type StatusClass string // "2xx","3xx","4xx","5xx","error","timeout" + +type Key struct { + Step string + StatusClass StatusClass +} + +type Frame struct { + IntervalStart time.Time + Interval time.Duration + Entries map[Key]Entry +} +type Entry struct { + Count int64 + ErrorCount int64 + Bytes int64 + Hist *hdrhistogram.Histogram // 1µs..10min, 3 sig figs +} + +func NewAggregator(interval time.Duration) *Aggregator +func (a *Aggregator) Record(k Key, latency time.Duration, bytes int64, isErr bool) // goroutine-safe +func (a *Aggregator) Flush(now time.Time) Frame // drain current interval +func Merge(frames []Frame) Report + +type Report struct { + Total Stats + PerStep map[Key]Stats +} +type Stats struct { + Count, ErrorCount, Bytes int64 + P50, P90, P95, P99, Max time.Duration + RPS float64 // Count / covered wall time +} +``` + +TypeSpec models (names frozen; shapes may follow existing proto conventions): `LoadMetricFrame`, `LoadMetricEntry` (step, status_class, count, error_count, bytes, hdr_histogram bytes, plus convenience p50/p90/p95/p99/max in µs), `LoadRunReport`, `LoadFailureArtifact` (step, vu, iteration, request{method,url,headers,body_sample}, response{status,headers,body_sample}, resolved_variables, error, captured_at). + +- [ ] **Step 1: failing tests** for the Go package: + - percentile correctness: record uniform 1..1000ms (step 1ms), assert P50 within 1% of 500ms, P99 within 1% of 990ms (HDR 3-sig-fig tolerance). + - merge equivalence: two aggregators each fed half the data; `Merge(f1, f2)` percentiles equal one aggregator fed everything (within HDR tolerance). + - status-class keying: 200→"2xx", 404→"4xx", transport error→"error" (provide `ClassifyStatus(code int, err error) StatusClass` — include timeout detection via `errors.Is(err, context.DeadlineExceeded)`/`os.IsTimeout`). + - concurrency: 8 goroutines × 10k Records under `-race`. + - RPS: 100 records over a flushed 5s frame → 20.0. +- [ ] **Step 2: implement** with `hdrhistogram-go` (`go get github.com/HdrHistogram/hdrhistogram-go` in `packages/server`). Run tests (`-race`) → PASS, commit (`feat: loadmetrics HDR aggregation package`). +- [ ] **Step 3: TypeSpec.** Read `packages/spec` layout; add the four models following existing conventions; run `direnv exec . env NX_SOCKET_DIR=/tmp/nx-tmp pnpm nx run spec:build`; verify generated Go + TS appear in `dist/` and the workspace still builds (`go build ./...`). Commit source + generated output in one commit per repo convention (check `git log -- packages/spec/dist` to confirm generated files are committed; follow whatever the repo does). (`feat: load metrics envelope in TypeSpec`). +- [ ] **Step 4: full gates** — `server:test`, `server:lint`, `spec:build` idempotent (second run = no diff). Commit outstanding. + +--- + +### Task 4: GitHub Action `run-flows` (Phase 1, DevTools brand) + +**Files:** + +- Create: `actions/run-flows/action.yml` (composite) +- Create: `actions/run-flows/README.md` +- Create: `actions/run-flows/testdata/smoke.yamlflow.yaml` +- Create: `.github/workflows/action-test.yaml` +- Modify: `docs/cli.md` (replace the DIY CI snippet at :73-97 with the Action; keep the manual path as an alternative) + +**Interfaces:** + +- Consumes: released CLI binaries (naming per `.github/workflows/release-go.yaml:9-47` matrix) and `apps/cli/install.sh` (supports `INSTALL_DIR`; check whether it supports version pinning — if not, download release assets directly by tag). +- Produces: `uses: the-dev-tools/dev-tools/actions/run-flows@` with the contract below. Phase 2's Stresseur App links to the same report JSON schema. + +Action contract (implement exactly): + +```yaml +inputs: + file: { required: true, description: 'Path to .yamlflow.yaml' } + flow: { required: false, description: 'Single flow name; default = run: block' } + version: { required: false, default: 'latest', description: 'CLI release tag, e.g. cli@1.0.3' } + report-dir: { required: false, default: '.devtools-reports' } + fail-on-error: { required: false, default: 'true' } +outputs: + json-report: { description: 'Path to JSON report' } + junit-report: { description: 'Path to JUnit XML' } + success: { description: 'true/false' } +``` + +Steps inside the composite (all `shell: bash`): resolve + download the right binary for `runner.os/arch` into `$RUNNER_TEMP/devtools/bin`, chmod, run `devtoolscli flow run "$file" ${flow:+"$flow"} --report console --report "json:$report_dir/report.json" --report "junit:$report_dir/junit.xml"`, always generate a job-summary table into `$GITHUB_STEP_SUMMARY` from the JSON (flows, per-flow ✅/❌, durations — `jq` is preinstalled on GitHub runners), set outputs, exit per `fail-on-error`. + +- [ ] **Step 1:** Write `smoke.yamlflow.yaml` — 2 flows hitting `https://jsonplaceholder.typicode.com` (mirror the working syntax from `apps/cli/test/yamlflow/ws_run_example.yaml` — do NOT copy the stale `${var}` examples), one with a `run:` block dependency. +- [ ] **Step 2:** Implement `action.yml` per the contract. Every run/step must be OS-guarded for linux/macos (windows out of scope — document in README). +- [ ] **Step 3:** `action-test.yaml` workflow: `on: pull_request: paths: ['actions/**']` + `workflow_dispatch`; job matrix `ubuntu-latest` + `macos-latest`; steps: checkout, `uses: ./actions/run-flows` with `file: actions/run-flows/testdata/smoke.yamlflow.yaml`, assert outputs (`test -f` the reports, `grep` the summary file). Since the runner needs a released binary, `version:` pins the latest published release — verify the download URL resolves with `curl -fsI` in the action itself and fail with a clear message if the release asset naming drifts. +- [ ] **Step 4:** Validate locally what's validatable: `actionlint` if available (`command -v actionlint`), YAML parse (`python3 -c 'import yaml,sys; yaml.safe_load(open(sys.argv[1]))'`), and run the composite's bash blocks standalone against a locally built CLI (`cd apps/cli && direnv exec ../.. task build`) to prove the report/summary generation logic. Document in the report what could only be verified in real CI. +- [ ] **Step 5:** Update `docs/cli.md`, README for the action (inputs/outputs table, two copy-paste examples: PR check, nightly cron). Commit (`feat: run-flows GitHub Action`). + +--- + +### Task 5 (WAVE 2 — dispatch only after Tasks 1–3 merge): CLI load mode + +**Files:** + +- Modify: `apps/cli/cmd/flow.go` (flags :40-42, run path) +- Create: `apps/cli/internal/loadrun/loadrun.go` (+`loadrun_test.go`) — wires scenariorunner + loadmetrics + flow build/exec per iteration +- Modify: `packages/server/pkg/translate/yamlflowsimplev2/types.go` (+converter/exporter/golden updates) — additive `load:` block +- Modify: `apps/cli/internal/reporter/reporter.go` (aggregate table + additive JSON fields) + +**Interfaces:** + +- Consumes: Task 1 (`Version`, typed run parsing), Task 2 (`scenariorunner.Run`, `WithMaxConcurrency`, lean mode), Task 3 (`loadmetrics`). +- Produces: `devtoolscli flow run f.yaml --scenario checkout-baseline` and `--vus N --duration 60s [--iterations M]`; `load:` YAML block per spec §3.1 (executor `constant-vus` only in this task; `stages`/`ramping-vus`/thresholds are Phase 2); JSON report gains top-level `load_report` (loadmetrics.Report serialized); console prints: + +``` +Step p50 p95 p99 RPS Err% +CreateOrder 120ms 310ms 480ms 142.0 0.2 +TOTAL 95ms 290ms 470ms 285.1 0.1 +``` + +Key execution requirements: fresh node state per iteration (investigate whether `BuildNodes` must re-run per iteration or per-VU — measure both, pick correctness first, note cost in report); lean mode ON for load runs; no per-iteration DB writes; `--vus`+`--scenario` mutually exclusive (cobra `MarkFlagsMutuallyExclusive`); default invocation (no load flags) byte-identical (existing CLI tests unmodified; goldens prove the YAML side). + +**Wave 1 review-derived contract additions (binding):** + +1. **One `loadmetrics.Aggregator` per VU**, `Merge` at run end — the package's designed contention-avoidance path; do not share one aggregator across VUs. +2. **`Frame.Interval` is real elapsed time**, not the nominal constructor interval — RPS math must use it as such. +3. **Do not wrap the scenario context in `context.WithTimeout(ctx, prof.Duration)`** — `scenariorunner.Run` returns `ctx.Err()` when the caller's ctx dies, so duration-via-ctx would make every successful timed run return `DeadlineExceeded`. Pass Duration only via `RunProfile`. +4. **Drain or bypass `NodeRequestSideRespChan` in load mode** — lean mode drops the decoded body from VarMap but raw response bytes still flow down the persistence side-channel (`nrequest.go:238-248`); load mode must consume/discard that channel so memory stays flat and nothing persists per-iteration. +5. **Lean-mode coverage is request nodes only** (sub-flow propagation needs an `ExecuteSubFlow` signature change — out of scope): flows containing sub-flows/GraphQL/WS still run under load, but the memory-flatness guarantee is documented as request-node-scoped in `--help` and the report. +6. **StatusClass conversion**: write the Go↔generated-proto (`LoadStatusClass`) conversion helper next to where the JSON report is assembled; `loadmetrics.StatusClass` string values are the source of truth. +7. **New nx target `server:test:race`** running `go test -race` for `./pkg/flow/runner/scenariorunner/ ./pkg/loadmetrics/` (the two concurrency-critical packages), and run it in this task's gates. CI-workflow wiring is deferred (hygiene backlog). +8. **`load:` block YAML types are additive** (`YamlLoadScenario` in types.go), exported deterministically, `executor` value `constant-vus` only — any other value errors naming the valid alternatives and that `ramping-vus`/`constant-arrival-rate` arrive in Phase 2. One new golden fixture with a `load:` block. +9. **Exit codes**: a load run that completes reports exit 0 even with request errors (thresholds are Phase 2's gate mechanism); setup/infra failure (bad scenario name, bad flags, target unreachable at iteration 1 for ALL VUs) exits 1. +10. **Console aggregate table** exactly per the Produces block; JSON report gains additive top-level `load_report` (serialized `loadmetrics.Report` + run metadata: scenario name, VUs, duration, iterations, worker version). + +Steps follow the same TDD cycle as Wave 1 (failing test → implement → gates → commit); the implementer drafts the step list from these requirements and the Task 1–3 interfaces, and the reviewer holds it to this contract. + +### Task 6 (WAVE 2 — after Task 5): RPS/worker benchmark + +**Files:** Create `apps/cli/test/loadbench/` (self-contained: local `net/http/httptest` target server with fixed 5ms handler — no external dependencies), bench script + `docs/superpowers/specs/phase0-bench.md` results. + +**Contract:** measure sustained RPS at VUs ∈ {1, 10, 50} for (a) single-GET flow, (b) 5-step chained flow, on local hardware; record CPU count, Go version, lean-mode on; write the table to the doc. This number gates spec §3.5 capacity math and §6 pricing — flag in the doc that Fly-hardware numbers are still pending. + +--- + +## Integration notes (controller, not a dispatched task) + +- Merge order after Wave 1 review: Task 1 → 2 → 3 → 4 (only expected overlap: none; `go.sum` churn from Task 3 only). +- After merge: run `server:test`, `cli:test`, `db:test`, `server:lint`, `client:lint`, `root:lint:format`; then goldens once more. +- Wave 2 dispatch requires: all Wave 1 tasks merged + gates green. diff --git a/docs/superpowers/specs/2026-08-08-load-testing-design.md b/docs/superpowers/specs/2026-08-08-load-testing-design.md new file mode 100644 index 000000000..b2fe5d588 --- /dev/null +++ b/docs/superpowers/specs/2026-08-08-load-testing-design.md @@ -0,0 +1,400 @@ +# Stresseur: Load Testing & Agent-Native SaaS — Design + +**Date:** 2026-08-08 +**Status:** Draft for review +**Scope:** Phase 0 in implementation-ready detail; Phases 1–3 at planning detail; GTM strategy. + +## 1. Product thesis + +Development is now agent-driven. Teams ship endpoints in hours via Claude Code/Cursor, +and the API-testing tools of the last era assume a human with a GUI and an afternoon. +Meanwhile API traffic itself is turning agentic — burstier, chained, less forgiving. +APIs have never been produced faster or verified less. + +DevTools' unfair advantage: **the test artifact already exists and agents can operate it.** +Recorded traffic → flow → YAML in the repo. The same YAML is the functional test and the +load test. Coding agents read, write, diff, and fix it like any other code. + +- **Postman** cannot say this: cloud workspace, human GUI, collections agents don't maintain. +- **k6** cannot say this: the load script is a separate, hand-written artifact divorced + from functional tests. +- **DevTools**: one artifact, two modes (functional / load), three runtimes (desktop / CI / + cloud), operable by humans and agents. + +One-liner: _"Your API tests are already load tests — and your agents keep them green."_ + +**Brand architecture (decided 2026-08-08): two brands, one contract.** +**DevTools** stays the free, open-source platform: desktop app, CLI, GitHub Action, +the YAML format, and local load mode. **Stresseur** (stresseur.com) is the commercial +layer: hosted load generation, baselines/history, the PR bot, the AI maintenance +agents, and the MCP server — all running the DevTools engine against the same +repo-resident YAML. + +The free DevTools product is the funnel (honors the no-signup, local-first brand). +Stresseur sells the three things that structurally cannot be self-hosted freebies: + +1. **Hosted load generation** — fleets, geo, concurrency pools (metered VU-hours) +2. **History & baselines** — "p95 vs last release" as a PR gate (team plans) +3. **AI test maintenance** — bug bot + auto-fix of flow YAML on API changes (seats) + +## 2. Current state (verified 2026-08-08) + +| Capability | State | Evidence | +| ------------------------- | ------------------------------------------------------------------------------------------------ | --------------------------------------------------------------- | +| Portable YAML flow format | ✅ Self-contained by name; no DB IDs | `packages/server/pkg/translate/yamlflowsimplev2/types.go:16-27` | +| CLI runs YAML headlessly | ✅ Embeds full server engine; static `CGO_ENABLED=0` binary; in-memory SQLite per run | `apps/cli/cmd/flow.go:87-336`, `sqlitemem` | +| JS nodes | ✅ Node spawned only when flow has JS nodes; ConnectRPC over unix socket | `apps/cli/cmd/flow.go:266-291`, `jsrunner.go` | +| Per-request latency | ✅ Measured in Go (`response.duration` ms; node wall-clock ns) | `packages/server/pkg/flow/node/nrequest/nrequest.go:81,190` | +| Reporters | ✅ console / json / junit, exit codes 0/1 | `apps/cli/internal/reporter/reporter.go` | +| Engine parallelism | ⚠️ Dependency-free steps run concurrently; cap hardcoded to CPU count; no external lever | `flowlocalrunner.go:232-241`, `strategy_multi.go:63-114` | +| FOR/FOR_EACH loops | Sequential iterations (`for i := range nr.IterCount`) | `nfor.go:129` | +| Cancellation | ✅ `context.WithCancel` | `flowlocalrunner.go:142` | +| Event pub/sub | ✅ Generic `SyncStreamer[Topic, Payload]`, in-memory impl (single-node) | `packages/server/pkg/eventstream/eventstream.go` | +| Auth foothold | ✅ BetterAuth tables (user/session/account+OAuth) already in schema | `packages/db/pkg/sqlc/schema/08_betterauth.sql` | +| Plans/quotas/billing | ❌ None anywhere in schema | schema grep 2026-08-08 | +| YAML schema versioning | ❌ No `version:` field; parser is lenient (no `KnownFields`) → additive fields are backward-safe | translate pkg grep 2026-08-08 | +| Load/iteration levers | ❌ No iterations/VU/duration/rate anywhere (YAML, CLI flags, engine API) | `flow.go:40-42` | +| Aggregate stats | ❌ No percentiles/histograms/throughput anywhere | reporter + engine grep | +| GitHub Action | ❌ Docs show DIY snippet only; no published action | `docs/cli.md:73-97` | + +### Known defects that intersect this work + +| Defect | Impact | Where | +| --------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- | +| HTTP assertions silently dropped on YAML import | Export→import round-trip loses assertions; imported "tests" assert nothing | `converter_node.go:282-373` (never populates asserts); contrast GraphQL `converter_node.go:642-653` | +| `run:` block ignores dependency order | Strict list-order execution; unknown dep names silently skip the flow | `apps/cli/internal/runner/runner.go:109-144` | +| Stale `${var}` syntax in shipped CLI examples | Examples send literal `${user_id}` strings; correct syntax is `{{ Node.index }}` / `{{ Node.item }}` | `apps/cli/test/yamlflow/example_run_yamlflow.yaml:20` et al. | +| `flow.timeout` / `flow.metadata` parsed but never read | Dead schema fields | `types.go:67-73` | +| GraphQL-only / WS-only exports use ad-hoc shapes that cannot be re-imported | Round-trip broken for those exports | `rexportv2/export.go:257-342` | +| CLI JS nodes broken on Windows | Go dials `unix`; worker binds named pipe on win32 | `jsrunner.go:63` vs `worker-js/src/main.ts:32-36` | +| `devtools version` prints v0.1.0 (package is 1.0.3) | Cosmetic; breaks support triage | `apps/cli/cmd/version.go:13` | + +## 3. Target architecture + +### 3.1 The contract: repo-resident YAML + run profiles + +Flows stay exactly as they are. Load configuration is a **separate `load:` section +referencing flows by name** — a flow is never edited to be load-tested: + +```yaml +version: 2 # NEW — additive, required going forward on export +workspace_name: Shop +flows: + - name: Checkout Flow + steps: [...] # unchanged + +load: # NEW — Phase 1 (local), Phase 2 (cloud adds regions/etc.) + - name: checkout-baseline + flow: Checkout Flow + executor: ramping-vus # constant-vus | ramping-vus | constant-arrival-rate + stages: + - { target: 200, duration: 2m } + - { target: 200, duration: 5m } + thresholds: + - p95(CreateOrder) < 300ms # step-scoped percentile + - error_rate < 0.5% # scenario-scoped +``` + +Thresholds make load runs **gate-able**: pass/fail, exit code, PR comment. +A `load:` entry with no thresholds is exploratory; with thresholds it is CI-enforceable. + +Design rules for the YAML (they serve the AI fixer later): + +- **Deterministic export** — stable key order, minimal churn, so agent-authored patches diff cleanly. +- **Versioned** — `version: 2` emitted on export; parser accepts absent (=2) or 2; errors + clearly on >2. Old binaries ignore the unknown key (lenient parser, verified). + +### 3.2 The engine: our CLI is the load worker (not k6) + +The CLI already executes the full flow semantics in Go. A load worker is the same +binary with new levers. k6 is rejected as the core engine because it would require a +permanently-maintained lossy transpiler (chaining, extraction, assertions, `{{ }}` +resolution, JS nodes) and its results would subtly disagree with functional runs. +k6 remains a possible future adapter for extreme raw-RPS; not in any planned phase. + +New engine concepts (all opt-in; default behavior byte-identical to today): + +- **RunProfile** — `{ executor, vus, stages, duration, maxIterations }` consumed by a + scenario scheduler that runs N concurrent flow executions (VU = one flow-execution + loop). Sits _above_ `flowlocalrunner`; loop nodes are untouched. +- **Lean execution mode** — response bodies released after extraction/assertion; + no per-iteration persistence; aggregates only. Required to keep memory flat at volume. +- **Configurable concurrency** — `CreateFlowRunner` gains functional options + (`WithMaxConcurrency(n)`); absent options preserve today's CPU-derived default. + +### 3.3 Metrics envelope (the hard-to-retrofit piece — designed in Phase 0) + +Defined in TypeSpec (`packages/spec`) so Go/TS types are generated, one source of truth: + +- **MetricFrame** — per (scenario, step, status-class) HDR histogram of latency + + counters (requests, errors by taxonomy, bytes), flushed at a fixed interval (5s) + and at run end. Workers stream frames; controllers/CLI merge them (HDR histograms + merge losslessly). +- **RunReport** — merged frames + threshold verdicts + environment fingerprint + (worker version, region, machine class) for baseline comparability. +- **FailureArtifact** — _sampled_ full request/response captures with resolved + variables, keyed by (step, error class). Cap per run. This is the substrate the + bug bot and AI fixer reason over; it is not optional telemetry. + +The existing CLI JSON report becomes the N=1 degenerate case of RunReport +(new fields are additive; old consumers ignore them). + +### 3.4 Three runtimes, one contract + +| Runtime | Brand | What runs | Who pays | Phase | +| ---------------------------- | ------------- | ------------------------------------------------ | --------------------- | ----- | +| Desktop | DevTools | flows from workspace (today) + local load charts | free | 1 | +| CI (published GitHub Action) | DevTools | functional flows on every PR; smoke-load allowed | customer's CI minutes | 1 | +| Cloud fleet (Fly Machines) | **Stresseur** | load scenarios; geo; baselines | metered VU-hours | 2 | + +### 3.5 Stresseur control plane (Phase 2) + +Own it; do not outsource to Trigger.dev. It is core product logic, small, and the team +already writes exactly this kind of Go. (Trigger.dev Cloud pricing/concurrency also fits +poorly: we'd pay for orchestration concurrency we can express as one DB state machine.) + +- **Service**: new module in the monorepo reusing `idwrap`, `eventstream` patterns, sqlc + discipline. DB: LibSQL/Turso (stays in the SQLite family) — Postgres only if + multi-writer needs force it later. +- **State machine**: `pending → validating → provisioning → ramping → running → +draining → complete | failed | canceled`, persisted per run; transitions idempotent. +- **Worker lifecycle**: Fly Machines API; workers boot the DevTools CLI image with a + short-lived run token (Stresseur orchestrates; the DevTools engine executes — one + execution semantics everywhere); heartbeat + frame ingest over ConnectRPC; controller destroys machines at + `draining`; a reconciler sweep destroys orphans (crash-safety) and enforces max-age. +- **Ingest**: frames land in run-scoped tables; live progress fans out over the existing + eventstream abstraction backed by a durable bridge (LibSQL table poll or NATS — + decision deferred to Phase 2 detail plan). +- **Capacity math**: VUs are concurrent flow-loops per worker; workers are sized by + benchmark (Phase 0 produces the RPS/worker baseline). 500 VUs ≈ a handful of + machines, not 500. + +### 3.6 Multi-tenancy, quotas, abuse + +- Accounts: BetterAuth (already in schema) + orgs. +- Stresseur plans + per-org limits enforced at `validating`: + +| | Stresseur Free | Stresseur Team | Stresseur Scale | +| ------------------------ | -------------- | -------------- | --------------- | +| Concurrent load runs/org | 1 | 3 | 10 | +| Max VUs | 50 | 500 | 5,000 | +| Max duration | 5 min | 30 min | 4 h | +| Regions | 1 | 3 | all | +| History/baselines | 7 days | 90 days | custom | + +(Values illustrative; finalize with design partners.) + +- Excess runs **queue** rather than fail (per-org FIFO; fairness cap per org so one + tenant cannot drain the fleet). +- **Target verification is launch-blocking**: hosted load-gen without it is a DDoS + cannon. Require proof of ownership per target host before any cloud run + (DNS TXT record or response-header echo — the Loader.io/Grafana k6 model), plus + hard per-plan egress caps and a global blocklist (gov, known-shared infra). + +### 3.7 The PR-native surfaces (Phase 2/3 — the attention engine) + +**Namespace status (2026-08-08):** stresseur.com + stresseur.dev owned; +`github.com/stresseur` org owned; npm `stresseur` published (placeholder 0.0.2, +homepage stresseur.com). Remaining: register the GitHub **App** named `stresseur` to +hold the slug — the bot identity (`stresseur[bot]`) is the product surface, and App +slugs are first-come. Brand split per §1: DevTools = platform, Stresseur = service. + +**GitHub App ("Stresseur")**, the `cursor review`-style motion: + +- On PR open: run the repo's functional flows against the preview/staging URL + (env-mapping config in repo maps environments → URLs/secrets refs; file naming is + open decision §8.3). +- On comment **`@stresseur load checkout-baseline`**: fire a cloud load run against the + preview env, reply with a rich comment — pass/fail vs thresholds, p50/p95/p99 vs + baseline, slowest steps, error taxonomy, link to full report. +- Every comment in a public repo is distribution ("Powered by Stresseur" footer). + +**MCP server** (thin layer over the same runs API): `list_flows`, `run_flow`, +`run_load_scenario`, `get_report`, `compare_to_baseline`, `verify_target`. This makes +DevTools the tool coding agents reach for on their own — same API, zero extra backend. + +### 3.8 Stresseur AI: test maintenance (Phase 3 — the moat) + +- **Bug bot**: on functional failure in CI/cloud, reads FailureArtifacts + the PR diff, + classifies (API regression vs test rot vs flaky infra), comments a triage verdict. +- **YAML fixer**: for test-rot cases (endpoint renamed, field moved, auth header + changed), proposes a patch to the flow YAML as a suggested commit on the same PR. + Deterministic YAML export + rich artifacts make these patches small and reviewable. +- Trust ladder: suggest-only → auto-commit-behind-approval → auto-fix-with-audit-log. + Never silently change thresholds — the fixer maintains _tests_, not _standards_. + +### 3.9 Repo topology & the open-core line + +Stresseur is proprietary; the DevTools monorepo is public Apache-2.0. Mixed visibility +in one repo is impossible, so: **new private repo `stresseur/stresseur`** under the +existing org, created when Phase 2 starts (Phase 0/1 are entirely open-repo work). + +The line that keeps two repos from becoming a coordination tax: **the contract lives +in the open repo.** TypeSpec definitions (MetricFrame / RunReport / FailureArtifact, +worker ingest protocol) stay in `packages/spec`; the open CLI emits them, and +Stresseur consumes the generated Go/TS packages by version. Stresseur never imports +DevTools internals — only published packages and the YAML/spec contracts. (Apache-2.0 +makes reuse legally trivial; the separation is for IP hygiene, security of +billing/abuse code, independent deploy cadence, and a clean story under diligence.) + +Open (trust + adoption assets): engine, CLI including load mode, YAML format, GitHub +Action, worker protocol spec, worker image Dockerfile — "the thing that fires requests +at your API is auditable" is a selling point for a load-testing product, and it +preempts what-does-the-worker-phone-home FUD. Closed (the business): control plane, +fleet orchestration, quotas/billing, baselines service, GitHub App, AI agents. +Undecided: the thin MCP shim (lean open — it's distribution; value stays server-side). +Never move an existing open capability closed — that's the community rug-pull this +plan is explicitly structured to avoid. + +## 4. Phase 0 — foundations without breakage + +**Goal:** cut every seam load testing needs, ship zero behavior change by default, +and retire the correctness debt that would otherwise become load-bearing. + +### 4.0 Guardrail zero: characterization tests before surgery + +Before touching the converter: **golden-file round-trip tests** (export→import→export +byte-stable across the testdata corpus + a new corpus covering every step type). +Lock current behavior first; then each intentional change lands with an updated golden +and a changelog entry. This is the mechanism that makes "we won't break things" a +property instead of a hope. + +### 4.1 Work items + +1. **YAML `version:` field.** Emit `version: 2` on export; accept absent/2 on import; + clear error on >2. Old binaries ignore it (lenient parser — verified, no + `KnownFields` anywhere). +2. **Fix assertion import drop** (`converter_node.go`): populate `HTTPAsserts` the way + GraphQL already does; add round-trip assertion tests. ⚠️ Behavior change — see 4.2. +3. **Fix `run:` ordering**: topological sort from `depends_on`; **error** on unknown + dep names instead of silent skip. ⚠️ Behavior change — see 4.2. +4. **Engine levers**: `CreateFlowRunner` functional options (`WithMaxConcurrency`); + RunProfile scheduler skeleton (constant-vus only in Phase 0); lean execution mode + flag threaded through node request context. Defaults preserve current behavior + exactly; server call sites untouched. +5. **CLI local load mode**: `flow run --scenario ` (reads `load:` block) plus + shorthand `--vus/--duration/--iterations`; prints aggregate table (p50/95/99, RPS, + error rate); `--report json` gains additive RunReport fields. No flags → today's + behavior, byte-identical output. +6. **TypeSpec: MetricFrame / RunReport / FailureArtifact** + codegen. HDR histogram + dependency in Go (`hdrhistogram-go`); merge helper + tests. +7. **Benchmark harness**: measure RPS/worker for a simple-GET flow and a 5-step chained + flow on dev hardware + one Fly shared-cpu machine class. Output feeds capacity math + and pricing. (Engine was built for correctness, not throughput — this number gates + Phase 2 promises.) +8. **CI**: extend `cli:test:integration` with scenario-mode cases; keep full server + suite green (`-p 8`) as the no-breakage gate for the shared engine packages. + +### 4.2 Behavior-change register (the "won't break things" contract) + +Every intentional change, its blast radius, and its comms. Nothing else may change +observable behavior; golden tests enforce that. + +| # | Change | Who feels it | Mitigation / comms | +| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | YAML assertions now enforced on import | Files whose assertions were silently ignored may now fail runs — **their tests start testing** | Minor version bump; changelog headline; release note shows how to delete/adjust assertions; docs page "why did my flow start failing" | +| 2 | `run:` executes in dependency order with strict failure modes: unknown flow/dep names abort **pre-flight** (nothing executes; previously flows before the bad entry ran first), malformed `run:` entries error instead of being silently skipped, dependency-failure skips are reported explicitly, and the aggregate failure message now includes the failing flow's name and status | Files listing flows out of dep order; typo'd flow/dep names that silently skipped flows; scripts parsing the old failure-message shape | Same release; error messages name the offending value and list valid flow names; changelog enumerates all four deltas | +| 3 | `version: 2` appears in exports | None (old parsers ignore unknown keys — verified) | Changelog note | +| 4 | New CLI flags / report fields | None (additive; JSON consumers ignoring unknown fields unaffected) | Changelog note | + +Release as **CLI/desktop minor** via Nx version plan (never manual bumps). +Explicitly _not_ in Phase 0 (logged, tracked, separate hygiene PRs): Windows JS-node +socket fix, `devtools version` string, stale `${var}` examples cleanup, dead +`flow.timeout/metadata` fields, GraphQL/WS export shapes. + +### 4.3 Phase 0 acceptance + +- `devtools flow run f.yaml` output is byte-identical to pre-change for the golden corpus. +- `devtools flow run f.yaml --vus 50 --duration 60s` produces a RunReport with correct + percentiles (validated against a reference implementation on synthetic latencies). +- Round-trip retains assertions; `run:` respects dependencies; full server + CLI suites green. +- Benchmark note published in-repo: RPS/worker for the two reference flows. + +## 5. Phases 1–3 — the attention plan + +Each phase ends in a **launch**, each launch is standalone news, each widens the funnel +for the next. Sequencing: credibility (DevTools, free, OSS) → distribution (Stresseur +PR bot) → differentiation (Stresseur AI). + +### Phase 1 — "Your recorded traffic is now a test suite" (DevTools, free wedge) + +**Ships:** published GitHub Action (`devtools/run-flows@v1`: install binary, run flows, +JUnit + PR annotation + job summary); local load mode polished; desktop load-results +view (charts: latency percentiles over time, RPS, errors). +**Launches:** + +- _Show HN: record your API traffic in Chrome, replay it as CI tests_ — the + record→flow→PR-gate loop demoed in 90 seconds. OSS + local + no signup = HN-native. +- _Load test from your laptop with the tests you already have_ — dev-Twitter/Reddit + follow-up; k6 comparison content ("no script, same flow"). + **Metric:** Action installs; weekly PRs gated by a DevTools check. + +### Phase 2 — "Tag the bot, get a load test" (the Stresseur launch) + +**Ships:** control plane + Fly fleet, target verification, plans/quotas/queueing, +baselines & history, Stresseur GitHub App with `@stresseur load ` comment +trigger and rich result comments, billing (seats + metered VU-hours). +**Launches:** + +- _`@stresseur load` on any PR_ — the cursor-review-style motion; public-repo comments + are the growth loop. Free tier for OSS repos to seed visibility. +- _Perf budgets as PR gates_ — "don't merge if p95 regresses" with baseline diffs. +- Design-partner case studies (5–10 agent-native teams recruited during Phase 1). + **Metric:** orgs with ≥1 cloud run/week; VU-hours; % of runs triggered from PRs. + +### Phase 3 — "The test suite that maintains itself" (Stresseur AI, the moat) + +**Ships:** Stresseur MCP server; bug bot triage; YAML fixer with suggested commits; +scheduled runs; multi-region scenarios. +**Launches:** + +- _Your agent load-tests before it merges_ — MCP demo inside Claude Code/Cursor: + agent ships endpoint → runs scenario → reads report → adjusts → merges. +- _Self-healing API tests_ — the fixer patching a flow live on a real PR. This is the + viral AI moment; time it with a model-partner or launch-week slot if possible. + **Metric:** fixer-suggested commits merged; MCP tool-calls/week; net revenue retention. + +## 6. Pricing shape (v1, keep simple) + +- **DevTools (free forever)** — desktop, CLI, Action, local load: unlimited. No paywall + ever crosses into the platform. +- **Stresseur Free** — cloud taste-tier (see quota table in §3.6). +- **Stresseur Team ($/seat/mo)** — history/baselines, PR bot, org management + included + VU-hour credits; overage metered. +- **Stresseur Scale (custom)** — high concurrency, all regions, SSO, audit, private + regions. + +VU-hour COGS on Fly shared-cpu machines is cents/hour → healthy margin on metered; +benchmark (Phase 0 item 7) finalizes the unit economics. Seats price the AI +maintenance + collaboration value, not the compute. + +## 7. Risks & mitigations + +| Risk | Mitigation | +| ----------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| Engine RPS/worker disappoints | Phase 0 benchmark gates all Phase 2 promises; lean mode; worker fan-out is linear; k6 adapter remains an escape hatch | +| Assertion fix breaks users' green builds | It un-breaks them (tests silently asserted nothing); comms per register 4.2; version-gated release | +| Hosted load-gen abuse | Target verification launch-blocking; egress caps; blocklist; per-org fairness | +| eventstream is in-memory/single-node | Cloud uses durable bridge (decision in Phase 2 plan); local behavior unchanged | +| YAML becomes public contract while under-tested | Guardrail zero: golden round-trip corpus before any converter change | +| Free tier funds nothing | It's the funnel by design; costs are customer-side (CI) or laptop-side; cloud free tier is capped tightly | +| Fly single-provider dependency | Worker protocol is provider-agnostic (machine = container + token + ingest URL); second provider addable later | + +## 8. Open decisions (deferred, with owners-to-be) + +1. Durable event bridge for cloud (LibSQL poll vs NATS) — Phase 2 plan. +2. Control-plane DB (LibSQL/Turso vs Postgres) — Phase 2 plan; start LibSQL. +3. Env-mapping config for the GitHub App (how PR → target URL) — Phase 2. Includes the + file's name: `stresseur.yml` vs `devtools.yml` (the DevTools Action and Stresseur + App may share one file; brand split argues for `stresseur.yml` owning cloud concerns). +4. OSS-repo free-tier abuse guardrails (verification still required?) — Phase 2. +5. Fixer trust ladder defaults (suggest-only at launch) — Phase 3. +6. Whether `constant-arrival-rate` (open-model workload) ships in Phase 1 or 2 — after + Phase 0 benchmark shows scheduler overhead. + +## 9. Out of scope (this design) + +- Replacing k6/Gatling for extreme-RPS synthetic benchmarking (>50k RPS single-target). +- Browser-level load testing (we load-test APIs, not pages). +- Windows CLI JS-node fix and other §2 hygiene defects (tracked separately). +- Self-hosted cloud runners (possible later: "bring your own Fly org" — not v1). From b11cd19196b4e7281b389a0402a417e6eea205f2 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 20:19:08 +0300 Subject: [PATCH 22/38] feat(yamlflow): additive load: block with constant-vus scenarios Adds the `load:` section from the load-testing design spec: named scenarios that reference a flow by name, so flows are never edited to be load-tested. Only the constant-vus executor is implemented; any other value errors naming the offending executor, the supported set, and that ramping-vus and constant-arrival-rate arrive in Phase 2. Every validation error names the scenario it came from. Scenarios ride on WorkspaceBundle purely so the YAML round trip preserves them - they are not database-backed, and the field documents that. Existing goldens are byte-unchanged; the new load_scenarios fixture pins the block's export, including duration normalization to Go's canonical form. --- packages/server/pkg/ioworkspace/types.go | 10 + packages/server/pkg/model/mload/mload.go | 46 +++ .../translate/yamlflowsimplev2/converter.go | 8 + .../translate/yamlflowsimplev2/exporter.go | 4 + .../pkg/translate/yamlflowsimplev2/load.go | 140 ++++++++ .../translate/yamlflowsimplev2/load_test.go | 316 ++++++++++++++++++ .../testdata/golden/load_scenarios.golden | 64 ++++ .../testdata/golden/load_scenarios.yaml | 49 +++ .../pkg/translate/yamlflowsimplev2/types.go | 22 ++ 9 files changed, 659 insertions(+) create mode 100644 packages/server/pkg/model/mload/mload.go create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/load.go create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/load_test.go create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/load_scenarios.golden create mode 100644 packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/load_scenarios.yaml diff --git a/packages/server/pkg/ioworkspace/types.go b/packages/server/pkg/ioworkspace/types.go index bcafdc798..c7d761ad6 100644 --- a/packages/server/pkg/ioworkspace/types.go +++ b/packages/server/pkg/ioworkspace/types.go @@ -8,6 +8,7 @@ import ( "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mflow" "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mgraphql" "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mhttp" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mload" "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mwebsocket" "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mworkspace" ) @@ -69,6 +70,15 @@ type WorkspaceBundle struct { // Credentials (metadata only, secrets are never exported) Credentials []mcredential.Credential + + // LoadScenarios carries the yamlflow `load:` block so it survives the + // YAML import -> export round trip. + // + // Unlike every other field here it is NOT database-backed: Import ignores + // it and Export never populates it, because there is no storage for load + // scenarios yet (Phase 2). Only the file-to-file path (the CLI and the + // yamlflow translator) reads and writes it. + LoadScenarios []mload.Scenario } // CountEntities returns a map containing the count of each entity type in the bundle. diff --git a/packages/server/pkg/model/mload/mload.go b/packages/server/pkg/model/mload/mload.go new file mode 100644 index 000000000..c85fa3f85 --- /dev/null +++ b/packages/server/pkg/model/mload/mload.go @@ -0,0 +1,46 @@ +// Package mload holds the domain model for load-test scenarios: the `load:` +// block of a yamlflow document, decoded into engine-ready values. +// +// It deliberately knows nothing about YAML or about the load runner. The +// yamlflow translator produces these, and the CLI's load runner consumes +// them, so neither has to depend on the other. +package mload + +import "time" + +// Executor names the scheduling strategy a scenario uses. +type Executor string + +const ( + // ExecutorConstantVUs holds a fixed number of virtual users for the + // scenario's duration or iteration budget. It is the only executor this + // build implements; ramping-vus and constant-arrival-rate are Phase 2. + ExecutorConstantVUs Executor = "constant-vus" +) + +// SupportedExecutors lists the executors this build accepts, for error +// messages that have to name the valid alternatives. +var SupportedExecutors = []Executor{ExecutorConstantVUs} + +// Scenario is one entry of the `load:` block: a named load profile applied to +// an existing flow. Flows are never edited to be load-tested, so a Scenario +// refers to its flow by name rather than owning it. +// +// Duration and MaxIterations are stop conditions; at least one is set. When +// both are set, whichever is reached first ends the scenario. +type Scenario struct { + // Name identifies the scenario, e.g. for `flow run --scenario `. + Name string + // FlowName is the flow this scenario drives, by its `flows:` entry name. + FlowName string + // Executor is the scheduling strategy; always ExecutorConstantVUs today. + Executor Executor + // VUs is the number of concurrent virtual users. Always >= 1. + VUs int + // Duration bounds the window during which new iterations start. Zero + // means unbounded, in which case MaxIterations is set. + Duration time.Duration + // MaxIterations bounds the total iterations issued. Zero means + // unbounded, in which case Duration is set. + MaxIterations int64 +} diff --git a/packages/server/pkg/translate/yamlflowsimplev2/converter.go b/packages/server/pkg/translate/yamlflowsimplev2/converter.go index 59464190c..d7cea5141 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/converter.go +++ b/packages/server/pkg/translate/yamlflowsimplev2/converter.go @@ -36,12 +36,20 @@ func ConvertSimplifiedYAML(data []byte, opts ConvertOptionsV2) (*ioworkspace.Wor return nil, fmt.Errorf("invalid YAML semantics: %w", err) } + // Decode the load: block before any flow work, so a malformed load + // profile fails fast rather than after the whole workspace is built. + loadScenarios, err := convertLoadScenarios(yamlFormat) + if err != nil { + return nil, fmt.Errorf("invalid load block: %w", err) + } + // Initialize resolved data structure with workspace metadata result := &ioworkspace.WorkspaceBundle{ Workspace: mworkspace.Workspace{ ID: opts.WorkspaceID, Name: yamlFormat.WorkspaceName, }, + LoadScenarios: loadScenarios, } // Prepare request templates map from both Sources diff --git a/packages/server/pkg/translate/yamlflowsimplev2/exporter.go b/packages/server/pkg/translate/yamlflowsimplev2/exporter.go index e6f7532f0..8c5caac77 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/exporter.go +++ b/packages/server/pkg/translate/yamlflowsimplev2/exporter.go @@ -792,6 +792,10 @@ func MarshalSimplifiedYAML(data *ioworkspace.WorkspaceBundle) ([]byte, error) { } } + // 7. Load scenarios, in declaration order (the bundle carries them + // verbatim; nothing here reorders or synthesizes them). + yamlFormat.Load = buildLoadScenarios(data.LoadScenarios) + return yaml.Marshal(yamlFormat) } diff --git a/packages/server/pkg/translate/yamlflowsimplev2/load.go b/packages/server/pkg/translate/yamlflowsimplev2/load.go new file mode 100644 index 000000000..3e375f1d3 --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/load.go @@ -0,0 +1,140 @@ +package yamlflowsimplev2 + +import ( + "fmt" + "strings" + "time" + + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mload" +) + +// convertLoadScenarios validates the `load:` block and decodes it into the +// engine-ready domain model, preserving declaration order. +// +// Every error names the offending scenario and, where there is a closed set of +// legal values, spells that set out - a load block is usually written by hand +// (or by an agent) and a nameless "invalid executor" is useless to both. +func convertLoadScenarios(yamlFormat *YamlFlowFormatV2) ([]mload.Scenario, error) { + if len(yamlFormat.Load) == 0 { + return nil, nil + } + + flowNames := make([]string, 0, len(yamlFormat.Flows)) + knownFlows := make(map[string]bool, len(yamlFormat.Flows)) + for _, flow := range yamlFormat.Flows { + flowNames = append(flowNames, flow.Name) + knownFlows[flow.Name] = true + } + + scenarios := make([]mload.Scenario, 0, len(yamlFormat.Load)) + seen := make(map[string]bool, len(yamlFormat.Load)) + + for i, entry := range yamlFormat.Load { + if entry.Name == "" { + return nil, NewYamlFlowErrorWithLineV2("load scenario name is required", "load.name", nil, i) + } + if seen[entry.Name] { + return nil, NewYamlFlowErrorV2( + fmt.Sprintf("duplicate load scenario name: %s", entry.Name), "load.name", entry.Name) + } + seen[entry.Name] = true + + scenario, err := convertLoadScenario(entry, knownFlows, flowNames) + if err != nil { + return nil, err + } + scenarios = append(scenarios, scenario) + } + + return scenarios, nil +} + +func convertLoadScenario(entry YamlLoadScenario, knownFlows map[string]bool, flowNames []string) (mload.Scenario, error) { + fail := func(format string, args ...any) (mload.Scenario, error) { + return mload.Scenario{}, NewYamlFlowErrorV2( + fmt.Sprintf("load scenario %q: ", entry.Name)+fmt.Sprintf(format, args...), + "load", entry.Name) + } + + if entry.Flow == "" { + return fail("flow is required (known flows: %s)", strings.Join(flowNames, ", ")) + } + if !knownFlows[entry.Flow] { + return fail("references unknown flow %q (known flows: %s)", entry.Flow, strings.Join(flowNames, ", ")) + } + + executor := mload.Executor(entry.Executor) + if entry.Executor == "" { + executor = mload.ExecutorConstantVUs + } + if executor != mload.ExecutorConstantVUs { + return fail( + "unsupported executor %q (this build supports: %s; ramping-vus and constant-arrival-rate arrive in Phase 2)", + entry.Executor, joinExecutors(mload.SupportedExecutors)) + } + + if entry.VUs < 1 { + return fail("vus must be >= 1, got %d", entry.VUs) + } + if entry.Iterations < 0 { + return fail("iterations must be >= 0, got %d", entry.Iterations) + } + + var duration time.Duration + if entry.Duration != "" { + parsed, err := time.ParseDuration(entry.Duration) + if err != nil { + return fail("duration %q is not a valid Go duration (e.g. 30s, 2m, 1h30m)", entry.Duration) + } + if parsed <= 0 { + return fail("duration %q must be positive", entry.Duration) + } + duration = parsed + } + + if duration == 0 && entry.Iterations == 0 { + return fail("needs a stop condition: set duration, iterations, or both") + } + + return mload.Scenario{ + Name: entry.Name, + FlowName: entry.Flow, + Executor: executor, + VUs: entry.VUs, + Duration: duration, + MaxIterations: entry.Iterations, + }, nil +} + +func joinExecutors(executors []mload.Executor) string { + names := make([]string, 0, len(executors)) + for _, e := range executors { + names = append(names, string(e)) + } + return strings.Join(names, ", ") +} + +// buildLoadScenarios renders the domain scenarios back to their YAML shape. +// Declaration order is preserved and durations are emitted in Go's canonical +// form, so exporting an already-exported document is a no-op. +func buildLoadScenarios(scenarios []mload.Scenario) []YamlLoadScenario { + if len(scenarios) == 0 { + return nil + } + + out := make([]YamlLoadScenario, 0, len(scenarios)) + for _, s := range scenarios { + entry := YamlLoadScenario{ + Name: s.Name, + Flow: s.FlowName, + Executor: string(s.Executor), + VUs: s.VUs, + Iterations: s.MaxIterations, + } + if s.Duration > 0 { + entry.Duration = s.Duration.String() + } + out = append(out, entry) + } + return out +} diff --git a/packages/server/pkg/translate/yamlflowsimplev2/load_test.go b/packages/server/pkg/translate/yamlflowsimplev2/load_test.go new file mode 100644 index 000000000..5bf8de6fe --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/load_test.go @@ -0,0 +1,316 @@ +package yamlflowsimplev2 + +import ( + "strings" + "testing" + "time" + + "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mload" +) + +const loadTestFlows = ` +workspace_name: Load Test +flows: + - name: Checkout Flow + steps: + - manual_start: + name: Start +` + +func convertLoadYAML(t *testing.T, yamlDoc string) ([]mload.Scenario, error) { + t.Helper() + + bundle, err := ConvertSimplifiedYAML([]byte(yamlDoc), GetDefaultOptions(idwrap.NewNow())) + if err != nil { + return nil, err + } + return bundle.LoadScenarios, nil +} + +// TestLoadBlockImport pins the shape of the additive `load:` block: every +// field lands on the bundle's scenarios, durations are parsed, and the +// executor defaults to constant-vus when omitted. +func TestLoadBlockImport(t *testing.T) { + yamlDoc := loadTestFlows + ` +load: + - name: checkout-baseline + flow: Checkout Flow + executor: constant-vus + vus: 10 + duration: 30s + iterations: 500 +` + + scenarios, err := convertLoadYAML(t, yamlDoc) + if err != nil { + t.Fatalf("ConvertSimplifiedYAML failed: %v", err) + } + if len(scenarios) != 1 { + t.Fatalf("expected 1 load scenario, got %d", len(scenarios)) + } + + got := scenarios[0] + want := mload.Scenario{ + Name: "checkout-baseline", + FlowName: "Checkout Flow", + Executor: mload.ExecutorConstantVUs, + VUs: 10, + Duration: 30 * time.Second, + MaxIterations: 500, + } + if got != want { + t.Fatalf("scenario mismatch:\n got: %+v\nwant: %+v", got, want) + } +} + +func TestLoadBlockDefaultsExecutorToConstantVUs(t *testing.T) { + yamlDoc := loadTestFlows + ` +load: + - name: no-executor + flow: Checkout Flow + vus: 2 + iterations: 4 +` + + scenarios, err := convertLoadYAML(t, yamlDoc) + if err != nil { + t.Fatalf("ConvertSimplifiedYAML failed: %v", err) + } + if len(scenarios) != 1 { + t.Fatalf("expected 1 load scenario, got %d", len(scenarios)) + } + if scenarios[0].Executor != mload.ExecutorConstantVUs { + t.Fatalf("executor = %q, want %q", scenarios[0].Executor, mload.ExecutorConstantVUs) + } +} + +// TestLoadBlockRejectsUnsupportedExecutor holds the error message to the +// contract: it must name the offending value, the executors this build +// accepts, and where the rest are coming from. +func TestLoadBlockRejectsUnsupportedExecutor(t *testing.T) { + for _, executor := range []string{"ramping-vus", "constant-arrival-rate", "nonsense"} { + t.Run(executor, func(t *testing.T) { + yamlDoc := loadTestFlows + ` +load: + - name: bad-executor + flow: Checkout Flow + executor: ` + executor + ` + vus: 1 + iterations: 1 +` + + _, err := convertLoadYAML(t, yamlDoc) + if err == nil { + t.Fatalf("expected executor %q to be rejected", executor) + } + for _, want := range []string{executor, "constant-vus", "Phase 2"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } + }) + } +} + +func TestLoadBlockRejectsUnknownFlow(t *testing.T) { + yamlDoc := loadTestFlows + ` +load: + - name: orphan + flow: Nonexistent Flow + vus: 1 + iterations: 1 +` + + _, err := convertLoadYAML(t, yamlDoc) + if err == nil { + t.Fatal("expected unknown flow reference to be rejected") + } + for _, want := range []string{"Nonexistent Flow", "Checkout Flow"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } +} + +func TestLoadBlockValidation(t *testing.T) { + cases := []struct { + name string + body string + wantSubs []string + }{ + { + name: "missing name", + body: ` +load: + - flow: Checkout Flow + vus: 1 + iterations: 1 +`, + wantSubs: []string{"name is required"}, + }, + { + name: "duplicate name", + body: ` +load: + - name: dupe + flow: Checkout Flow + vus: 1 + iterations: 1 + - name: dupe + flow: Checkout Flow + vus: 1 + iterations: 1 +`, + wantSubs: []string{"duplicate", "dupe"}, + }, + { + name: "missing flow", + body: ` +load: + - name: no-flow + vus: 1 + iterations: 1 +`, + wantSubs: []string{"flow is required", "no-flow"}, + }, + { + name: "non-positive vus", + body: ` +load: + - name: zero-vus + flow: Checkout Flow + vus: 0 + iterations: 1 +`, + wantSubs: []string{"vus", "zero-vus"}, + }, + { + name: "no stop condition", + body: ` +load: + - name: unbounded + flow: Checkout Flow + vus: 1 +`, + wantSubs: []string{"duration", "iterations", "unbounded"}, + }, + { + name: "unparseable duration", + body: ` +load: + - name: bad-duration + flow: Checkout Flow + vus: 1 + duration: 30 seconds +`, + wantSubs: []string{"30 seconds", "bad-duration"}, + }, + { + name: "negative iterations", + body: ` +load: + - name: negative-iters + flow: Checkout Flow + vus: 1 + iterations: -5 +`, + wantSubs: []string{"iterations", "negative-iters"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := convertLoadYAML(t, loadTestFlows+tc.body) + if err == nil { + t.Fatal("expected validation error, got nil") + } + for _, want := range tc.wantSubs { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } + }) + } +} + +// TestLoadBlockExportsDeterministically proves the load block survives the +// import -> export round trip in declaration order, with durations normalized +// to Go's canonical form so re-exporting is a no-op. +func TestLoadBlockExportsDeterministically(t *testing.T) { + yamlDoc := loadTestFlows + ` +load: + - name: zeta-scenario + flow: Checkout Flow + executor: constant-vus + vus: 4 + duration: 90s + - name: alpha-scenario + flow: Checkout Flow + vus: 2 + iterations: 10 +` + + bundle, err := ConvertSimplifiedYAML([]byte(yamlDoc), GetDefaultOptions(idwrap.NewNow())) + if err != nil { + t.Fatalf("ConvertSimplifiedYAML failed: %v", err) + } + + out, err := MarshalSimplifiedYAML(bundle) + if err != nil { + t.Fatalf("MarshalSimplifiedYAML failed: %v", err) + } + + got := string(out) + // Declaration order is preserved (not sorted), so zeta comes first. + zetaAt := strings.Index(got, "zeta-scenario") + alphaAt := strings.Index(got, "alpha-scenario") + if zetaAt < 0 || alphaAt < 0 { + t.Fatalf("exported YAML lost a scenario:\n%s", got) + } + if zetaAt > alphaAt { + t.Errorf("expected declaration order to be preserved, got:\n%s", got) + } + // 90s normalizes to Go's canonical 1m30s. + if !strings.Contains(got, "duration: 1m30s") { + t.Errorf("expected canonical duration in export, got:\n%s", got) + } + // A scenario with no duration must not emit an empty duration key. + if strings.Contains(got, `duration: ""`) { + t.Errorf("expected absent duration to be omitted, got:\n%s", got) + } + + // Re-exporting the exported document is a no-op. + reBundle, err := ConvertSimplifiedYAML(out, GetDefaultOptions(idwrap.NewNow())) + if err != nil { + t.Fatalf("re-import failed: %v", err) + } + reOut, err := MarshalSimplifiedYAML(reBundle) + if err != nil { + t.Fatalf("re-export failed: %v", err) + } + if string(reOut) != got { + t.Errorf("export is not stable:\nfirst:\n%s\nsecond:\n%s", got, reOut) + } +} + +// TestLoadBlockAbsentEmitsNothing guards the zero-default-behavior-change +// constraint on the YAML side: documents without a load block must export +// exactly as they did before the block existed. +func TestLoadBlockAbsentEmitsNothing(t *testing.T) { + bundle, err := ConvertSimplifiedYAML([]byte(loadTestFlows), GetDefaultOptions(idwrap.NewNow())) + if err != nil { + t.Fatalf("ConvertSimplifiedYAML failed: %v", err) + } + if len(bundle.LoadScenarios) != 0 { + t.Fatalf("expected no load scenarios, got %d", len(bundle.LoadScenarios)) + } + + out, err := MarshalSimplifiedYAML(bundle) + if err != nil { + t.Fatalf("MarshalSimplifiedYAML failed: %v", err) + } + if strings.Contains(string(out), "load:") { + t.Errorf("expected no load key in export, got:\n%s", out) + } +} diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/load_scenarios.golden b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/load_scenarios.golden new file mode 100644 index 000000000..5d82b2ec2 --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/load_scenarios.golden @@ -0,0 +1,64 @@ +version: 2 +workspace_name: Golden Load Scenarios +run: + - flow: Checkout + - flow: Browse +requests: + - name: ConfirmOrder + method: GET + url: https://api.example.com/orders/latest + - name: CreateOrder + method: POST + url: https://api.example.com/orders + body: '{"sku":"widget-1"}' + - name: ListProducts + method: GET + url: https://api.example.com/products +flows: + - name: Checkout + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - request: + name: CreateOrder + depends_on: Start + position_x: 300 + position_y: 0 + use_request: CreateOrder + - request: + name: ConfirmOrder + depends_on: CreateOrder + position_x: 600 + position_y: 0 + use_request: ConfirmOrder + - name: Browse + steps: + - manual_start: + name: Start + position_x: 0 + position_y: 0 + - request: + name: ListProducts + depends_on: Start + position_x: 300 + position_y: 0 + use_request: ListProducts +load: + - name: checkout-baseline + flow: Checkout + executor: constant-vus + vus: 10 + duration: 30s + - name: soak-checkout + flow: Checkout + executor: constant-vus + vus: 4 + duration: 1m30s + iterations: 2000 + - name: browse-smoke + flow: Browse + executor: constant-vus + vus: 1 + iterations: 25 diff --git a/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/load_scenarios.yaml b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/load_scenarios.yaml new file mode 100644 index 000000000..c0c3fa705 --- /dev/null +++ b/packages/server/pkg/translate/yamlflowsimplev2/testdata/golden/load_scenarios.yaml @@ -0,0 +1,49 @@ +# Exercises the additive `load:` block: scenarios reference flows by name and +# are never merged into them. Note the non-canonical `90s` duration in +# soak-checkout — export normalizes it to Go's `1m30s`, which is what makes +# re-export a no-op. +workspace_name: Golden Load Scenarios +run: + - flow: Checkout + - flow: Browse +flows: + - name: Checkout + steps: + - manual_start: + name: Start + - request: + name: CreateOrder + depends_on: Start + method: POST + url: https://api.example.com/orders + body: + sku: widget-1 + - request: + name: ConfirmOrder + depends_on: CreateOrder + method: GET + url: https://api.example.com/orders/latest + - name: Browse + steps: + - manual_start: + name: Start + - request: + name: ListProducts + depends_on: Start + method: GET + url: https://api.example.com/products +load: + - name: checkout-baseline + flow: Checkout + executor: constant-vus + vus: 10 + duration: 30s + - name: soak-checkout + flow: Checkout + vus: 4 + duration: 90s + iterations: 2000 + - name: browse-smoke + flow: Browse + vus: 1 + iterations: 25 diff --git a/packages/server/pkg/translate/yamlflowsimplev2/types.go b/packages/server/pkg/translate/yamlflowsimplev2/types.go index 876b76029..59d5fd838 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/types.go +++ b/packages/server/pkg/translate/yamlflowsimplev2/types.go @@ -32,6 +32,28 @@ type YamlFlowFormatV2 struct { GraphQLRequests []YamlGraphQLDefV2 `yaml:"graphql_requests,omitempty"` Flows []YamlFlowFlowV2 `yaml:"flows"` Environments []YamlEnvironmentV2 `yaml:"environments,omitempty"` + Load []YamlLoadScenario `yaml:"load,omitempty"` +} + +// YamlLoadScenario is one entry of the additive `load:` block: a named load +// profile applied to a flow declared in `flows:`. Flows are never edited to be +// load-tested, so a scenario references its flow by name. +// +// Only the constant-vus executor exists in this build; ramping-vus, +// constant-arrival-rate, stages and thresholds arrive in Phase 2. Unknown keys +// are ignored by the parser (as everywhere else in this format), so a document +// written for a later build still imports here. +type YamlLoadScenario struct { + Name string `yaml:"name"` + Flow string `yaml:"flow"` + // Executor defaults to constant-vus when omitted. + Executor string `yaml:"executor,omitempty"` + VUs int `yaml:"vus"` + // Duration is a Go duration string ("30s", "2m"). Exported in Go's + // canonical form so re-export is a no-op. + Duration string `yaml:"duration,omitempty"` + // Iterations caps the total iterations issued across all VUs. + Iterations int64 `yaml:"iterations,omitempty"` } // YamlCredentialV2 represents an LLM provider credential From c9235e785116816f07337501f5b37f5ffdcd931d Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 20:27:14 +0300 Subject: [PATCH 23/38] feat(cli): loadrun package driving flows as constant-VU load scenarios Wires the VU scheduler, the flow engine and the metrics envelope together: each virtual user gets its own HTTP client (so its own cookie jar and connection pool), its own instance of every flow node, its own persistence side-channels and its own metrics aggregator, which are merged at the end. Duration reaches the scheduler through RunProfile only - deriving it from a context deadline would make every successful timed run report DeadlineExceeded. Lean mode is always on, and the response side-channel is drained and discarded so nothing persists per iteration; the drain measures response sizes on the way past, which is where the report's byte counts come from. A completed run is a success even with failing requests. Only an unreachable target - every VU failing its very first iteration - is a setup failure. Node graphs are built per VU rather than per iteration: node implementations hold configuration only, so a rebuild buys no isolation while costing ~52% of a zero-latency iteration. --- apps/cli/internal/loadrun/loadrun.go | 562 +++++++++++++++++++++ apps/cli/internal/loadrun/loadrun_test.go | 576 ++++++++++++++++++++++ 2 files changed, 1138 insertions(+) create mode 100644 apps/cli/internal/loadrun/loadrun.go create mode 100644 apps/cli/internal/loadrun/loadrun_test.go diff --git a/apps/cli/internal/loadrun/loadrun.go b/apps/cli/internal/loadrun/loadrun.go new file mode 100644 index 000000000..554776e25 --- /dev/null +++ b/apps/cli/internal/loadrun/loadrun.go @@ -0,0 +1,562 @@ +// Package loadrun executes a flow as a load scenario: N virtual users each +// running the flow in a loop, with per-request latency and outcome aggregated +// into a merged report. +// +// It is the wiring layer between three pieces that know nothing about each +// other - the VU scheduler (scenariorunner), the flow engine +// (flowlocalrunner) and the metrics envelope (loadmetrics). It deliberately +// contains no YAML parsing (that lives in yamlflowsimplev2) and no +// presentation (that lives in the reporter). +// +// # What a load run costs +// +// A load run reads the database exactly once, at setup: the flow's nodes, +// edges and variables, and then one node graph per VU. The iteration loop +// itself holds no database or service handle at all - see vuWorker's fields - +// and the per-iteration response persistence side-channel is drained and +// discarded rather than written. (Sub-flow nodes are the exception: they +// resolve their target through the services they captured at build time, so a +// flow containing them does read the database per iteration.) +// +// Rebuilding the node graph every iteration was measured and rejected: node +// implementations hold configuration only, all per-execution mutable state +// lives in node.FlowNodeRequest (built fresh by each Run) and in the variable +// map (deep-copied per iteration, ~32ns), so a rebuild buys no isolation. It +// costs ~52% of a zero-latency iteration for a three-node flow, and more as +// flows grow, since its cost scales with node count. +// +// # Memory flatness is request-node-scoped +// +// Lean mode - which is always on for load runs - drops decoded response +// bodies from request nodes once assertions have run. It does not propagate +// into sub-flows (that needs an ExecuteSubFlow signature change), and GraphQL +// and WebSocket nodes do not implement it. Flows containing those still run +// under load; their memory does not stay flat, and their requests are not +// counted in the report. +package loadrun + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "sync" + "time" + + "github.com/the-dev-tools/dev-tools/apps/cli/internal/runner" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/node" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/node/ngraphql" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/node/nrequest" + flowrunner "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/runner" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/runner/flowlocalrunner" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/runner/scenariorunner" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/httpclient" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/loadmetrics" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mflow" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mload" +) + +// defaultNodeTimeout matches the CLI's functional run path, so a step that +// would time out in a normal run times out the same way under load. +const defaultNodeTimeout = 60 * time.Second + +// Config is a resolved load profile: what to run, how many virtual users, and +// when to stop. +type Config struct { + // ScenarioName is the `load:` block entry this profile came from, or "" + // when the profile was assembled from --vus/--duration/--iterations. + ScenarioName string + // Flow is the already-imported flow to drive. + Flow *mflow.Flow + // VUs is the number of concurrent virtual users. Must be >= 1. + VUs int + // Duration bounds the window during which new iterations start. + Duration time.Duration + // MaxIterations bounds the total iterations issued across all VUs. + MaxIterations int64 +} + +// ConfigFromScenario adapts a `load:` block scenario to a runnable Config. +// The flow must be the one the scenario names; resolving the name is the +// caller's job, since only it knows the imported workspace. +func ConfigFromScenario(scenario mload.Scenario, flow *mflow.Flow) Config { + return Config{ + ScenarioName: scenario.Name, + Flow: flow, + VUs: scenario.VUs, + Duration: scenario.Duration, + MaxIterations: scenario.MaxIterations, + } +} + +func (c Config) validate() error { + if c.Flow == nil { + return errors.New("load run: flow is required") + } + if c.VUs < 1 { + return fmt.Errorf("load run: vus must be >= 1, got %d", c.VUs) + } + if c.Duration <= 0 && c.MaxIterations <= 0 { + return errors.New("load run: needs a stop condition, set duration or iterations") + } + return nil +} + +// Result is everything a completed load run produced. +type Result struct { + // Config is the profile that was executed. + Config Config + // Summary is the scheduler's view: iterations completed, iterations that + // returned an error, wall time. + Summary scenariorunner.Summary + // Report is the merged metrics report keyed by (step, status class). + Report loadmetrics.Report + // ByStep is the same data folded across status classes, so each step has + // exactly one row. This is what the console table renders. + ByStep loadmetrics.Report +} + +// Run executes cfg and returns the merged report. +// +// A completed run is a success even when individual requests failed: request +// errors are data, reported in Summary.Errors and in the report's error +// counts. Run returns an error only when the run could not meaningfully +// happen - invalid configuration, a failure setting up the flow graph, or +// every virtual user failing its very first iteration (which means the target +// was never reachable, not that the system under test is slow). +func Run(ctx context.Context, cfg Config, services runner.RunnerServices, logger *slog.Logger) (Result, error) { + if err := cfg.validate(); err != nil { + return Result{}, err + } + + workers, release, err := newWorkers(ctx, cfg, services, logger) + if err != nil { + return Result{}, err + } + defer release() + + tracker := newFirstIterationTracker(cfg.VUs) + + // Duration is passed through RunProfile only. Deriving it from a context + // deadline instead would make scenariorunner.Run return ctx.Err() at the + // end of every successful timed run, since it reports the caller's + // context state on the way out. + summary, err := scenariorunner.Run(ctx, scenariorunner.RunProfile{ + VUs: cfg.VUs, + Duration: cfg.Duration, + MaxIterations: cfg.MaxIterations, + }, func(ctx context.Context, vu int, _ int64) error { + iterErr := workers[vu].iterate(ctx) + tracker.observe(vu, iterErr) + return iterErr + }) + if err != nil { + return Result{}, fmt.Errorf("load run: %w", err) + } + + if err := tracker.setupFailure(); err != nil { + return Result{}, err + } + + flushedAt := time.Now() + frames := make([]loadmetrics.Frame, 0, len(workers)) + for _, w := range workers { + frames = append(frames, w.agg.Flush(flushedAt)) + } + + return Result{ + Config: cfg, + Summary: summary, + Report: loadmetrics.Merge(frames), + ByStep: loadmetrics.Merge(foldByStep(frames)), + }, nil +} + +// foldByStep rewrites frames so every entry's status class is dropped, +// collapsing a step's buckets into one. Each entry becomes its own frame, +// because two entries of the same step would otherwise collide on the shared +// key inside a single frame's map; Merge unions the frames' (identical) time +// ranges, so the folded report's RPS matches the unfolded one. +// +// Histograms are shared with the input frames rather than copied. Merge only +// ever reads them, merging into freshly allocated histograms of its own. +func foldByStep(frames []loadmetrics.Frame) []loadmetrics.Frame { + folded := make([]loadmetrics.Frame, 0, len(frames)) + for _, f := range frames { + for key, entry := range f.Entries { + folded = append(folded, loadmetrics.Frame{ + IntervalStart: f.IntervalStart, + Interval: f.Interval, + Entries: map[loadmetrics.Key]loadmetrics.Entry{{Step: key.Step}: entry}, + }) + } + if len(f.Entries) == 0 { + // Keep the empty frame so the merged wall time - and therefore + // RPS - still covers this VU's window. + folded = append(folded, f) + } + } + return folded +} + +// firstIterationTracker records how each VU's first iteration went, which is +// what distinguishes "the target was never up" from "the target is failing +// some requests". +type firstIterationTracker struct { + mu sync.Mutex + outcome []*bool // nil until the VU has run its first iteration +} + +func newFirstIterationTracker(vus int) *firstIterationTracker { + return &firstIterationTracker{outcome: make([]*bool, vus)} +} + +func (t *firstIterationTracker) observe(vu int, err error) { + t.mu.Lock() + defer t.mu.Unlock() + + if vu < 0 || vu >= len(t.outcome) || t.outcome[vu] != nil { + return + } + ok := err == nil + t.outcome[vu] = &ok +} + +// setupFailure reports an error when every VU that got as far as running an +// iteration failed on that first attempt. A VU that never ran (because the +// iteration budget was exhausted by its siblings) is not evidence either way. +func (t *firstIterationTracker) setupFailure() error { + t.mu.Lock() + defer t.mu.Unlock() + + ran := 0 + for _, outcome := range t.outcome { + if outcome == nil { + continue + } + ran++ + if *outcome { + return nil + } + } + if ran == 0 { + return nil + } + return fmt.Errorf( + "load run: every virtual user (%d of %d) failed its first iteration - the target was not reachable", ran, len(t.outcome)) +} + +// vuWorker is one virtual user's private world: its own HTTP client (and so +// its own cookie jar and connection pool), its own instance of every flow +// node, its own persistence side-channels, and its own metrics aggregator. +// +// The isolation is what makes a VU a believable simulated user rather than +// one of N goroutines sharing a session, and it is why node graphs are built +// per VU instead of once for the whole run. +type vuWorker struct { + flowID idwrap.IDWrap + flowName string + httpClient *http.Client + flowNodeMap map[idwrap.IDWrap]node.FlowNode + requestNodes map[idwrap.IDWrap]bool + runnerInst *flowlocalrunner.FlowLocalRunner + agg *loadmetrics.Aggregator + baseVars map[string]any + + // respChan and gqlChan are written once at construction and never + // reassigned; closeOnce makes teardown idempotent so the drain + // goroutines never observe a mutating field. + respChan chan nrequest.NodeRequestSideResp + gqlChan chan ngraphql.NodeGraphQLSideResp + closeOnce sync.Once + + // bytesByExecution carries response sizes from the side-channel drain to + // the metrics recorder. The drain records a size before closing the + // request's Done channel, and the node cannot finish - so its status + // cannot be emitted - until Done is closed, which is what makes the + // lookup below reliable. TestRunRecordsResponseBytes guards that ordering. + bytesMu sync.Mutex + bytesByExecution map[idwrap.IDWrap]int64 +} + +// aggregatorFlushInterval documents the cadence the aggregator was built for. +// Load runs flush once at the end today; streaming interval frames is Phase 2. +const aggregatorFlushInterval = 5 * time.Second + +// newWorkers reads the flow's topology once, then builds one isolated worker +// per VU. The returned release function tears every worker down. +func newWorkers(ctx context.Context, cfg Config, services runner.RunnerServices, logger *slog.Logger) ([]*vuWorker, func(), error) { + if err := cfg.validate(); err != nil { + return nil, nil, err + } + + nodes, err := services.NodeService.GetNodesByFlowID(ctx, cfg.Flow.ID) + if err != nil { + return nil, nil, fmt.Errorf("load run: get nodes for flow %q: %w", cfg.Flow.Name, err) + } + edges, err := services.EdgeService.GetEdgesByFlowID(ctx, cfg.Flow.ID) + if err != nil { + return nil, nil, fmt.Errorf("load run: get edges for flow %q: %w", cfg.Flow.Name, err) + } + edgeMap := mflow.NewEdgesMap(edges) + + flowVars, err := services.FlowVariableService.GetFlowVariablesByFlowID(ctx, cfg.Flow.ID) + if err != nil { + return nil, nil, fmt.Errorf("load run: get variables for flow %q: %w", cfg.Flow.Name, err) + } + baseVars, err := services.Builder.BuildVariables(ctx, cfg.Flow.WorkspaceID, flowVars) + if err != nil { + return nil, nil, fmt.Errorf("load run: build variables for flow %q: %w", cfg.Flow.Name, err) + } + nodeTimeout := resolveNodeTimeout(baseVars) + + workers := make([]*vuWorker, 0, cfg.VUs) + release := func() { + for _, w := range workers { + w.close() + } + } + + for range cfg.VUs { + w, err := newVUWorker(ctx, cfg, services, nodes, edgeMap, baseVars, nodeTimeout, logger) + if err != nil { + release() + return nil, nil, err + } + workers = append(workers, w) + } + + return workers, release, nil +} + +func newVUWorker( + ctx context.Context, + cfg Config, + services runner.RunnerServices, + nodes []mflow.Node, + edgeMap mflow.EdgesMap, + baseVars map[string]any, + nodeTimeout time.Duration, + logger *slog.Logger, +) (*vuWorker, error) { + w := &vuWorker{ + flowID: cfg.Flow.ID, + flowName: cfg.Flow.Name, + httpClient: httpclient.New(), + agg: loadmetrics.NewAggregator(aggregatorFlushInterval), + baseVars: baseVars, + bytesByExecution: make(map[idwrap.IDWrap]int64), + } + + // The side-channels exist so responses can be persisted during a normal + // run. A load run must not persist anything per iteration, so both are + // drained and discarded here - but they still have to be consumed, + // because request nodes block on the Done handshake. + bufferSize := max(len(nodes)*100, 1) + respChan := make(chan nrequest.NodeRequestSideResp, bufferSize) + gqlChan := make(chan ngraphql.NodeGraphQLSideResp, bufferSize) + w.respChan = respChan + w.gqlChan = gqlChan + + go func() { + for resp := range respChan { + // The size is recorded before Done is closed, which is what lets + // the metrics recorder read it back later (see bytesByExecution). + w.addBytes(resp.ExecutionID, int64(len(resp.Resp.HTTPResponse.Body))) + if resp.Done != nil { + close(resp.Done) + } + } + }() + go func() { + for resp := range gqlChan { + if resp.Done != nil { + close(resp.Done) + } + } + }() + + flowNodeMap, startNodeIDs, err := services.Builder.BuildNodes( + ctx, *cfg.Flow, nodes, nodeTimeout, w.httpClient, w.respChan, w.gqlChan, services.JSClient, + ) + if err != nil { + w.close() + return nil, fmt.Errorf("load run: build nodes for flow %q: %w", cfg.Flow.Name, err) + } + + w.flowNodeMap = flowNodeMap + w.requestNodes = make(map[idwrap.IDWrap]bool, len(flowNodeMap)) + for id, n := range flowNodeMap { + if _, ok := n.(*nrequest.NodeRequest); ok { + w.requestNodes[id] = true + } + } + + w.runnerInst = flowlocalrunner.CreateFlowRunner( + idwrap.NewNow(), cfg.Flow.ID, startNodeIDs, flowNodeMap, edgeMap, nodeTimeout, logger, + flowlocalrunner.WithLeanMode(true), + ) + + return w, nil +} + +// close stops this worker's drain goroutines. It is safe to call more than +// once, which matters because the setup error path tears a half-built worker +// down and the caller's release function then tears every worker down again. +func (w *vuWorker) close() { + w.closeOnce.Do(func() { + close(w.respChan) + close(w.gqlChan) + }) +} + +func (w *vuWorker) addBytes(executionID idwrap.IDWrap, n int64) { + w.bytesMu.Lock() + defer w.bytesMu.Unlock() + w.bytesByExecution[executionID] += n +} + +func (w *vuWorker) takeBytes(executionID idwrap.IDWrap) int64 { + w.bytesMu.Lock() + defer w.bytesMu.Unlock() + n := w.bytesByExecution[executionID] + delete(w.bytesByExecution, executionID) + return n +} + +// iterate runs the flow once and records every request node's outcome. +func (w *vuWorker) iterate(ctx context.Context) error { + // Nodes write their output into the variable map, so each iteration needs + // its own copy - otherwise iterations would read each other's results. + vars, _ := node.DeepCopyValue(w.baseVars).(map[string]any) + if vars == nil { + vars = make(map[string]any, len(w.baseVars)) + } + + statusChan := make(chan flowrunner.FlowNodeStatus, len(w.flowNodeMap)*4+8) + flowChan := make(chan flowrunner.FlowStatus, 8) + + var runErr error + done := make(chan struct{}) + go func() { + defer close(done) + runErr = w.runnerInst.Run(ctx, statusChan, flowChan, vars) + }() + + // Drain both channels to completion - the runner closes them on the way + // out - so no goroutine outlives an iteration. + var final flowrunner.FlowStatus + for statusChan != nil || flowChan != nil { + select { + case status, ok := <-statusChan: + if !ok { + statusChan = nil + continue + } + w.record(status) + case status, ok := <-flowChan: + if !ok { + flowChan = nil + continue + } + final = status + } + } + <-done + + // Anything left behind belongs to a request whose node never reported; + // dropping it keeps the map bounded across a long run. + w.resetBytes() + + if runErr != nil { + return runErr + } + if final != flowrunner.FlowStatusSuccess { + return fmt.Errorf("flow %q finished with status %s", w.flowName, flowrunner.FlowStatusString(final)) + } + return nil +} + +func (w *vuWorker) resetBytes() { + w.bytesMu.Lock() + defer w.bytesMu.Unlock() + clear(w.bytesByExecution) +} + +// record aggregates one terminal node status. Only HTTP request nodes are +// counted: they are the ones lean mode covers, and the ones whose latency the +// report is about. +func (w *vuWorker) record(status flowrunner.FlowNodeStatus) { + if status.State == mflow.NODE_STATE_RUNNING { + return + } + if !w.requestNodes[status.NodeID] { + return + } + + class := loadmetrics.ClassifyStatus(statusCodeOf(status.OutputData), status.Error) + w.agg.Record( + loadmetrics.Key{Step: status.Name, StatusClass: class}, + status.RunDuration, + w.takeBytes(status.ExecutionID), + isFailureClass(class), + ) +} + +// isFailureClass decides what counts towards the report's error rate. It +// follows the load-testing convention: anything that is not a 2xx or 3xx is a +// failed request, whether the failure came from the server or the transport. +func isFailureClass(class loadmetrics.StatusClass) bool { + return class != loadmetrics.StatusClass2xx && class != loadmetrics.StatusClass3xx +} + +// statusCodeOf digs the HTTP status out of a request node's output. Lean mode +// drops the response body but keeps the status, which is exactly what +// classification needs. A missing status yields 0, which ClassifyStatus +// buckets as an error - correct, since a request node that produced no status +// did not complete a request. +func statusCodeOf(output any) int { + m, ok := output.(map[string]any) + if !ok { + return 0 + } + resp, ok := m[nrequest.OUTPUT_RESPONSE_NAME].(map[string]any) + if !ok { + return 0 + } + switch status := resp["status"].(type) { + case float64: + return int(status) + case int: + return status + case int32: + return int(status) + case int64: + return int(status) + default: + return 0 + } +} + +// resolveNodeTimeout mirrors the functional run path's timeout resolution, so +// a step behaves the same under load as it does in a normal run. +func resolveNodeTimeout(baseVars map[string]any) time.Duration { + req := &node.FlowNodeRequest{VarMap: baseVars, ReadWriteLock: &sync.RWMutex{}} + raw, err := node.ReadVarRaw(req, "timeout") + if err != nil { + return defaultNodeTimeout + } + switch seconds := raw.(type) { + case float64: + if seconds > 0 { + return time.Duration(seconds) * time.Second + } + case int: + if seconds > 0 { + return time.Duration(seconds) * time.Second + } + } + return defaultNodeTimeout +} diff --git a/apps/cli/internal/loadrun/loadrun_test.go b/apps/cli/internal/loadrun/loadrun_test.go new file mode 100644 index 000000000..c8af5d9d8 --- /dev/null +++ b/apps/cli/internal/loadrun/loadrun_test.go @@ -0,0 +1,576 @@ +package loadrun + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/the-dev-tools/dev-tools/apps/cli/internal/common" + "github.com/the-dev-tools/dev-tools/apps/cli/internal/runner" + "github.com/the-dev-tools/dev-tools/packages/db/pkg/sqlitemem" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/flowbuilder" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/node/nrequest" + gqlresolver "github.com/the-dev-tools/dev-tools/packages/server/pkg/graphql/resolver" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/http/resolver" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/ioworkspace" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/loadmetrics" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mflow" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mload" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/service/scredential" + yamlflowsimplev2 "github.com/the-dev-tools/dev-tools/packages/server/pkg/translate/yamlflowsimplev2" +) + +// twoStepFlowYAML is the load-test workhorse: two chained request steps, so a +// completed iteration must produce exactly two recorded requests. +func twoStepFlowYAML(baseURL string) string { + return fmt.Sprintf(` +workspace_name: Load Test Workspace +flows: + - name: LoadFlow + steps: + - manual_start: + name: Start + - request: + name: StepOne + depends_on: Start + method: GET + url: %s/one + - request: + name: StepTwo + depends_on: StepOne + method: GET + url: %s/two +`, baseURL, baseURL) +} + +// setupFlow imports a yamlflow document into a fresh in-memory workspace and +// returns the flow plus the services a load run needs. It mirrors the CLI's +// own setup in cmd/flow.go: the import happens exactly once, before any load +// iteration runs. +func setupFlow(t *testing.T, yamlDoc, flowName string) (*mflow.Flow, runner.RunnerServices) { + t.Helper() + + ctx := t.Context() + logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelError})) + + db, cleanup, err := sqlitemem.NewSQLiteMem(ctx) + if err != nil { + t.Fatalf("create in-memory db: %v", err) + } + t.Cleanup(cleanup) + + services, err := common.CreateServices(ctx, db, logger) + if err != nil { + t.Fatalf("create services: %v", err) + } + + workspaceID := idwrap.NewNow() + bundle, err := yamlflowsimplev2.ConvertSimplifiedYAML([]byte(yamlDoc), yamlflowsimplev2.ConvertOptionsV2{ + WorkspaceID: workspaceID, + }) + if err != nil { + t.Fatalf("convert yaml: %v", err) + } + + builder := flowbuilder.New( + &services.Node, &services.NodeRequest, &services.NodeFor, &services.NodeForEach, + &services.NodeIf, &services.NodeJS, &services.NodeAI, &services.NodeAiProvider, + &services.NodeMemory, &services.NodeGraphQL, &services.NodeWsConnection, + &services.NodeWsSend, &services.NodeWait, &services.NodeSubFlowTrigger, + &services.NodeSubFlowReturn, &services.NodeRunSubFlow, &services.WebSocket, + &services.WebSocketHeader, &services.GraphQL, &services.GraphQLHeader, + &services.Workspace, &services.Variable, &services.FlowVariable, + resolver.NewStandardResolver( + &services.HTTP, &services.HTTPHeader, services.HTTPSearchParam, + services.HTTPBodyRaw, services.HTTPBodyForm, services.HTTPBodyUrlEncoded, + services.HTTPAssert, + ), + gqlresolver.NewStandardResolver( + services.GraphQL.Reader(), &services.GraphQLHeader, &services.GraphQLAssert, + ), + services.Logger, + scredential.NewLLMProviderFactory(&services.Credential), + ) + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin tx: %v", err) + } + bundle.Workspace.ID = workspaceID + if err := services.Workspace.TX(tx).Create(ctx, &bundle.Workspace); err != nil { + _ = tx.Rollback() + t.Fatalf("create workspace: %v", err) + } + importOpts := ioworkspace.GetDefaultImportOptions(workspaceID) + importOpts.PreserveIDs = true + if _, err := ioworkspace.New(services.Queries, logger).Import(ctx, tx, bundle, importOpts); err != nil { + _ = tx.Rollback() + t.Fatalf("import bundle: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit import: %v", err) + } + + flows, err := services.Flow.GetFlowsByWorkspaceID(ctx, workspaceID) + if err != nil { + t.Fatalf("get flows: %v", err) + } + var flow *mflow.Flow + for i := range flows { + if flows[i].Name == flowName { + flow = &flows[i] + break + } + } + if flow == nil { + t.Fatalf("flow %q not found among %d imported flows", flowName, len(flows)) + } + + return flow, runner.RunnerServices{ + NodeService: services.Node, + EdgeService: services.FlowEdge, + FlowVariableService: services.FlowVariable, + Builder: builder, + } +} + +// countingServer is a deterministic target: fixed latency, fixed payload, and +// a request counter so tests can cross-check the aggregated Count. +type countingServer struct { + *httptest.Server + requests atomic.Int64 +} + +const testPayload = `{"ok":true,"payload":"deterministic"}` + +func newCountingServer(t *testing.T, latency time.Duration, status int) *countingServer { + t.Helper() + + cs := &countingServer{} + cs.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cs.requests.Add(1) + if latency > 0 { + time.Sleep(latency) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(testPayload)) + })) + t.Cleanup(cs.Close) + return cs +} + +// TestRunAggregatesAcrossVUs is the end-to-end proof: four concurrent VUs +// drive a two-step flow for a bounded iteration count, and the merged report +// accounts for every request exactly once, per step. +func TestRunAggregatesAcrossVUs(t *testing.T) { + if testing.Short() { + t.Skip("skipping load run in short mode") + } + + const ( + vus = 4 + iterations = 12 + steps = 2 + ) + + srv := newCountingServer(t, 5*time.Millisecond, http.StatusOK) + flow, services := setupFlow(t, twoStepFlowYAML(srv.URL), "LoadFlow") + + result, err := Run(t.Context(), Config{ + ScenarioName: "test-scenario", + Flow: flow, + VUs: vus, + MaxIterations: iterations, + }, services, nil) + if err != nil { + t.Fatalf("Run failed: %v", err) + } + + if result.Summary.Iterations != iterations { + t.Errorf("Summary.Iterations = %d, want %d", result.Summary.Iterations, iterations) + } + if result.Summary.Errors != 0 { + t.Errorf("Summary.Errors = %d, want 0", result.Summary.Errors) + } + + wantCount := int64(iterations * steps) + if result.Report.Total.Count != wantCount { + t.Errorf("Report.Total.Count = %d, want %d", result.Report.Total.Count, wantCount) + } + if got := srv.requests.Load(); got != wantCount { + t.Errorf("server saw %d requests, report counted %d", got, result.Report.Total.Count) + } + if result.Report.Total.RPS <= 0 { + t.Errorf("Report.Total.RPS = %v, want > 0", result.Report.Total.RPS) + } + if result.Report.Total.ErrorCount != 0 { + t.Errorf("Report.Total.ErrorCount = %d, want 0", result.Report.Total.ErrorCount) + } + + for _, step := range []string{"StepOne", "StepTwo"} { + key := loadmetrics.Key{Step: step, StatusClass: loadmetrics.StatusClass2xx} + stats, ok := result.Report.PerStep[key] + if !ok { + t.Fatalf("missing per-step key %+v; got keys %v", key, reportKeys(result.Report)) + } + if stats.Count != iterations { + t.Errorf("%s count = %d, want %d", step, stats.Count, iterations) + } + if stats.P50 <= 0 { + t.Errorf("%s p50 = %v, want > 0", step, stats.P50) + } + } + + // Concurrency actually happened: four VUs each sleeping 5ms per step + // cannot have run serially in the wall time we measured. + serialFloor := time.Duration(iterations*steps) * 5 * time.Millisecond + if result.Summary.Elapsed >= serialFloor { + t.Errorf("elapsed %v >= serial floor %v: VUs did not overlap", result.Summary.Elapsed, serialFloor) + } + + // The by-step fold that feeds the console table sees the same totals. + byStep, ok := result.ByStep.PerStep[loadmetrics.Key{Step: "StepOne"}] + if !ok { + t.Fatalf("ByStep missing StepOne; got keys %v", reportKeys(result.ByStep)) + } + if byStep.Count != iterations { + t.Errorf("ByStep StepOne count = %d, want %d", byStep.Count, iterations) + } +} + +func reportKeys(r loadmetrics.Report) []loadmetrics.Key { + keys := make([]loadmetrics.Key, 0, len(r.PerStep)) + for k := range r.PerStep { + keys = append(keys, k) + } + return keys +} + +// TestRunClassifiesServerErrors proves the status taxonomy survives the trip +// from the engine to the report: an all-500 target lands in the 5xx bucket +// and counts as an error, without failing the run. +func TestRunClassifiesServerErrors(t *testing.T) { + if testing.Short() { + t.Skip("skipping load run in short mode") + } + + srv := newCountingServer(t, 0, http.StatusInternalServerError) + flow, services := setupFlow(t, twoStepFlowYAML(srv.URL), "LoadFlow") + + const iterations = 6 + result, err := Run(t.Context(), Config{ + Flow: flow, + VUs: 2, + MaxIterations: iterations, + }, services, nil) + if err != nil { + t.Fatalf("Run failed: %v", err) + } + + key := loadmetrics.Key{Step: "StepOne", StatusClass: loadmetrics.StatusClass5xx} + stats, ok := result.Report.PerStep[key] + if !ok { + t.Fatalf("missing 5xx key %+v; got keys %v", key, reportKeys(result.Report)) + } + if stats.Count != iterations { + t.Errorf("5xx count = %d, want %d", stats.Count, iterations) + } + if stats.ErrorCount != iterations { + t.Errorf("5xx ErrorCount = %d, want %d (non-2xx/3xx counts as an error)", stats.ErrorCount, iterations) + } + if _, unexpected := result.Report.PerStep[loadmetrics.Key{Step: "StepOne", StatusClass: loadmetrics.StatusClass2xx}]; unexpected { + t.Error("did not expect a 2xx bucket from an all-500 target") + } +} + +// TestRunRecordsResponseBytes guards the assumption that lets load mode +// account for bytes at all: the persistence side-channel hands the raw +// response to the drain before the node's status is emitted. If the engine +// ever reorders that, this test fails instead of silently zeroing bytes. +func TestRunRecordsResponseBytes(t *testing.T) { + if testing.Short() { + t.Skip("skipping load run in short mode") + } + + srv := newCountingServer(t, 0, http.StatusOK) + flow, services := setupFlow(t, twoStepFlowYAML(srv.URL), "LoadFlow") + + const iterations = 5 + result, err := Run(t.Context(), Config{ + Flow: flow, + VUs: 2, + MaxIterations: iterations, + }, services, nil) + if err != nil { + t.Fatalf("Run failed: %v", err) + } + + wantBytes := int64(iterations * 2 * len(testPayload)) + if result.Report.Total.Bytes != wantBytes { + t.Errorf("Report.Total.Bytes = %d, want %d", result.Report.Total.Bytes, wantBytes) + } +} + +// TestRunUsesLeanMode proves lean mode reaches the request nodes end to end: +// StepTwo interpolates StepOne's response body into a header, and what the +// server receives is the lean placeholder rather than the decoded body. +func TestRunUsesLeanMode(t *testing.T) { + if testing.Short() { + t.Skip("skipping load run in short mode") + } + + var ( + mu sync.Mutex + echoed []string + observed = func(v string) { + mu.Lock() + defer mu.Unlock() + echoed = append(echoed, v) + } + ) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if v := r.Header.Get("X-Echo-Body"); v != "" { + observed(v) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(testPayload)) + })) + t.Cleanup(srv.Close) + + yamlDoc := fmt.Sprintf(` +workspace_name: Lean Mode Workspace +flows: + - name: LeanFlow + steps: + - manual_start: + name: Start + - request: + name: StepOne + depends_on: Start + method: GET + url: %s/one + - request: + name: StepTwo + depends_on: StepOne + method: GET + url: %s/two + headers: + X-Echo-Body: '{{ StepOne.response.body }}' +`, srv.URL, srv.URL) + + flow, services := setupFlow(t, yamlDoc, "LeanFlow") + + if _, err := Run(t.Context(), Config{Flow: flow, VUs: 1, MaxIterations: 2}, services, nil); err != nil { + t.Fatalf("Run failed: %v", err) + } + + mu.Lock() + defer mu.Unlock() + if len(echoed) == 0 { + t.Fatal("StepTwo never sent the echo header") + } + for _, v := range echoed { + if v != nrequest.LeanBodyPlaceholder { + t.Errorf("echoed body = %q, want lean placeholder %q", v, nrequest.LeanBodyPlaceholder) + } + } +} + +// TestRunDurationBoundedRunSucceeds pins contract addition #3: Duration is +// passed through RunProfile only. Wrapping the scenario context in a timeout +// would make this run - which completed exactly as configured - return +// context.DeadlineExceeded. +func TestRunDurationBoundedRunSucceeds(t *testing.T) { + if testing.Short() { + t.Skip("skipping load run in short mode") + } + + srv := newCountingServer(t, 0, http.StatusOK) + flow, services := setupFlow(t, twoStepFlowYAML(srv.URL), "LoadFlow") + + result, err := Run(t.Context(), Config{ + Flow: flow, + VUs: 2, + Duration: 250 * time.Millisecond, + }, services, nil) + if err != nil { + t.Fatalf("duration-bounded run returned error: %v", err) + } + if result.Summary.Iterations == 0 { + t.Error("duration-bounded run completed zero iterations") + } + if result.Report.Total.Count == 0 { + t.Error("duration-bounded run recorded no requests") + } +} + +// TestRunSetupFailureWhenEveryVUFailsFirstIteration pins contract addition +// #9's infra-failure case: an unreachable target on every VU's first +// iteration is a setup failure (exit 1), not a completed run with errors. +func TestRunSetupFailureWhenEveryVUFailsFirstIteration(t *testing.T) { + if testing.Short() { + t.Skip("skipping load run in short mode") + } + + srv := newCountingServer(t, 0, http.StatusOK) + baseURL := srv.URL + srv.Close() // nothing is listening any more + + flow, services := setupFlow(t, twoStepFlowYAML(baseURL), "LoadFlow") + + _, err := Run(t.Context(), Config{ + Flow: flow, + VUs: 2, + MaxIterations: 4, + }, services, nil) + if err == nil { + t.Fatal("expected an unreachable target to be reported as a setup failure") + } + if !strings.Contains(err.Error(), "first iteration") { + t.Errorf("error %q does not explain that every VU failed its first iteration", err) + } +} + +// TestRunErrorsDoNotFailACompletedRun is #9's other half: once at least one +// VU gets going, request-level failures are data, not a run failure. +func TestRunErrorsDoNotFailACompletedRun(t *testing.T) { + if testing.Short() { + t.Skip("skipping load run in short mode") + } + + var n atomic.Int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // First two requests succeed, everything after is a hard failure. + if n.Add(1) > 2 { + hj, ok := w.(http.Hijacker) + if ok { + conn, _, err := hj.Hijack() + if err == nil { + _ = conn.Close() + return + } + } + } + _, _ = w.Write([]byte(testPayload)) + })) + t.Cleanup(srv.Close) + + flow, services := setupFlow(t, twoStepFlowYAML(srv.URL), "LoadFlow") + + result, err := Run(t.Context(), Config{ + Flow: flow, + VUs: 1, + MaxIterations: 4, + }, services, nil) + if err != nil { + t.Fatalf("a run whose first iteration succeeded must not fail: %v", err) + } + if result.Summary.Errors == 0 { + t.Error("expected later iterations to be counted as errors") + } +} + +func TestRunValidatesConfig(t *testing.T) { + cases := []struct { + name string + cfg Config + wantSub string + }{ + {"no flow", Config{VUs: 1, MaxIterations: 1}, "flow"}, + {"zero vus", Config{Flow: &mflow.Flow{}, MaxIterations: 1}, "vus"}, + {"no stop condition", Config{Flow: &mflow.Flow{}, VUs: 1}, "duration"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := Run(context.Background(), tc.cfg, runner.RunnerServices{}, nil) + if err == nil { + t.Fatal("expected a validation error") + } + if !strings.Contains(strings.ToLower(err.Error()), tc.wantSub) { + t.Errorf("error %q does not mention %q", err, tc.wantSub) + } + }) + } +} + +// TestConfigFromScenario checks the mload.Scenario -> Config adapter, which +// is what `--scenario ` resolves through. +func TestConfigFromScenario(t *testing.T) { + flow := &mflow.Flow{Name: "Checkout"} + scenario := mload.Scenario{ + Name: "checkout-baseline", + FlowName: "Checkout", + Executor: mload.ExecutorConstantVUs, + VUs: 7, + Duration: 45 * time.Second, + MaxIterations: 900, + } + + got := ConfigFromScenario(scenario, flow) + want := Config{ + ScenarioName: "checkout-baseline", + Flow: flow, + VUs: 7, + Duration: 45 * time.Second, + MaxIterations: 900, + } + if got != want { + t.Errorf("ConfigFromScenario() = %+v, want %+v", got, want) + } +} + +// TestVUWorkersAreIsolated pins the reason node graphs are built per VU +// rather than once: each virtual user is a distinct simulated user, so it +// needs its own HTTP client (and therefore its own cookie jar and connection +// pool) and its own metrics aggregator. +func TestVUWorkersAreIsolated(t *testing.T) { + if testing.Short() { + t.Skip("skipping load run in short mode") + } + + srv := newCountingServer(t, 0, http.StatusOK) + flow, services := setupFlow(t, twoStepFlowYAML(srv.URL), "LoadFlow") + + const vus = 3 + workers, release, err := newWorkers(t.Context(), Config{Flow: flow, VUs: vus, MaxIterations: 1}, services, nil) + if err != nil { + t.Fatalf("newWorkers failed: %v", err) + } + defer release() + + if len(workers) != vus { + t.Fatalf("got %d workers, want %d", len(workers), vus) + } + + seenClients := make(map[any]bool, vus) + seenAggs := make(map[*loadmetrics.Aggregator]bool, vus) + seenNodeMaps := make(map[any]bool, vus) + for _, w := range workers { + seenClients[w.httpClient] = true + seenAggs[w.agg] = true + for id := range w.flowNodeMap { + seenNodeMaps[w.flowNodeMap[id]] = true + break + } + } + if len(seenClients) != vus { + t.Errorf("%d distinct HTTP clients across %d VUs, want %d", len(seenClients), vus, vus) + } + if len(seenAggs) != vus { + t.Errorf("%d distinct aggregators across %d VUs, want %d", len(seenAggs), vus, vus) + } + if len(seenNodeMaps) != vus { + t.Errorf("%d distinct node instances across %d VUs, want %d", len(seenNodeMaps), vus, vus) + } +} From b79ec0b7ab9175a1950a67afb4aa216a2db3e985 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 20:38:43 +0300 Subject: [PATCH 24/38] feat(cli): load report table and additive load_report JSON field The console gets the aggregate table - p50/p95/p99, RPS and error rate per step plus a TOTAL row - and a context block above it that states which steps the numbers cover, since lean mode only reaches HTTP request nodes. The JSON report keeps writing the bare array of flow results it always has. Only a load run switches it to an object, so the additive load_report has somewhere to live and no existing consumer sees a change. Inside it the metrics ride as the spec's LoadRunReport, which is what makes the CLI's report the N=1 case of the message the fleet will stream later. Status classes cross that boundary through an explicit mapping, with loadmetrics.StatusClass as the source of truth; a round-trip test fails if either side gains a value the other lacks. Per-step rows are sorted so map iteration order never reaches the file. --- apps/cli/internal/reporter/load.go | 316 +++++++++++++++++++++ apps/cli/internal/reporter/load_test.go | 358 ++++++++++++++++++++++++ apps/cli/internal/reporter/reporter.go | 45 ++- 3 files changed, 715 insertions(+), 4 deletions(-) create mode 100644 apps/cli/internal/reporter/load.go create mode 100644 apps/cli/internal/reporter/load_test.go diff --git a/apps/cli/internal/reporter/load.go b/apps/cli/internal/reporter/load.go new file mode 100644 index 000000000..8d732e277 --- /dev/null +++ b/apps/cli/internal/reporter/load.go @@ -0,0 +1,316 @@ +package reporter + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "google.golang.org/protobuf/encoding/protojson" + + "github.com/the-dev-tools/dev-tools/apps/cli/internal/model" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/loadmetrics" + load_metricsv1 "github.com/the-dev-tools/dev-tools/packages/spec/dist/buf/go/api/load_metrics/v1" +) + +// LoadRunMeta describes the load run that produced a report: which profile +// ran, and what the scheduler actually managed to do. +type LoadRunMeta struct { + // ScenarioName is the `load:` block entry the profile came from, empty + // for a run configured entirely from flags. + ScenarioName string + FlowName string + VUs int + // Duration and MaxIterations are the configured stop conditions. + Duration time.Duration + MaxIterations int64 + // Iterations, Errors and Elapsed are what actually happened. + Iterations int64 + Errors int64 + Elapsed time.Duration + // WorkerVersion identifies the binary that produced the numbers, so + // baselines from different builds are not silently compared. + WorkerVersion string +} + +// LoadReport is a completed load run in the shape the reporters need: +// metadata, the full (step, status-class) breakdown, and the same data folded +// to one row per step for the console table. +type LoadReport struct { + Meta LoadRunMeta + Report loadmetrics.Report + ByStep loadmetrics.Report +} + +// loadReportSink is implemented by reporters that can render a load report. +// Reporters that cannot (JUnit has no place to put it) simply do not. +type loadReportSink interface { + SetLoadReport(report *LoadReport) +} + +// SetLoadReport hands the load report to every reporter that can use it. It +// must be called before Flush. Reporters that never receive one behave +// exactly as they did before load mode existed. +func (g *ReporterGroup) SetLoadReport(report *LoadReport) { + for _, reporter := range g.reporters { + if sink, ok := reporter.(loadReportSink); ok { + sink.SetLoadReport(report) + } + } +} + +const ( + // loadTableMinStepWidth keeps the Step column at its published width for + // ordinary step names; longer names widen it rather than being truncated, + // since a truncated step name cannot be matched back to the flow. + loadTableMinStepWidth = 16 + loadTableStatWidth = 9 + loadTableRPSWidth = 8 + // loadTableTotalRow is the label of the whole-run row, always printed + // last. + loadTableTotalRow = "TOTAL" +) + +// FormatLoadTable renders the aggregate console table: one row per step plus +// a TOTAL row, sorted by step name so the output is stable across runs. +func FormatLoadTable(report LoadReport) string { + steps := make([]string, 0, len(report.ByStep.PerStep)) + stepWidth := loadTableMinStepWidth + for key := range report.ByStep.PerStep { + steps = append(steps, key.Step) + if len(key.Step)+1 > stepWidth { + stepWidth = len(key.Step) + 1 + } + } + sort.Strings(steps) + + var b strings.Builder + writeRow := func(step, p50, p95, p99, rps, errPct string) { + fmt.Fprintf(&b, "%-*s%-*s%-*s%-*s%-*s%s\n", + stepWidth, step, + loadTableStatWidth, p50, + loadTableStatWidth, p95, + loadTableStatWidth, p99, + loadTableRPSWidth, rps, + errPct) + } + writeStats := func(label string, stats loadmetrics.Stats) { + writeRow(label, + formatLoadDuration(stats.P50), + formatLoadDuration(stats.P95), + formatLoadDuration(stats.P99), + fmt.Sprintf("%.1f", stats.RPS), + fmt.Sprintf("%.1f", errorPercent(stats))) + } + + writeRow("Step", "p50", "p95", "p99", "RPS", "Err%") + for _, step := range steps { + writeStats(step, report.ByStep.PerStep[loadmetrics.Key{Step: step}]) + } + writeStats(loadTableTotalRow, report.ByStep.Total) + + return b.String() +} + +// LoadMetricsScope states what a load run measures, and by extension what it +// keeps memory-flat. It is surfaced in both the console output and the JSON +// report so nobody has to infer the boundary from a suspiciously empty table. +const LoadMetricsScope = "Counts HTTP request steps only; GraphQL, WebSocket and sub-flow steps run but are neither counted nor memory-bounded." + +// FormatLoadHeader renders the one-off context lines printed above the table. +// It is deliberately separate so the table itself stays exactly as published. +func FormatLoadHeader(meta LoadRunMeta) string { + title := "Load Run" + if meta.ScenarioName != "" { + title = "Load Run: " + meta.ScenarioName + } + + stop := make([]string, 0, 2) + if meta.Duration > 0 { + stop = append(stop, "Duration: "+meta.Duration.String()) + } + if meta.MaxIterations > 0 { + stop = append(stop, fmt.Sprintf("Max iterations: %d", meta.MaxIterations)) + } + + var b strings.Builder + fmt.Fprintf(&b, "\n=== %s ===\n", title) + fmt.Fprintf(&b, "Flow: %s | VUs: %d", meta.FlowName, meta.VUs) + for _, s := range stop { + fmt.Fprintf(&b, " | %s", s) + } + fmt.Fprintf(&b, "\nIterations: %d | Iteration errors: %d | Elapsed: %s\n", + meta.Iterations, meta.Errors, formatLoadDuration(meta.Elapsed)) + fmt.Fprintf(&b, "%s\n\n", LoadMetricsScope) + return b.String() +} + +func errorPercent(stats loadmetrics.Stats) float64 { + if stats.Count == 0 { + return 0 + } + return 100 * float64(stats.ErrorCount) / float64(stats.Count) +} + +// formatLoadDuration renders a latency compactly enough to fit the table's +// columns: microseconds below a millisecond, whole milliseconds below a +// second, seconds above that. +func formatLoadDuration(d time.Duration) string { + switch { + case d <= 0: + return "0s" + case d < time.Millisecond: + return fmt.Sprintf("%dµs", d.Microseconds()) + case d < time.Second: + return fmt.Sprintf("%dms", d.Milliseconds()) + default: + return fmt.Sprintf("%.2fs", d.Seconds()) + } +} + +// loadStatusClassToProto maps the Go status classes onto the generated enum. +// loadmetrics.StatusClass is the source of truth for these values; the +// generated LoadStatusClass mirrors it (see api/load-metrics.tsp). +var loadStatusClassToProto = map[loadmetrics.StatusClass]load_metricsv1.LoadStatusClass{ + loadmetrics.StatusClass2xx: load_metricsv1.LoadStatusClass_LOAD_STATUS_CLASS_TWO_XX, + loadmetrics.StatusClass3xx: load_metricsv1.LoadStatusClass_LOAD_STATUS_CLASS_THREE_XX, + loadmetrics.StatusClass4xx: load_metricsv1.LoadStatusClass_LOAD_STATUS_CLASS_FOUR_XX, + loadmetrics.StatusClass5xx: load_metricsv1.LoadStatusClass_LOAD_STATUS_CLASS_FIVE_XX, + loadmetrics.StatusClassError: load_metricsv1.LoadStatusClass_LOAD_STATUS_CLASS_ERROR, + loadmetrics.StatusClassTimeout: load_metricsv1.LoadStatusClass_LOAD_STATUS_CLASS_TIMEOUT, +} + +// LoadStatusClassToProto converts an aggregation status class to the wire +// enum. An unrecognized class yields UNSPECIFIED rather than guessing. +func LoadStatusClassToProto(class loadmetrics.StatusClass) load_metricsv1.LoadStatusClass { + return loadStatusClassToProto[class] +} + +// LoadStatusClassFromProto is the inverse of LoadStatusClassToProto. +// UNSPECIFIED, and any value this build does not know, yield "". +func LoadStatusClassFromProto(class load_metricsv1.LoadStatusClass) loadmetrics.StatusClass { + for goClass, protoClass := range loadStatusClassToProto { + if protoClass == class { + return goClass + } + } + return "" +} + +// jsonLoadReport is the additive `load_report` object of the JSON report. The +// run's metrics ride in Report as the spec's LoadRunReport, serialized with +// protojson so the CLI's report really is the N=1 case of the same message +// the Phase 2 wire protocol carries. +type jsonLoadReport struct { + Scenario string `json:"scenario,omitempty"` + Flow string `json:"flow"` + VUs int `json:"vus"` + Duration string `json:"duration,omitempty"` + MaxIterations int64 `json:"max_iterations,omitempty"` + Iterations int64 `json:"iterations"` + Errors int64 `json:"errors"` + Elapsed string `json:"elapsed"` + WorkerVersion string `json:"worker_version"` + MetricsScope string `json:"metrics_scope"` + Report json.RawMessage `json:"report"` +} + +// jsonLoadDocument is what the JSON reporter writes when a load report is +// present. Without one it keeps writing the bare array of flow results it +// always has, so nothing changes for existing consumers. +type jsonLoadDocument struct { + Flows []model.FlowRunResult `json:"flows"` + LoadReport *jsonLoadReport `json:"load_report"` +} + +func buildJSONLoadReport(report *LoadReport) (*jsonLoadReport, error) { + proto := loadRunReportProto(report) + // Deterministic protojson output: the package intentionally randomizes + // whitespace, so it is normalized back through encoding/json. + raw, err := protojson.Marshal(proto) + if err != nil { + return nil, fmt.Errorf("serializing load report: %w", err) + } + var normalized any + if err := json.Unmarshal(raw, &normalized); err != nil { + return nil, fmt.Errorf("normalizing load report: %w", err) + } + compact, err := json.Marshal(normalized) + if err != nil { + return nil, fmt.Errorf("normalizing load report: %w", err) + } + + out := &jsonLoadReport{ + Scenario: report.Meta.ScenarioName, + Flow: report.Meta.FlowName, + VUs: report.Meta.VUs, + MaxIterations: report.Meta.MaxIterations, + Iterations: report.Meta.Iterations, + Errors: report.Meta.Errors, + Elapsed: report.Meta.Elapsed.String(), + WorkerVersion: report.Meta.WorkerVersion, + MetricsScope: LoadMetricsScope, + Report: compact, + } + if report.Meta.Duration > 0 { + out.Duration = report.Meta.Duration.String() + } + return out, nil +} + +// loadRunReportProto converts the merged Go report into the generated +// LoadRunReport. Per-step rows are sorted by (step, status class) so the +// serialized report never depends on Go's map iteration order. +// +// Thresholds and the environment fingerprint are left unset: their shapes are +// frozen in Phase 0 but nothing evaluates or collects them until Phase 2. +func loadRunReportProto(report *LoadReport) *load_metricsv1.LoadRunReport { + keys := make([]loadmetrics.Key, 0, len(report.Report.PerStep)) + for key := range report.Report.PerStep { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].Step != keys[j].Step { + return keys[i].Step < keys[j].Step + } + return keys[i].StatusClass < keys[j].StatusClass + }) + + perStep := make([]*load_metricsv1.LoadRunStepStats, 0, len(keys)) + for _, key := range keys { + stats := report.Report.PerStep[key] + perStep = append(perStep, &load_metricsv1.LoadRunStepStats{ + Step: key.Step, + StatusClass: LoadStatusClassToProto(key.StatusClass), + Count: stats.Count, + ErrorCount: stats.ErrorCount, + Bytes: stats.Bytes, + P50Us: stats.P50.Microseconds(), + P90Us: stats.P90.Microseconds(), + P95Us: stats.P95.Microseconds(), + P99Us: stats.P99.Microseconds(), + MaxUs: stats.Max.Microseconds(), + Rps: float32(stats.RPS), + }) + } + + return &load_metricsv1.LoadRunReport{ + Total: loadStatsProto(report.Report.Total), + PerStep: perStep, + } +} + +func loadStatsProto(stats loadmetrics.Stats) *load_metricsv1.LoadStats { + return &load_metricsv1.LoadStats{ + Count: stats.Count, + ErrorCount: stats.ErrorCount, + Bytes: stats.Bytes, + P50Us: stats.P50.Microseconds(), + P90Us: stats.P90.Microseconds(), + P95Us: stats.P95.Microseconds(), + P99Us: stats.P99.Microseconds(), + MaxUs: stats.Max.Microseconds(), + Rps: float32(stats.RPS), + } +} diff --git a/apps/cli/internal/reporter/load_test.go b/apps/cli/internal/reporter/load_test.go new file mode 100644 index 000000000..05e18fa15 --- /dev/null +++ b/apps/cli/internal/reporter/load_test.go @@ -0,0 +1,358 @@ +package reporter + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/the-dev-tools/dev-tools/apps/cli/internal/model" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/loadmetrics" + load_metricsv1 "github.com/the-dev-tools/dev-tools/packages/spec/dist/buf/go/api/load_metrics/v1" +) + +func sampleLoadReport() LoadReport { + return LoadReport{ + Meta: LoadRunMeta{ + ScenarioName: "checkout-baseline", + FlowName: "Checkout", + VUs: 4, + Duration: 30 * time.Second, + Iterations: 512, + Errors: 3, + Elapsed: 30 * time.Second, + WorkerVersion: "v9.9.9", + }, + Report: loadmetrics.Report{ + Total: loadmetrics.Stats{Count: 1024, ErrorCount: 1, Bytes: 4096, P50: 95 * time.Millisecond, RPS: 285.14}, + PerStep: map[loadmetrics.Key]loadmetrics.Stats{ + {Step: "CreateOrder", StatusClass: loadmetrics.StatusClass2xx}: { + Count: 511, Bytes: 2048, P50: 120 * time.Millisecond, RPS: 141.9, + }, + {Step: "CreateOrder", StatusClass: loadmetrics.StatusClass5xx}: { + Count: 1, ErrorCount: 1, P50: 5 * time.Millisecond, RPS: 0.1, + }, + }, + }, + ByStep: loadmetrics.Report{ + Total: loadmetrics.Stats{ + Count: 1024, ErrorCount: 1, + P50: 95 * time.Millisecond, P95: 290 * time.Millisecond, P99: 470 * time.Millisecond, + RPS: 285.14, + }, + PerStep: map[loadmetrics.Key]loadmetrics.Stats{ + {Step: "CreateOrder"}: { + Count: 512, ErrorCount: 1, + P50: 120 * time.Millisecond, P95: 310 * time.Millisecond, P99: 480 * time.Millisecond, + RPS: 142.0, + }, + {Step: "ConfirmOrder"}: { + Count: 512, + P50: 70 * time.Millisecond, P95: 200 * time.Millisecond, P99: 300 * time.Millisecond, + RPS: 143.1, + }, + }, + }, + } +} + +// TestFormatLoadTable pins the aggregate table's exact shape - column +// headings, widths, ordering and rounding - because it is a published output +// format, not incidental formatting. +func TestFormatLoadTable(t *testing.T) { + got := FormatLoadTable(sampleLoadReport()) + + want := "" + + "Step p50 p95 p99 RPS Err%\n" + + "ConfirmOrder 70ms 200ms 300ms 143.1 0.0\n" + + "CreateOrder 120ms 310ms 480ms 142.0 0.2\n" + + "TOTAL 95ms 290ms 470ms 285.1 0.1\n" + + if got != want { + t.Errorf("table mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +// TestFormatLoadTableWidensForLongStepNames keeps the columns aligned when a +// step name outgrows the default column, instead of truncating it. +func TestFormatLoadTableWidensForLongStepNames(t *testing.T) { + report := LoadReport{ + ByStep: loadmetrics.Report{ + Total: loadmetrics.Stats{Count: 1, P50: time.Millisecond, RPS: 1}, + PerStep: map[loadmetrics.Key]loadmetrics.Stats{ + {Step: "AnExtremelyLongStepNameIndeed"}: {Count: 1, P50: time.Millisecond, RPS: 1}, + }, + }, + } + + got := FormatLoadTable(report) + want := "" + + "Step p50 p95 p99 RPS Err%\n" + + "AnExtremelyLongStepNameIndeed 1ms 0s 0s 1.0 0.0\n" + + "TOTAL 1ms 0s 0s 1.0 0.0\n" + + if got != want { + t.Errorf("table mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +// TestFormatLoadHeader covers the context block printed above the table, +// including the scope note that tells the reader which steps the numbers +// actually cover. +func TestFormatLoadHeader(t *testing.T) { + got := FormatLoadHeader(sampleLoadReport().Meta) + + want := "" + + "\n=== Load Run: checkout-baseline ===\n" + + "Flow: Checkout | VUs: 4 | Duration: 30s\n" + + "Iterations: 512 | Iteration errors: 3 | Elapsed: 30.00s\n" + + LoadMetricsScope + "\n\n" + + if got != want { + t.Errorf("header mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +func TestFormatLoadHeaderWithoutScenario(t *testing.T) { + got := FormatLoadHeader(LoadRunMeta{FlowName: "Solo", VUs: 2, MaxIterations: 50, Iterations: 50}) + + want := "" + + "\n=== Load Run ===\n" + + "Flow: Solo | VUs: 2 | Max iterations: 50\n" + + "Iterations: 50 | Iteration errors: 0 | Elapsed: 0s\n" + + LoadMetricsScope + "\n\n" + + if got != want { + t.Errorf("header mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } +} + +// TestLoadMetricsScopeNamesTheUncoveredNodes pins the substance of the note +// rather than its wording, so the promise cannot quietly narrow. +func TestLoadMetricsScopeNamesTheUncoveredNodes(t *testing.T) { + for _, want := range []string{"HTTP request", "GraphQL", "WebSocket", "sub-flow"} { + if !strings.Contains(LoadMetricsScope, want) { + t.Errorf("LoadMetricsScope %q does not mention %q", LoadMetricsScope, want) + } + } +} + +func TestFormatLoadDuration(t *testing.T) { + cases := []struct { + in time.Duration + want string + }{ + {0, "0s"}, + {443 * time.Microsecond, "443µs"}, + {time.Millisecond, "1ms"}, + {120 * time.Millisecond, "120ms"}, + {1500 * time.Millisecond, "1.50s"}, + {2 * time.Minute, "120.00s"}, + } + for _, tc := range cases { + if got := formatLoadDuration(tc.in); got != tc.want { + t.Errorf("formatLoadDuration(%v) = %q, want %q", tc.in, got, tc.want) + } + } +} + +// TestLoadStatusClassProtoRoundTrip is the guard on the Go <-> generated-proto +// status class mapping. loadmetrics.StatusClass is the source of truth; if +// either side grows a value without the other, this fails. +func TestLoadStatusClassProtoRoundTrip(t *testing.T) { + all := []loadmetrics.StatusClass{ + loadmetrics.StatusClass2xx, + loadmetrics.StatusClass3xx, + loadmetrics.StatusClass4xx, + loadmetrics.StatusClass5xx, + loadmetrics.StatusClassError, + loadmetrics.StatusClassTimeout, + } + + seen := make(map[load_metricsv1.LoadStatusClass]bool, len(all)) + for _, class := range all { + proto := LoadStatusClassToProto(class) + if proto == load_metricsv1.LoadStatusClass_LOAD_STATUS_CLASS_UNSPECIFIED { + t.Errorf("StatusClass %q maps to UNSPECIFIED", class) + } + if seen[proto] { + t.Errorf("StatusClass %q collides with an earlier class on %v", class, proto) + } + seen[proto] = true + + if back := LoadStatusClassFromProto(proto); back != class { + t.Errorf("round trip of %q produced %q", class, back) + } + } + + // Every enum value the generated code declares, except UNSPECIFIED, must + // be reachable - otherwise the spec has a class Go cannot produce. + for value, name := range load_metricsv1.LoadStatusClass_name { + enum := load_metricsv1.LoadStatusClass(value) + if enum == load_metricsv1.LoadStatusClass_LOAD_STATUS_CLASS_UNSPECIFIED { + continue + } + if !seen[enum] { + t.Errorf("generated enum %s (%d) has no loadmetrics.StatusClass", name, value) + } + } + + if got := LoadStatusClassToProto("not-a-class"); got != load_metricsv1.LoadStatusClass_LOAD_STATUS_CLASS_UNSPECIFIED { + t.Errorf("unknown class mapped to %v, want UNSPECIFIED", got) + } + if got := LoadStatusClassFromProto(load_metricsv1.LoadStatusClass_LOAD_STATUS_CLASS_UNSPECIFIED); got != "" { + t.Errorf("UNSPECIFIED mapped to %q, want empty", got) + } +} + +// TestJSONReporterWithoutLoadReportIsUnchanged is the zero-default-change +// guard: with no load flags the JSON report is still a bare array of flow +// results, exactly as it has always been. +func TestJSONReporterWithoutLoadReportIsUnchanged(t *testing.T) { + path := filepath.Join(t.TempDir(), "report.json") + rep := newJSONReporter(path) + rep.HandleFlowResult(model.FlowRunResult{FlowName: "FlowA", Status: "success"}) + if err := rep.Flush(); err != nil { + t.Fatalf("Flush failed: %v", err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read report: %v", err) + } + var results []model.FlowRunResult + if err := json.Unmarshal(data, &results); err != nil { + t.Fatalf("report is not a bare array of flow results: %v\n%s", err, data) + } + if len(results) != 1 || results[0].FlowName != "FlowA" { + t.Errorf("unexpected results: %+v", results) + } +} + +// TestJSONReporterWithLoadReport pins the additive load_report object: run +// metadata plus the spec's LoadRunReport, with status classes carried as the +// generated enum's canonical names. +func TestJSONReporterWithLoadReport(t *testing.T) { + path := filepath.Join(t.TempDir(), "report.json") + group, err := NewReporterGroup([]ReportSpec{{Format: ReportFormatJSON, Path: path}}, ReporterOptions{}) + if err != nil { + t.Fatalf("NewReporterGroup failed: %v", err) + } + group.HandleFlowResult(model.FlowRunResult{FlowName: "Checkout", Status: "success"}) + + loadReport := sampleLoadReport() + group.SetLoadReport(&loadReport) + if err := group.Flush(); err != nil { + t.Fatalf("Flush failed: %v", err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read report: %v", err) + } + + var doc struct { + Flows []model.FlowRunResult `json:"flows"` + LoadReport *struct { + Scenario string `json:"scenario"` + Flow string `json:"flow"` + VUs int `json:"vus"` + Duration string `json:"duration"` + Iterations int64 `json:"iterations"` + Errors int64 `json:"errors"` + Elapsed string `json:"elapsed"` + WorkerVersion string `json:"worker_version"` + Report struct { + Total struct { + Count string `json:"count"` + Rps float64 `json:"rps"` + } `json:"total"` + PerStep []struct { + Step string `json:"step"` + StatusClass string `json:"statusClass"` + Count string `json:"count"` + } `json:"perStep"` + } `json:"report"` + } `json:"load_report"` + } + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatalf("unmarshal report: %v\n%s", err, data) + } + + if len(doc.Flows) != 1 || doc.Flows[0].FlowName != "Checkout" { + t.Errorf("flow results not carried through: %+v", doc.Flows) + } + if doc.LoadReport == nil { + t.Fatalf("load_report missing:\n%s", data) + } + if doc.LoadReport.Scenario != "checkout-baseline" { + t.Errorf("scenario = %q", doc.LoadReport.Scenario) + } + if doc.LoadReport.Flow != "Checkout" { + t.Errorf("flow = %q", doc.LoadReport.Flow) + } + if doc.LoadReport.VUs != 4 { + t.Errorf("vus = %d", doc.LoadReport.VUs) + } + if doc.LoadReport.Duration != "30s" { + t.Errorf("duration = %q", doc.LoadReport.Duration) + } + if doc.LoadReport.Iterations != 512 { + t.Errorf("iterations = %d", doc.LoadReport.Iterations) + } + if doc.LoadReport.Errors != 3 { + t.Errorf("errors = %d", doc.LoadReport.Errors) + } + if doc.LoadReport.WorkerVersion != "v9.9.9" { + t.Errorf("worker_version = %q", doc.LoadReport.WorkerVersion) + } + if doc.LoadReport.Report.Total.Count != "1024" { + t.Errorf("total.count = %q, want \"1024\"", doc.LoadReport.Report.Total.Count) + } + if len(doc.LoadReport.Report.PerStep) != 2 { + t.Fatalf("per-step rows = %d, want 2: %s", len(doc.LoadReport.Report.PerStep), data) + } + // Sorted for determinism: 2xx before 5xx for the same step. + first := doc.LoadReport.Report.PerStep[0] + if first.Step != "CreateOrder" || first.StatusClass != "LOAD_STATUS_CLASS_TWO_XX" || first.Count != "511" { + t.Errorf("first per-step row = %+v", first) + } + second := doc.LoadReport.Report.PerStep[1] + if second.StatusClass != "LOAD_STATUS_CLASS_FIVE_XX" { + t.Errorf("second per-step row = %+v", second) + } +} + +// TestJSONLoadReportIsDeterministic guards against map-iteration order +// leaking into the file: two serializations of the same report are identical. +func TestJSONLoadReportIsDeterministic(t *testing.T) { + loadReport := sampleLoadReport() + + var first []byte + for i := range 5 { + path := filepath.Join(t.TempDir(), "report.json") + rep := newJSONReporter(path) + rep.HandleFlowResult(model.FlowRunResult{FlowName: "Checkout"}) + if setter, ok := rep.(loadReportSink); ok { + setter.SetLoadReport(&loadReport) + } else { + t.Fatal("json reporter does not accept a load report") + } + if err := rep.Flush(); err != nil { + t.Fatalf("Flush failed: %v", err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read report: %v", err) + } + if i == 0 { + first = data + continue + } + if string(data) != string(first) { + t.Fatalf("serialization %d differs:\n%s\n---\n%s", i, first, data) + } + } +} diff --git a/apps/cli/internal/reporter/reporter.go b/apps/cli/internal/reporter/reporter.go index 3a04447eb..0cbe533a6 100644 --- a/apps/cli/internal/reporter/reporter.go +++ b/apps/cli/internal/reporter/reporter.go @@ -163,15 +163,22 @@ func NewReporterGroup(specs []ReportSpec, opts ReporterOptions) (*ReporterGroup, // Internal implementations below... type jsonReporter struct { - path string - mu sync.Mutex - results []model.FlowRunResult + path string + mu sync.Mutex + results []model.FlowRunResult + loadReport *LoadReport } func newJSONReporter(path string) Reporter { return &jsonReporter{path: path, results: make([]model.FlowRunResult, 0)} } +func (j *jsonReporter) SetLoadReport(report *LoadReport) { + j.mu.Lock() + defer j.mu.Unlock() + j.loadReport = report +} + func (j *jsonReporter) HandleFlowStart(info FlowStartInfo) {} func (j *jsonReporter) HandleNodeStatus(event NodeStatusEvent) {} @@ -194,7 +201,20 @@ func (j *jsonReporter) Flush() error { return fmt.Errorf("creating json report directory: %w", err) } - data, err := json.MarshalIndent(j.results, "", " ") + // Without a load report the document is the bare array of flow results it + // has always been. A load run - which no existing consumer can be reading + // yet - gets the object form so the additive load_report has somewhere to + // live. + payload := any(j.results) + if j.loadReport != nil { + loadReport, err := buildJSONLoadReport(j.loadReport) + if err != nil { + return err + } + payload = jsonLoadDocument{Flows: j.results, LoadReport: loadReport} + } + + data, err := json.MarshalIndent(payload, "", " ") if err != nil { return fmt.Errorf("serializing json report: %w", err) } @@ -322,6 +342,7 @@ type consoleReporter struct { mu sync.Mutex flows map[string]*consoleFlowState showOutput bool + loadReport *LoadReport } type consoleFlowState struct { @@ -458,7 +479,23 @@ func (c *consoleReporter) HandleFlowResult(result model.FlowRunResult) { fmt.Printf("Flow Duration: %v | Steps: %d/%d Successful\n", result.Duration, state.successCount, state.totalNodes) } +func (c *consoleReporter) SetLoadReport(report *LoadReport) { + c.mu.Lock() + defer c.mu.Unlock() + c.loadReport = report +} + func (c *consoleReporter) Flush() error { + c.mu.Lock() + report := c.loadReport + c.mu.Unlock() + + if report == nil { + return nil + } + + fmt.Print(FormatLoadHeader(report.Meta)) + fmt.Print(FormatLoadTable(*report)) return nil } From dc29289b8a312f6cf26bf69edc02736a1fd493b6 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 20:38:54 +0300 Subject: [PATCH 25/38] feat(cli): --scenario and --vus/--duration/--iterations load flags Adds load mode to `flow run`. --scenario picks an entry of the file's load: block; --vus with --duration and/or --iterations describes a profile inline. The two forms are mutually exclusive, since a scenario already carries a complete profile and combining them would silently discard one. Load mode is decided from which flags the user passed rather than from their values, so `--vus 0` is a load run with a bad profile - and says so - rather than a silent fall-through to a functional run. A load run drives exactly one flow: the scenario's, the positional argument, or the file's only flow. Anything ambiguous names the candidates instead of guessing. Exit codes follow the load-testing convention: a run that completed is a success even with failing requests, since gating on error rates is what thresholds will do. Only a run that could not happen exits non-zero. Without any load flag the command behaves exactly as before, verified by diffing console and JSON output against a binary built from the base commit. --- apps/cli/cmd/flow.go | 58 ++++++- apps/cli/cmd/flow_load.go | 65 +++++++ apps/cli/internal/loadrun/resolve.go | 147 ++++++++++++++++ apps/cli/internal/loadrun/resolve_test.go | 201 ++++++++++++++++++++++ 4 files changed, 468 insertions(+), 3 deletions(-) create mode 100644 apps/cli/cmd/flow_load.go create mode 100644 apps/cli/internal/loadrun/resolve.go create mode 100644 apps/cli/internal/loadrun/resolve_test.go diff --git a/apps/cli/cmd/flow.go b/apps/cli/cmd/flow.go index 400e09db6..7f41feab0 100644 --- a/apps/cli/cmd/flow.go +++ b/apps/cli/cmd/flow.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/the-dev-tools/dev-tools/apps/cli/internal/common" + "github.com/the-dev-tools/dev-tools/apps/cli/internal/loadrun" "github.com/the-dev-tools/dev-tools/apps/cli/internal/reporter" "github.com/the-dev-tools/dev-tools/apps/cli/internal/runner" "github.com/the-dev-tools/dev-tools/packages/db/pkg/sqlitemem" @@ -31,6 +32,7 @@ import ( var ( quietMode bool showOutput bool + loadOpts loadrun.Options ) func init() { @@ -40,6 +42,21 @@ func init() { yamlflowRunCmd.Flags().StringSliceVar(&reportFormats, "report", []string{"console"}, "Report outputs to produce (format[:path]). Supported formats: console, json, junit.") yamlflowRunCmd.Flags().BoolVarP(&quietMode, "quiet", "q", false, "Suppress non-essential output for CI/CD usage") yamlflowRunCmd.Flags().BoolVar(&showOutput, "show-output", false, "Show node output data (including AI metrics) after each node completes") + + yamlflowRunCmd.Flags().StringVar(&loadOpts.Scenario, "scenario", "", + "Run the named entry of the file's load: block as a load test") + yamlflowRunCmd.Flags().IntVar(&loadOpts.VUs, "vus", 0, + "Run a load test with this many concurrent virtual users (requires --duration and/or --iterations)") + yamlflowRunCmd.Flags().DurationVar(&loadOpts.Duration, "duration", 0, + "How long a load test keeps starting new iterations, e.g. 60s") + yamlflowRunCmd.Flags().Int64Var(&loadOpts.Iterations, "iterations", 0, + "Total iterations a load test runs across all virtual users") + + // A scenario already carries a complete profile, so combining it with the + // inline profile flags would silently discard one of the two. + yamlflowRunCmd.MarkFlagsMutuallyExclusive("scenario", "vus") + yamlflowRunCmd.MarkFlagsMutuallyExclusive("scenario", "duration") + yamlflowRunCmd.MarkFlagsMutuallyExclusive("scenario", "iterations") } var flowCmd = &cobra.Command{ @@ -54,11 +71,39 @@ var flowCmd = &cobra.Command{ var yamlflowRunCmd = &cobra.Command{ Use: "run [yamlflow-file] [flow-name]", Short: "Run flow from yamlflow file", - Long: `Running Flow from a yamlflow format file. If flow-name is not provided, executes all flows from the 'run' field in order.`, - Args: cobra.RangeArgs(1, 2), + Long: `Running Flow from a yamlflow format file. If flow-name is not provided, executes all flows from the 'run' field in order. + +Load mode + --scenario runs an entry of the file's load: block; --vus with + --duration and/or --iterations describes a profile inline. The two are + mutually exclusive, since a scenario already carries a complete profile. A + load run drives exactly one flow and reports aggregate latency percentiles, + throughput and error rate instead of a per-step table. + + A load run that completes exits 0 even when requests inside it failed; + thresholds that turn an error rate into a failing exit code arrive in a + later release. Only a run that could not happen - an unknown scenario, an + unusable profile, or a target that was never reachable - exits non-zero. + + Only HTTP request steps are measured. GraphQL, WebSocket and sub-flow steps + still execute, but they are neither counted in the report nor covered by + the lean execution mode that keeps memory flat, so a flow built from them + can grow its memory use over a long run.`, + Args: cobra.RangeArgs(1, 2), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() + // A load flag that was passed with a zero value (--vus 0) is still a + // request for load mode, so ask cobra what was set rather than + // inferring it from the values. + loadOpts.Requested = false + for _, name := range []string{"scenario", "vus", "duration", "iterations"} { + if cmd.Flags().Changed(name) { + loadOpts.Requested = true + break + } + } + var logLevel slog.Level logLevelStr := os.Getenv("LOG_LEVEL") switch logLevelStr { @@ -104,7 +149,10 @@ var yamlflowRunCmd = &cobra.Command{ } } - if !runMultiple { + // A load run picks its own single flow (from --scenario, from the + // positional argument, or from a file with exactly one flow), so + // it does not need a run: block. + if !runMultiple && !loadOpts.Enabled() { return fmt.Errorf("no flow name provided and no run field found in workflow file") } } @@ -299,6 +347,10 @@ var yamlflowRunCmd = &cobra.Command{ JSClient: jsClient, } + if loadOpts.Enabled() { + return runLoad(ctx, loadOpts, resolved.LoadScenarios, flows, flowName, runnerServices, logger, reporters) + } + var runErr error if runMultiple { // Execute multiple flows based on run field diff --git a/apps/cli/cmd/flow_load.go b/apps/cli/cmd/flow_load.go new file mode 100644 index 000000000..c84b142ac --- /dev/null +++ b/apps/cli/cmd/flow_load.go @@ -0,0 +1,65 @@ +package cmd + +import ( + "context" + "log" + "log/slog" + + "github.com/the-dev-tools/dev-tools/apps/cli/internal/loadrun" + "github.com/the-dev-tools/dev-tools/apps/cli/internal/reporter" + "github.com/the-dev-tools/dev-tools/apps/cli/internal/runner" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mflow" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mload" +) + +// runLoad executes the workflow file as a load test instead of a functional +// run. +// +// Exit codes follow the load-testing convention rather than the functional +// one: a run that completed is a success even if requests inside it failed, +// because deciding whether an error rate is acceptable is what thresholds are +// for (Phase 2). Only a run that could not happen - a bad scenario name, an +// unusable profile, or a target that was never reachable - is an error, and +// therefore a non-zero exit. +func runLoad( + ctx context.Context, + opts loadrun.Options, + scenarios []mload.Scenario, + flows []mflow.Flow, + flowNameArg string, + services runner.RunnerServices, + logger *slog.Logger, + reporters *reporter.ReporterGroup, +) error { + cfg, err := loadrun.ResolveConfig(opts, scenarios, flows, flowNameArg) + if err != nil { + return err + } + + if !quietMode { + log.Printf("Load run: flow %q with %d VUs", cfg.Flow.Name, cfg.VUs) + } + + result, err := loadrun.Run(ctx, cfg, services, logger) + if err != nil { + return err + } + + reporters.SetLoadReport(&reporter.LoadReport{ + Meta: reporter.LoadRunMeta{ + ScenarioName: result.Config.ScenarioName, + FlowName: cfg.Flow.Name, + VUs: result.Config.VUs, + Duration: result.Config.Duration, + MaxIterations: result.Config.MaxIterations, + Iterations: result.Summary.Iterations, + Errors: result.Summary.Errors, + Elapsed: result.Summary.Elapsed, + WorkerVersion: version, + }, + Report: result.Report, + ByStep: result.ByStep, + }) + + return reporters.Flush() +} diff --git a/apps/cli/internal/loadrun/resolve.go b/apps/cli/internal/loadrun/resolve.go new file mode 100644 index 000000000..77a43e50b --- /dev/null +++ b/apps/cli/internal/loadrun/resolve.go @@ -0,0 +1,147 @@ +package loadrun + +import ( + "fmt" + "strings" + "time" + + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mflow" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mload" +) + +// Options is the load-mode command line, before it has been reconciled with +// the workflow file. +type Options struct { + // Requested is set by the command layer when the user passed any load + // flag at all - including one whose value happens to be a zero, like + // `--vus 0` or `--duration 30s` with no --vus. Those are load runs the + // user got wrong, and they deserve a load-mode error rather than a + // silent fall-through to a functional run. + Requested bool + // Scenario names an entry of the file's `load:` block. + Scenario string + // VUs, Duration and Iterations describe a profile inline, for runs that + // do not want a scenario in the file. + VUs int + Duration time.Duration + Iterations int64 +} + +// Enabled reports whether the user asked for a load run at all. Everything +// else about the invocation behaves exactly as it did before load mode +// existed when this is false. +// +// Values are honoured as well as Requested, so Options assembled without a +// command line - in a test, or by a future caller - still work. +func (o Options) Enabled() bool { + return o.Requested || o.Scenario != "" || o.VUs > 0 +} + +// ResolveConfig turns the command line plus the workflow file into a runnable +// profile. +// +// flowNameArg is the optional positional flow argument. It is only consulted +// for flag-driven runs: a scenario already names its flow, and --scenario is +// mutually exclusive with the profile flags. +// +// The returned Config points into flows, so callers keep the identity of the +// flow they passed in. +func ResolveConfig(opts Options, scenarios []mload.Scenario, flows []mflow.Flow, flowNameArg string) (Config, error) { + if opts.Scenario != "" { + return resolveScenarioConfig(opts.Scenario, scenarios, flows) + } + return resolveFlagConfig(opts, flows, flowNameArg) +} + +func resolveScenarioConfig(name string, scenarios []mload.Scenario, flows []mflow.Flow) (Config, error) { + if len(scenarios) == 0 { + return Config{}, fmt.Errorf( + "unknown load scenario %q: this workflow file has no load: block", name) + } + + names := make([]string, 0, len(scenarios)) + for _, s := range scenarios { + names = append(names, s.Name) + } + + for _, scenario := range scenarios { + if scenario.Name != name { + continue + } + flow := findFlow(flows, scenario.FlowName) + if flow == nil { + return Config{}, fmt.Errorf( + "load scenario %q targets flow %q, which is not in this workflow file (flows: %s)", + name, scenario.FlowName, flowNames(flows)) + } + return ConfigFromScenario(scenario, flow), nil + } + + return Config{}, fmt.Errorf( + "unknown load scenario %q (scenarios in this file: %s)", name, strings.Join(names, ", ")) +} + +func resolveFlagConfig(opts Options, flows []mflow.Flow, flowNameArg string) (Config, error) { + if opts.VUs == 0 { + return Config{}, fmt.Errorf( + "a load run needs virtual users: pass --vus N, or --scenario NAME to run an entry of the file's load: block") + } + if opts.VUs < 0 { + return Config{}, fmt.Errorf("--vus must be >= 1, got %d", opts.VUs) + } + if opts.Duration <= 0 && opts.Iterations <= 0 { + return Config{}, fmt.Errorf( + "a load run needs a stop condition: pass --duration (e.g. --duration 60s), --iterations, or both") + } + if opts.Iterations < 0 { + return Config{}, fmt.Errorf("--iterations must be >= 0, got %d", opts.Iterations) + } + + flow, err := selectFlow(flows, flowNameArg) + if err != nil { + return Config{}, err + } + + return Config{ + Flow: flow, + VUs: opts.VUs, + Duration: opts.Duration, + MaxIterations: opts.Iterations, + }, nil +} + +// selectFlow picks the flow a flag-driven load run should drive. A load run +// drives exactly one flow, so an ambiguous file is an error rather than a +// guess. +func selectFlow(flows []mflow.Flow, flowNameArg string) (*mflow.Flow, error) { + if flowNameArg != "" { + flow := findFlow(flows, flowNameArg) + if flow == nil { + return nil, fmt.Errorf("flow %q is not in this workflow file (flows: %s)", flowNameArg, flowNames(flows)) + } + return flow, nil + } + if len(flows) == 1 { + return &flows[0], nil + } + return nil, fmt.Errorf( + "a load run drives one flow, but this file has %d: name one as an argument, or use --scenario to run a load: block entry (flows: %s)", + len(flows), flowNames(flows)) +} + +func findFlow(flows []mflow.Flow, name string) *mflow.Flow { + for i := range flows { + if flows[i].Name == name { + return &flows[i] + } + } + return nil +} + +func flowNames(flows []mflow.Flow) string { + names := make([]string, 0, len(flows)) + for _, f := range flows { + names = append(names, f.Name) + } + return strings.Join(names, ", ") +} diff --git a/apps/cli/internal/loadrun/resolve_test.go b/apps/cli/internal/loadrun/resolve_test.go new file mode 100644 index 000000000..f1b8566d1 --- /dev/null +++ b/apps/cli/internal/loadrun/resolve_test.go @@ -0,0 +1,201 @@ +package loadrun + +import ( + "strings" + "testing" + "time" + + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mflow" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mload" +) + +func testFlows() []mflow.Flow { + return []mflow.Flow{{Name: "Checkout"}, {Name: "Browse"}} +} + +func testScenarios() []mload.Scenario { + return []mload.Scenario{ + {Name: "checkout-baseline", FlowName: "Checkout", Executor: mload.ExecutorConstantVUs, VUs: 10, Duration: 30 * time.Second}, + {Name: "browse-smoke", FlowName: "Browse", Executor: mload.ExecutorConstantVUs, VUs: 1, MaxIterations: 25}, + } +} + +func TestOptionsEnabled(t *testing.T) { + cases := []struct { + name string + opts Options + want bool + }{ + {"nothing set", Options{}, false}, + {"scenario", Options{Scenario: "x"}, true}, + {"vus", Options{VUs: 4}, true}, + {"duration alone does not enable by value", Options{Duration: time.Second}, false}, + {"iterations alone does not enable by value", Options{Iterations: 10}, false}, + {"explicitly requested with no values", Options{Requested: true}, true}, + {"explicit zero vus is still a load run", Options{Requested: true, VUs: 0}, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := tc.opts.Enabled(); got != tc.want { + t.Errorf("Enabled() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestResolveConfigFromScenario(t *testing.T) { + flows := testFlows() + cfg, err := ResolveConfig(Options{Scenario: "checkout-baseline"}, testScenarios(), flows, "") + if err != nil { + t.Fatalf("ResolveConfig failed: %v", err) + } + if cfg.ScenarioName != "checkout-baseline" { + t.Errorf("ScenarioName = %q", cfg.ScenarioName) + } + if cfg.Flow == nil || cfg.Flow.Name != "Checkout" { + t.Fatalf("Flow = %+v, want the Checkout flow", cfg.Flow) + } + if cfg.Flow != &flows[0] { + t.Error("expected the resolved flow to point at the caller's slice entry") + } + if cfg.VUs != 10 || cfg.Duration != 30*time.Second { + t.Errorf("profile not carried through: %+v", cfg) + } +} + +func TestResolveConfigUnknownScenario(t *testing.T) { + _, err := ResolveConfig(Options{Scenario: "nope"}, testScenarios(), testFlows(), "") + if err == nil { + t.Fatal("expected an error for an unknown scenario") + } + for _, want := range []string{"nope", "checkout-baseline", "browse-smoke"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } +} + +func TestResolveConfigScenarioWithoutLoadBlock(t *testing.T) { + _, err := ResolveConfig(Options{Scenario: "anything"}, nil, testFlows(), "") + if err == nil { + t.Fatal("expected an error when the file has no load block") + } + for _, want := range []string{"anything", "load:"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } +} + +func TestResolveConfigScenarioNamesMissingFlow(t *testing.T) { + scenarios := []mload.Scenario{{Name: "orphan", FlowName: "Vanished", VUs: 1, MaxIterations: 1}} + _, err := ResolveConfig(Options{Scenario: "orphan"}, scenarios, testFlows(), "") + if err == nil { + t.Fatal("expected an error when the scenario's flow is not present") + } + for _, want := range []string{"orphan", "Vanished", "Checkout", "Browse"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } +} + +func TestResolveConfigFromFlags(t *testing.T) { + cfg, err := ResolveConfig( + Options{VUs: 5, Duration: 90 * time.Second, Iterations: 200}, + nil, testFlows(), "Browse") + if err != nil { + t.Fatalf("ResolveConfig failed: %v", err) + } + if cfg.ScenarioName != "" { + t.Errorf("ScenarioName = %q, want empty for a flag-driven run", cfg.ScenarioName) + } + if cfg.Flow == nil || cfg.Flow.Name != "Browse" { + t.Fatalf("Flow = %+v", cfg.Flow) + } + if cfg.VUs != 5 || cfg.Duration != 90*time.Second || cfg.MaxIterations != 200 { + t.Errorf("profile not carried through: %+v", cfg) + } +} + +func TestResolveConfigFromFlagsPicksTheOnlyFlow(t *testing.T) { + flows := []mflow.Flow{{Name: "Solo"}} + cfg, err := ResolveConfig(Options{VUs: 2, Iterations: 4}, nil, flows, "") + if err != nil { + t.Fatalf("ResolveConfig failed: %v", err) + } + if cfg.Flow == nil || cfg.Flow.Name != "Solo" { + t.Fatalf("Flow = %+v, want the single flow", cfg.Flow) + } +} + +func TestResolveConfigFromFlagsAmbiguousFlow(t *testing.T) { + _, err := ResolveConfig(Options{VUs: 2, Iterations: 4}, nil, testFlows(), "") + if err == nil { + t.Fatal("expected an error when the flow to load-test is ambiguous") + } + for _, want := range []string{"Checkout", "Browse", "--scenario"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } +} + +func TestResolveConfigFromFlagsUnknownFlow(t *testing.T) { + _, err := ResolveConfig(Options{VUs: 2, Iterations: 4}, nil, testFlows(), "Ghost") + if err == nil { + t.Fatal("expected an error for an unknown flow name") + } + for _, want := range []string{"Ghost", "Checkout", "Browse"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } +} + +func TestResolveConfigFromFlagsNeedsStopCondition(t *testing.T) { + _, err := ResolveConfig(Options{VUs: 2}, nil, testFlows(), "Checkout") + if err == nil { + t.Fatal("expected an error when neither duration nor iterations is set") + } + for _, want := range []string{"--duration", "--iterations"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } +} + +func TestResolveConfigRejectsNegativeVUs(t *testing.T) { + _, err := ResolveConfig(Options{VUs: -1, Iterations: 1}, nil, testFlows(), "Checkout") + if err == nil { + t.Fatal("expected an error for negative vus") + } + if !strings.Contains(err.Error(), "--vus") { + t.Errorf("error %q does not mention --vus", err) + } +} + +// TestResolveConfigMissingVUs covers the shapes that reach load mode without +// any virtual users: an explicit --vus 0, and --duration/--iterations passed +// on their own. Both must say what is missing rather than fall back to a +// functional run. +func TestResolveConfigMissingVUs(t *testing.T) { + cases := map[string]Options{ + "explicit zero": {Requested: true, VUs: 0, Iterations: 4}, + "duration only": {Requested: true, Duration: 30 * time.Second}, + "iterations only": {Requested: true, Iterations: 100}, + } + for name, opts := range cases { + t.Run(name, func(t *testing.T) { + _, err := ResolveConfig(opts, nil, testFlows(), "Checkout") + if err == nil { + t.Fatal("expected an error when no virtual users were requested") + } + for _, want := range []string{"--vus", "--scenario"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } + }) + } +} From 20aae1bc7474899493608e42df2b523fd50c12f0 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 20:39:29 +0300 Subject: [PATCH 26/38] test(server): add server:test:race for the concurrency-critical packages scenariorunner and loadmetrics are the two packages whose whole job is to be correct under concurrency, and the plain suite runs without -race. This gives them a dedicated target so the race detector is a gate rather than something someone remembers to run. Scoped to those two packages and -count=3 to stay fast enough to run routinely. Wiring it into CI is separate. --- packages/server/project.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/server/project.json b/packages/server/project.json index 6c530e904..30a0f4a1b 100644 --- a/packages/server/project.json +++ b/packages/server/project.json @@ -47,6 +47,17 @@ "command": "go test -p 8 ./... -timeout 30s" } }, + "test:race": { + "dependsOn": ["db:generate", "spec:build"], + "executor": "nx:run-commands", + "options": { + "cwd": "{projectRoot}", + "command": "go test -race -count=3 ./pkg/flow/runner/scenariorunner/ ./pkg/loadmetrics/ -timeout 120s", + "env": { + "CGO_ENABLED": "1" + } + } + }, "test:ci": { "dependsOn": ["db:generate-ci", "spec:build"], "executor": "nx:run-commands", From 3b86d9b87f863a7c203f3ccc7f8e74b35b196d1a Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 20:46:29 +0300 Subject: [PATCH 27/38] fix(cli): start load metrics intervals at the scenario, not at setup An aggregator's interval begins when it is constructed, and construction happens while VUs are being built. Left alone, the wall time the report divides by includes setup, so RPS reads low by however long building the node graphs took - which grows with the VU count, exactly when the number matters. Flushing the empty setup frame away restarts every interval together at the instant the scenario starts. The new test also pins the surrounding contract: RPS comes from real elapsed time, never from the aggregator's nominal flush interval. --- apps/cli/internal/loadrun/loadrun.go | 16 ++++++++ apps/cli/internal/loadrun/loadrun_test.go | 49 +++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/apps/cli/internal/loadrun/loadrun.go b/apps/cli/internal/loadrun/loadrun.go index 554776e25..af04fb473 100644 --- a/apps/cli/internal/loadrun/loadrun.go +++ b/apps/cli/internal/loadrun/loadrun.go @@ -139,6 +139,16 @@ func Run(ctx context.Context, cfg Config, services runner.RunnerServices, logger tracker := newFirstIterationTracker(cfg.VUs) + // An aggregator's interval starts when it is constructed, which was + // during setup. Flushing the empty setup frame away restarts every + // interval at the same instant the scenario does, so the wall time the + // report divides by is the scenario's, not the scenario's plus however + // long building VUs took. + startedAt := time.Now() + for _, w := range workers { + w.agg.Flush(startedAt) + } + // Duration is passed through RunProfile only. Deriving it from a context // deadline instead would make scenariorunner.Run return ctx.Err() at the // end of every successful timed run, since it reports the caller's @@ -488,6 +498,12 @@ func (w *vuWorker) resetBytes() { // record aggregates one terminal node status. Only HTTP request nodes are // counted: they are the ones lean mode covers, and the ones whose latency the // report is about. +// +// The latency recorded is the node's run duration, not the bare HTTP lap +// time. It is slightly wider - it includes building the request, evaluating +// assertions and handing the response to the side-channel drain - but it has +// nanosecond resolution, whereas the lap time reaches node output rounded to +// whole milliseconds, which cannot describe a fast local target at all. func (w *vuWorker) record(status flowrunner.FlowNodeStatus) { if status.State == mflow.NODE_STATE_RUNNING { return diff --git a/apps/cli/internal/loadrun/loadrun_test.go b/apps/cli/internal/loadrun/loadrun_test.go index c8af5d9d8..ea75321a9 100644 --- a/apps/cli/internal/loadrun/loadrun_test.go +++ b/apps/cli/internal/loadrun/loadrun_test.go @@ -415,6 +415,55 @@ func TestRunDurationBoundedRunSucceeds(t *testing.T) { } } +// TestRunRPSUsesRealElapsedTime pins contract addition #2: the aggregator's +// constructor interval is documentation, not arithmetic. RPS must come from +// the wall time the run actually covered. +// +// The run is far shorter than the nominal flush interval, so a report that +// divided by the nominal interval would understate RPS by more than an order +// of magnitude - which is what the first assertion catches without depending +// on machine speed. +func TestRunRPSUsesRealElapsedTime(t *testing.T) { + if testing.Short() { + t.Skip("skipping load run in short mode") + } + + srv := newCountingServer(t, time.Millisecond, http.StatusOK) + flow, services := setupFlow(t, twoStepFlowYAML(srv.URL), "LoadFlow") + + result, err := Run(t.Context(), Config{ + Flow: flow, + VUs: 2, + Duration: 200 * time.Millisecond, + }, services, nil) + if err != nil { + t.Fatalf("Run failed: %v", err) + } + + total := result.Report.Total + if total.Count == 0 { + t.Fatal("no requests recorded") + } + + nominalRPS := float64(total.Count) / aggregatorFlushInterval.Seconds() + if total.RPS < nominalRPS*2 { + t.Errorf("RPS %.1f looks like count/%v (%.1f), not count/elapsed", + total.RPS, aggregatorFlushInterval, nominalRPS) + } + + // The wall time the report implies must be the scenario's, not the + // scenario's plus setup. + impliedElapsed := time.Duration(float64(total.Count) / total.RPS * float64(time.Second)) + drift := impliedElapsed - result.Summary.Elapsed + if drift < 0 { + drift = -drift + } + if drift > 50*time.Millisecond { + t.Errorf("report implies %v of wall time, scheduler measured %v (drift %v)", + impliedElapsed, result.Summary.Elapsed, drift) + } +} + // TestRunSetupFailureWhenEveryVUFailsFirstIteration pins contract addition // #9's infra-failure case: an unreachable target on every VU's first // iteration is a setup failure (exit 1), not a completed run with errors. From b21eec822497ead85efdc1db78fff57820cf4ae0 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 21:11:00 +0300 Subject: [PATCH 28/38] test(cli): compare full node maps between VUs instead of sampling one The isolation assertion sampled a single node per VU through a randomly ordered map range, so a build-once-and-share regression only tripped it when the samples happened to differ. Under a shared-graph mutation it caught the regression 92 times out of 100 - and that rate falls as flows grow, since more nodes make distinct samples more likely, so it degraded exactly as fixtures got realistic. Now every node is compared across every pair of VUs over a sorted key list. The same mutation is caught 100 times out of 100, and the failure names the node and the pair. --- apps/cli/internal/loadrun/loadrun_test.go | 82 ++++++++++++++++++++--- 1 file changed, 73 insertions(+), 9 deletions(-) diff --git a/apps/cli/internal/loadrun/loadrun_test.go b/apps/cli/internal/loadrun/loadrun_test.go index ea75321a9..648fff035 100644 --- a/apps/cli/internal/loadrun/loadrun_test.go +++ b/apps/cli/internal/loadrun/loadrun_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "os" + "sort" "strings" "sync" "sync/atomic" @@ -478,10 +479,11 @@ func TestRunSetupFailureWhenEveryVUFailsFirstIteration(t *testing.T) { flow, services := setupFlow(t, twoStepFlowYAML(baseURL), "LoadFlow") - _, err := Run(t.Context(), Config{ + const iterations = 4 + result, err := Run(t.Context(), Config{ Flow: flow, VUs: 2, - MaxIterations: 4, + MaxIterations: iterations, }, services, nil) if err == nil { t.Fatal("expected an unreachable target to be reported as a setup failure") @@ -489,6 +491,44 @@ func TestRunSetupFailureWhenEveryVUFailsFirstIteration(t *testing.T) { if !strings.Contains(err.Error(), "first iteration") { t.Errorf("error %q does not explain that every VU failed its first iteration", err) } + + // The run still has to hand back what it measured: a failing run's numbers + // are the ones someone actually needs. + if !result.Ran() { + t.Fatal("Result.Ran() = false for a scenario that executed") + } + if result.Summary.Iterations != iterations { + t.Errorf("Summary.Iterations = %d, want %d", result.Summary.Iterations, iterations) + } + if result.Summary.Errors != iterations { + t.Errorf("Summary.Errors = %d, want %d", result.Summary.Errors, iterations) + } + if result.Report.Total.Count == 0 { + t.Fatalf("failed run reported no requests; keys: %v", reportKeys(result.Report)) + } + if result.Report.Total.ErrorCount != result.Report.Total.Count { + t.Errorf("ErrorCount %d != Count %d for an all-failing run", + result.Report.Total.ErrorCount, result.Report.Total.Count) + } + key := loadmetrics.Key{Step: "StepOne", StatusClass: loadmetrics.StatusClassError} + if _, ok := result.Report.PerStep[key]; !ok { + t.Errorf("missing %+v in a failed run; got keys %v", key, reportKeys(result.Report)) + } + if _, ok := result.ByStep.PerStep[loadmetrics.Key{Step: "StepOne"}]; !ok { + t.Errorf("folded report missing StepOne; got keys %v", reportKeys(result.ByStep)) + } +} + +// TestRunConfigFailureHasNothingToReport is Ran()'s other side: a run that +// never started must not offer a report for the caller to print. +func TestRunConfigFailureHasNothingToReport(t *testing.T) { + result, err := Run(context.Background(), Config{VUs: 1, MaxIterations: 1}, runner.RunnerServices{}, nil) + if err == nil { + t.Fatal("expected a validation error") + } + if result.Ran() { + t.Error("Result.Ran() = true for a run that never started") + } } // TestRunErrorsDoNotFailACompletedRun is #9's other half: once at least one @@ -604,14 +644,9 @@ func TestVUWorkersAreIsolated(t *testing.T) { seenClients := make(map[any]bool, vus) seenAggs := make(map[*loadmetrics.Aggregator]bool, vus) - seenNodeMaps := make(map[any]bool, vus) for _, w := range workers { seenClients[w.httpClient] = true seenAggs[w.agg] = true - for id := range w.flowNodeMap { - seenNodeMaps[w.flowNodeMap[id]] = true - break - } } if len(seenClients) != vus { t.Errorf("%d distinct HTTP clients across %d VUs, want %d", len(seenClients), vus, vus) @@ -619,7 +654,36 @@ func TestVUWorkersAreIsolated(t *testing.T) { if len(seenAggs) != vus { t.Errorf("%d distinct aggregators across %d VUs, want %d", len(seenAggs), vus, vus) } - if len(seenNodeMaps) != vus { - t.Errorf("%d distinct node instances across %d VUs, want %d", len(seenNodeMaps), vus, vus) + + // Every node, in every pair of VUs, must be a distinct instance. Sampling + // one node per VU would not do: a shared graph still yields distinct + // samples whenever the (randomly ordered) samples happen to differ, which + // is most of the time. + nodeIDs := make([]idwrap.IDWrap, 0, len(workers[0].flowNodeMap)) + for id := range workers[0].flowNodeMap { + nodeIDs = append(nodeIDs, id) + } + sort.Slice(nodeIDs, func(i, j int) bool { return nodeIDs[i].Compare(nodeIDs[j]) < 0 }) + if len(nodeIDs) == 0 { + t.Fatal("worker 0 built an empty node map") + } + + for i := range workers { + if len(workers[i].flowNodeMap) != len(nodeIDs) { + t.Fatalf("VU %d has %d nodes, VU 0 has %d", i, len(workers[i].flowNodeMap), len(nodeIDs)) + } + for j := i + 1; j < len(workers); j++ { + for _, id := range nodeIDs { + a, okA := workers[i].flowNodeMap[id] + b, okB := workers[j].flowNodeMap[id] + if !okA || !okB { + t.Fatalf("node %s missing from VU %d (%t) or VU %d (%t)", id, i, okA, j, okB) + } + if a == b { + t.Errorf("VUs %d and %d share the same instance of node %q - the node graph was built once instead of per VU", + i, j, a.GetName()) + } + } + } } } From 1e8356a5fda7a24c5f65b4687ce092904c7078ac Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 21:11:08 +0300 Subject: [PATCH 29/38] fix(cli): report a load run that failed instead of discarding it A soak whose first iteration per VU hit a cold target and then ran cleanly for half an hour exited 1 with no table and no JSON. The exit code was right; throwing the measurements away was not, because a failed run's numbers are the ones someone actually needs. Run now assembles the report before deciding the error and returns both, on the setup-failure path and the cancellation path alike. Result.Ran() tells a caller whether a report exists at all - false only when the run never started, so a bad profile still prints nothing but its error. The CLI renders the table and writes the JSON whenever a run executed, then exits non-zero on the failure. --- apps/cli/cmd/flow_load.go | 47 ++++++++++++++++------------ apps/cli/internal/loadrun/loadrun.go | 39 ++++++++++++++++------- 2 files changed, 55 insertions(+), 31 deletions(-) diff --git a/apps/cli/cmd/flow_load.go b/apps/cli/cmd/flow_load.go index c84b142ac..e7f8b47bc 100644 --- a/apps/cli/cmd/flow_load.go +++ b/apps/cli/cmd/flow_load.go @@ -40,26 +40,33 @@ func runLoad( log.Printf("Load run: flow %q with %d VUs", cfg.Flow.Name, cfg.VUs) } - result, err := loadrun.Run(ctx, cfg, services, logger) - if err != nil { - return err - } + result, runErr := loadrun.Run(ctx, cfg, services, logger) - reporters.SetLoadReport(&reporter.LoadReport{ - Meta: reporter.LoadRunMeta{ - ScenarioName: result.Config.ScenarioName, - FlowName: cfg.Flow.Name, - VUs: result.Config.VUs, - Duration: result.Config.Duration, - MaxIterations: result.Config.MaxIterations, - Iterations: result.Summary.Iterations, - Errors: result.Summary.Errors, - Elapsed: result.Summary.Elapsed, - WorkerVersion: version, - }, - Report: result.Report, - ByStep: result.ByStep, - }) + // A run that executed gets reported even when it also failed - the table + // and the JSON are how anyone works out what went wrong. The failure still + // decides the exit code, below. + var flushErr error + if result.Ran() { + reporters.SetLoadReport(&reporter.LoadReport{ + Meta: reporter.LoadRunMeta{ + ScenarioName: result.Config.ScenarioName, + FlowName: cfg.Flow.Name, + VUs: result.Config.VUs, + Duration: result.Config.Duration, + MaxIterations: result.Config.MaxIterations, + Iterations: result.Summary.Iterations, + Errors: result.Summary.Errors, + Elapsed: result.Summary.Elapsed, + WorkerVersion: version, + }, + Report: result.Report, + ByStep: result.ByStep, + }) + flushErr = reporters.Flush() + } - return reporters.Flush() + if runErr != nil { + return runErr + } + return flushErr } diff --git a/apps/cli/internal/loadrun/loadrun.go b/apps/cli/internal/loadrun/loadrun.go index af04fb473..16c984256 100644 --- a/apps/cli/internal/loadrun/loadrun.go +++ b/apps/cli/internal/loadrun/loadrun.go @@ -118,6 +118,19 @@ type Result struct { ByStep loadmetrics.Report } +// Ran reports whether the scenario got as far as executing, and therefore +// whether this Result is worth reporting. +// +// It is true even for runs that ended in an error, because those are exactly +// the runs whose numbers matter most: a soak that failed its first iteration +// per VU and then ran cleanly for half an hour still exits non-zero, but +// throwing its report away would be the worst possible response to it. It is +// false only when Run failed before any iteration could start - invalid +// configuration, or a flow graph that would not build. +func (r Result) Ran() bool { + return r.Config.Flow != nil +} + // Run executes cfg and returns the merged report. // // A completed run is a success even when individual requests failed: request @@ -153,7 +166,7 @@ func Run(ctx context.Context, cfg Config, services runner.RunnerServices, logger // deadline instead would make scenariorunner.Run return ctx.Err() at the // end of every successful timed run, since it reports the caller's // context state on the way out. - summary, err := scenariorunner.Run(ctx, scenariorunner.RunProfile{ + summary, runErr := scenariorunner.Run(ctx, scenariorunner.RunProfile{ VUs: cfg.VUs, Duration: cfg.Duration, MaxIterations: cfg.MaxIterations, @@ -162,26 +175,30 @@ func Run(ctx context.Context, cfg Config, services runner.RunnerServices, logger tracker.observe(vu, iterErr) return iterErr }) - if err != nil { - return Result{}, fmt.Errorf("load run: %w", err) - } - - if err := tracker.setupFailure(); err != nil { - return Result{}, err - } + // The report is assembled before any error is returned, and returned + // alongside it. Everything below this point describes a run that happened; + // discarding what it measured because it also ended badly would throw away + // precisely the numbers someone needs to understand why. flushedAt := time.Now() frames := make([]loadmetrics.Frame, 0, len(workers)) for _, w := range workers { frames = append(frames, w.agg.Flush(flushedAt)) } - - return Result{ + result := Result{ Config: cfg, Summary: summary, Report: loadmetrics.Merge(frames), ByStep: loadmetrics.Merge(foldByStep(frames)), - }, nil + } + + if runErr != nil { + return result, fmt.Errorf("load run: %w", runErr) + } + if err := tracker.setupFailure(); err != nil { + return result, err + } + return result, nil } // foldByStep rewrites frames so every entry's status class is dropped, From 2d8533fb57d0497528688013ee792315ec5a6614 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 21:11:14 +0300 Subject: [PATCH 30/38] fix(ioworkspace): warn when an import drops load scenarios LoadScenarios rides on WorkspaceBundle so the YAML round trip keeps the load: block, but there is no schema behind it. Importing such a file into a workspace therefore loses it, and a later export will not bring it back - silently, which is the wrong way to lose data. Import now says so at warn level, naming the count and the scenarios. Storage stays out of scope; this only makes the gap visible until it exists. --- packages/server/pkg/ioworkspace/importer.go | 36 ++++++ .../pkg/ioworkspace/importer_load_test.go | 107 ++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 packages/server/pkg/ioworkspace/importer_load_test.go diff --git a/packages/server/pkg/ioworkspace/importer.go b/packages/server/pkg/ioworkspace/importer.go index d8778ae68..4894e5cfe 100644 --- a/packages/server/pkg/ioworkspace/importer.go +++ b/packages/server/pkg/ioworkspace/importer.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" "github.com/the-dev-tools/dev-tools/packages/server/pkg/service/senv" @@ -63,12 +64,18 @@ type ImportResult struct { // Import imports a WorkspaceBundle into the database using the provided options. // This operation should be performed within a transaction for atomicity. +// +// Load scenarios are the one part of a bundle Import does not store: there is +// no schema for them yet. Rather than lose them quietly, an import that +// carries any says so - see warnUnstoredLoadScenarios. func (s *IOWorkspaceService) Import(ctx context.Context, tx *sql.Tx, bundle *WorkspaceBundle, opts ImportOptions) (*ImportResult, error) { // Validate options if err := opts.Validate(); err != nil { return nil, fmt.Errorf("invalid import options: %w", err) } + s.warnUnstoredLoadScenarios(ctx, bundle) + // Initialize result result := &ImportResult{ HTTPIDMap: make(map[idwrap.IDWrap]idwrap.IDWrap), @@ -338,4 +345,33 @@ func (s *IOWorkspaceService) Import(ctx context.Context, tx *sql.Tx, bundle *Wor return result, nil } +// LoadScenariosNotStoredMessage is logged when an imported bundle carries load +// scenarios. It is a constant so tests can assert on the warning without +// pinning the surrounding log format. +const LoadScenariosNotStoredMessage = "Load scenarios were not stored: this version keeps the load: block in the workflow file only, so exporting this workspace will not reproduce it" + +// warnUnstoredLoadScenarios reports load scenarios that this version cannot +// persist. +// +// WorkspaceBundle.LoadScenarios exists so the file-to-file YAML round trip +// preserves the load: block; there is no table behind it. Importing a document +// that has one therefore drops it, and an export of the resulting workspace +// will not bring it back. That is a real (if temporary) data loss, so it is +// stated out loud rather than left for someone to discover from a diff. +func (s *IOWorkspaceService) warnUnstoredLoadScenarios(ctx context.Context, bundle *WorkspaceBundle) { + if bundle == nil || len(bundle.LoadScenarios) == 0 { + return + } + + names := make([]string, 0, len(bundle.LoadScenarios)) + for _, scenario := range bundle.LoadScenarios { + names = append(names, scenario.Name) + } + + s.logger.WarnContext(ctx, LoadScenariosNotStoredMessage, + "count", len(bundle.LoadScenarios), + "scenarios", strings.Join(names, ", "), + ) +} + // Flow import functions have been moved to importer_flow.go diff --git a/packages/server/pkg/ioworkspace/importer_load_test.go b/packages/server/pkg/ioworkspace/importer_load_test.go new file mode 100644 index 000000000..2817a548a --- /dev/null +++ b/packages/server/pkg/ioworkspace/importer_load_test.go @@ -0,0 +1,107 @@ +package ioworkspace + +import ( + "bytes" + "context" + "log/slog" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/the-dev-tools/dev-tools/packages/db/pkg/sqlitemem" + "github.com/the-dev-tools/dev-tools/packages/db/pkg/sqlc/gen" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mflow" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mload" +) + +// importWithLogger runs a bundle through Import against a fresh in-memory +// database, capturing everything the service logged. +func importWithLogger(t *testing.T, bundle *WorkspaceBundle) string { + t.Helper() + + ctx := context.Background() + + db, _, err := sqlitemem.NewSQLiteMem(ctx) + require.NoError(t, err) + + queries := gen.New(db) + wsID := idwrap.NewNow() + require.NoError(t, queries.CreateWorkspace(ctx, gen.CreateWorkspaceParams{ + ID: wsID, + Name: "Load Warning WS", + Updated: 0, + })) + + var logs bytes.Buffer + logger := slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})) + + for i := range bundle.Flows { + bundle.Flows[i].WorkspaceID = wsID + } + + tx, err := db.BeginTx(ctx, nil) + require.NoError(t, err) + + _, err = New(queries, logger).Import(ctx, tx, bundle, ImportOptions{ + WorkspaceID: wsID, + PreserveIDs: true, + ImportFlows: true, + }) + require.NoError(t, err, "import must still succeed - load scenarios are dropped, not fatal") + require.NoError(t, tx.Commit()) + + return logs.String() +} + +func loadScenarioBundle(scenarios ...mload.Scenario) *WorkspaceBundle { + flowID := idwrap.NewNow() + return &WorkspaceBundle{ + Flows: []mflow.Flow{{ID: flowID, Name: "Checkout"}}, + LoadScenarios: scenarios, + } +} + +// TestImportWarnsThatLoadScenariosAreNotStored covers the one part of a bundle +// Import cannot persist. There is no schema for load scenarios yet, so a +// workspace imported from a file with a load: block will not export one back - +// which has to be said out loud rather than discovered from a diff. +func TestImportWarnsThatLoadScenariosAreNotStored(t *testing.T) { + logs := importWithLogger(t, loadScenarioBundle( + mload.Scenario{ + Name: "checkout-baseline", FlowName: "Checkout", + Executor: mload.ExecutorConstantVUs, VUs: 10, Duration: 30 * time.Second, + }, + mload.Scenario{ + Name: "browse-smoke", FlowName: "Checkout", + Executor: mload.ExecutorConstantVUs, VUs: 1, MaxIterations: 25, + }, + )) + + if !strings.Contains(logs, LoadScenariosNotStoredMessage) { + t.Errorf("import did not warn about unstored load scenarios; logs:\n%s", logs) + } + if !strings.Contains(logs, "count=2") { + t.Errorf("warning does not name how many scenarios were dropped; logs:\n%s", logs) + } + for _, name := range []string{"checkout-baseline", "browse-smoke"} { + if !strings.Contains(logs, name) { + t.Errorf("warning does not name scenario %q; logs:\n%s", name, logs) + } + } + if !strings.Contains(logs, "level=WARN") { + t.Errorf("expected the message at warn level; logs:\n%s", logs) + } +} + +// TestImportSilentWithoutLoadScenarios keeps the warning from becoming noise +// on the overwhelmingly common import that has no load: block at all. +func TestImportSilentWithoutLoadScenarios(t *testing.T) { + logs := importWithLogger(t, loadScenarioBundle()) + + if strings.Contains(logs, LoadScenariosNotStoredMessage) { + t.Errorf("warned about load scenarios on a bundle that has none; logs:\n%s", logs) + } +} From 911eef11b9ecdf69fe491e915497f6926984b2dc Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 22:30:29 +0300 Subject: [PATCH 31/38] test(cli): RPS/worker benchmark harness + Phase 0 results Measures what one load worker can actually generate, driving the real apps/cli/internal/loadrun engine (not a reimplementation of its scheduling) against a local, in-process httptest target with a fixed 5ms handler latency - no external network dependency of any kind. apps/cli/test/loadbench/loadbench_test.go: the target server (a /single route and a /chain route sharing one atomic token counter), the two flow fixtures, and fixture-correctness tests that run in the default suite (no guard). The chained-flow fixture chains for real - step N sends the exact token step N-1's response issued - which TestChainedFlowReallyChains and TestChainedFlowMultipleIterationsStillChain both pin. Chaining goes through a response header rather than the body: load mode always runs in lean mode, which replaces the whole decoded body with a fixed placeholder once assertions run, so a body-based chain would silently chain the placeholder instead of a real per-response value. setupFlow duplicates loadrun's own test helper (unexported, so it can't be imported) rather than inventing a third way to wire a flow - the load-mode docs already flag this pattern's duplication of cmd wiring as an accepted tradeoff. apps/cli/test/loadbench/integration_loadbench_test.go: the actual RPS/ percentile matrix across VUs x {1, 10, 50} x {single-get, chained-5-step}, gated by both a loadbench_integration build tag and RUN_LOADBENCH=true so it never runs by default. VUs=50 against this fast a target pushes throughput high enough (thousands/sec) that packages/server/pkg/httpclient.New()'s reliance on http.DefaultTransport's default MaxIdleConnsPerHost=2 exhausts this machine's ephemeral port range - confirmed independently of the flow engine with a raw net/http repro. Cells run VUs-major, each in its own t.Run (so its server and DB are torn down before the next cell starts), separated by a 35s cooldown longer than this machine's TIME_WAIT - otherwise one cell's port pressure bleeds into the next and inflates its error rate, which is exactly what an earlier, uninstrumented version of this harness got wrong (a VUs=50 cell reporting 98.5% errors that were mostly leftover pressure from the VUs=10 cell three seconds earlier). docs/superpowers/specs/phase0-bench.md: the results table, environment block, methodology, and the full anomaly writeup above - flagged throughout as dev-hardware-only numbers that gate spec section 3.5 capacity math and section 6 pricing as an upper bound, pending a Fly shared-cpu-machine run. --- .../loadbench/integration_loadbench_test.go | 260 +++++++++++ apps/cli/test/loadbench/loadbench_test.go | 432 ++++++++++++++++++ docs/superpowers/specs/phase0-bench.md | 244 ++++++++++ 3 files changed, 936 insertions(+) create mode 100644 apps/cli/test/loadbench/integration_loadbench_test.go create mode 100644 apps/cli/test/loadbench/loadbench_test.go create mode 100644 docs/superpowers/specs/phase0-bench.md diff --git a/apps/cli/test/loadbench/integration_loadbench_test.go b/apps/cli/test/loadbench/integration_loadbench_test.go new file mode 100644 index 000000000..40f5a7ad4 --- /dev/null +++ b/apps/cli/test/loadbench/integration_loadbench_test.go @@ -0,0 +1,260 @@ +//go:build loadbench_integration + +// This file is the actual RPS/percentile matrix: minutes of wall-clock time, +// and its numbers are only as good as the machine that produced them - which +// is exactly why it does not run by default. It is gated by both a build tag +// and an environment variable, following this repo's integration-test +// convention (see e.g. packages/server/pkg/flow/node/nai's +// integration_setup_test.go, which pairs //go:build ai_integration with +// RUN_AI_INTEGRATION_TESTS). +// +// Run it with: +// +// RUN_LOADBENCH=true go test -tags loadbench_integration \ +// -run TestLoadBenchMatrix ./apps/cli/test/loadbench/ -v -timeout 900s +// +// It prints a results table (and a per-step breakdown for the chained +// config) to the test log. Nothing in this file writes to +// docs/superpowers/specs/phase0-bench.md automatically - transcribe real +// numbers from the logged output by hand. That is deliberate: the table is +// proof of what ran, not a substitute for a human deciding the numbers are +// sane before they gate anything. +// +// # A note on VUs=50 and ephemeral ports +// +// At VUs=50 against this package's fast (5ms) local target, request +// throughput is high enough (thousands/sec) that packages/server/pkg/httpclient.New()'s +// reliance on http.DefaultTransport's default MaxIdleConnsPerHost=2 causes +// most connections to be closed and redialed rather than reused. On macOS +// that can exhaust the ephemeral port range (49152-65535, ~16k ports, 30s +// TIME_WAIT) within the measured window, surfacing as "dial tcp ...: connect: +// can't assign requested address" transport errors - confirmed with a raw +// net/http repro outside the flow engine entirely, see task-6-report.md. +// This is a real, reproducible property of the CLI's HTTP client under +// sustained high-throughput single-host load on this class of machine, not a +// bug in this harness. Two things keep one cell's port pressure from +// contaminating another's numbers: cells run VUs-major (all VUs=1, then all +// VUs=10, then both VUs=50 cells last) so a high-VU cell can never run +// before a lower one, each cell tears its server and DB down (via t.Run's +// own Cleanup stack) before the next one starts, and every cell is preceded +// by a `cooldown` sleep longer than this machine's TIME_WAIT duration so it +// starts from a genuinely clean ephemeral port range rather than whatever +// the previous cell left draining. +package loadbench_test + +import ( + "os" + "runtime" + "strconv" + "testing" + "time" + + "github.com/the-dev-tools/dev-tools/apps/cli/internal/loadrun" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/loadmetrics" +) + +// benchLatency is the target server's fixed per-request handler latency. It +// stands in for a real backend's floor cost: every RPS number this package +// produces is an upper bound on what the load engine could ever deliver +// against something slower than this. +const benchLatency = 5 * time.Millisecond + +// warmupDuration is discarded before every measured window. It exists so +// Go's HTTP connection pool and the scenario's goroutines have a moment to +// settle before the numbers that matter start accumulating - not because +// this in-process target has anything like a JIT or a cache to warm. +const warmupDuration = 1 * time.Second + +// cooldown separates cells so one cell's connection churn (see the port +// exhaustion note above) fully drains before the next cell starts dialing. +// It must exceed this machine's TCP TIME_WAIT duration (2*MSL; `sysctl +// net.inet.tcp.msl` read 15000ms here, so TIME_WAIT is ~30s) or the very +// next cell inherits whatever fraction of the ~16k-port ephemeral range the +// previous cell left draining - which is exactly what a first attempt at +// this harness got wrong: a 2s cooldown after a VUs=10 cell (12.5k +// connections opened) left ~10k ports still in TIME_WAIT, so the following +// VUs=50 cell started with almost no headroom and failed 98.5% of its +// requests - a number that measured leftover port pressure, not VUs=50 +// itself. 35s clears it with margin. +const cooldown = 35 * time.Second + +// benchVULevels is the VUs axis of the matrix, fixed by the task contract. +// Order matters: it is walked outermost (see TestLoadBenchMatrix), so the +// highest tier - the one that provokes ephemeral port exhaustion on this +// target - runs last and cannot contaminate a lower tier's numbers. +var benchVULevels = []int{1, 10, 50} + +// benchMatrixConfig is one axis of the config dimension: a named flow +// fixture plus the step names it produces, for the per-step breakdown. +type benchMatrixConfig struct { + name string + flowName string + yamlDoc func(baseURL string) string + steps []string +} + +var benchMatrixConfigs = []benchMatrixConfig{ + {name: "single-get", flowName: "SingleGet", yamlDoc: singleGetFlowYAML, steps: []string{"Step1"}}, + {name: "chained-5-step", flowName: "ChainedFlow", yamlDoc: chainedFlowYAML, steps: chainedFlowSteps}, +} + +// measureDuration returns how long to run the measured window (after +// warmup) for a given VU level. At fixed 5ms handler latency, one VU can +// issue at most ~200 requests/sec, so VUs=1 needs more wall time than VUs=50 +// to accumulate a comparable sample; higher VU levels clear a solid sample +// in a few seconds. Every tier still satisfies the >=5s floor. +func measureDuration(vus int) time.Duration { + switch { + case vus <= 1: + return 10 * time.Second + case vus <= 10: + return 8 * time.Second + default: + return 6 * time.Second + } +} + +// benchRow is one (config, VUs) cell's result, kept around after the loop so +// the summary table and the sanity check can both read it back. +type benchRow struct { + config string + vus int + result loadrun.Result + errorRate float64 +} + +func TestLoadBenchMatrix(t *testing.T) { + if os.Getenv("RUN_LOADBENCH") != "true" { + t.Skip("set RUN_LOADBENCH=true to run the RPS/worker benchmark (takes roughly 5 minutes of wall-clock, most of it cooldown - see the `cooldown` doc comment)") + } + + t.Logf("environment: OS=%s ARCH=%s NumCPU=%d GoVersion=%s LeanMode=on Target=local-in-process(+%s handler latency)", + runtime.GOOS, runtime.GOARCH, runtime.NumCPU(), runtime.Version(), benchLatency) + + var rows []benchRow + + first := true + for _, vus := range benchVULevels { + for _, cfg := range benchMatrixConfigs { + if first { + first = false + } else { + t.Logf("cooling down %s before %s/VUs=%d so leftover TIME_WAIT sockets from the previous cell drain first", + cooldown, cfg.name, vus) + time.Sleep(cooldown) + } + + t.Run(cfg.name+"/VUs="+strconv.Itoa(vus), func(t *testing.T) { + target := newBenchTarget(t, benchLatency, false) + flow, services := setupFlow(t, cfg.yamlDoc(target.URL), cfg.flowName) + + if _, err := loadrun.Run(t.Context(), loadrun.Config{ + Flow: flow, VUs: vus, Duration: warmupDuration, + }, services, nil); err != nil { + t.Fatalf("%s VUs=%d warmup failed: %v", cfg.name, vus, err) + } + + dur := measureDuration(vus) + result, err := loadrun.Run(t.Context(), loadrun.Config{ + Flow: flow, VUs: vus, Duration: dur, + }, services, nil) + if err != nil { + t.Fatalf("%s VUs=%d measurement failed: %v", cfg.name, vus, err) + } + + errorRate := 0.0 + if result.Summary.Iterations > 0 { + errorRate = float64(result.Summary.Errors) / float64(result.Summary.Iterations) + } + + switch { + case result.Summary.Errors == 0: + // expected at every tier + case vus <= 10: + // Low concurrency should never fail against this target; + // treat it as a real problem worth investigating. + t.Errorf("%s VUs=%d: %d/%d iterations errored (%.1f%%) against a target that should never fail at this concurrency", + cfg.name, vus, result.Summary.Errors, result.Summary.Iterations, 100*errorRate) + default: + // VUs=50: expected to show the ephemeral-port artifact + // documented at the top of this file. Logged, not + // failed, so the matrix still produces a full row. + t.Logf("%s VUs=%d: %d/%d iterations errored (%.1f%%) - see the port-exhaustion note in this file's package doc", + cfg.name, vus, result.Summary.Errors, result.Summary.Iterations, 100*errorRate) + } + + for k, s := range result.Report.PerStep { + if s.ErrorCount > 0 { + t.Logf(" DIAG key=%+v count=%d errCount=%d p50=%s max=%s", k, s.Count, s.ErrorCount, s.P50, s.Max) + } + } + + total := result.Report.Total + t.Logf("RESULT config=%-15s vus=%-3d requests=%-8d iterations=%-6d elapsed=%-12s RPS=%-9.1f errRate=%-6.1f%% p50=%-10s p95=%-10s p99=%-10s max=%s", + cfg.name, vus, total.Count, result.Summary.Iterations, result.Summary.Elapsed.Round(time.Millisecond), + total.RPS, 100*errorRate, total.P50, total.P95, total.P99, total.Max) + + for _, step := range cfg.steps { + if s, ok := result.ByStep.PerStep[loadmetrics.Key{Step: step}]; ok { + t.Logf(" step=%-8s count=%-8d p50=%-10s p95=%-10s p99=%s", step, s.Count, s.P50, s.P95, s.P99) + } + } + + rows = append(rows, benchRow{config: cfg.name, vus: vus, result: result, errorRate: errorRate}) + }) + } + } + + logMarkdownTable(t, rows) + sanityCheckScaling(t, rows) +} + +// logMarkdownTable prints the config x VUs -> RPS/p50/p95/p99 table in the +// exact shape it belongs in docs/superpowers/specs/phase0-bench.md, so +// filling in that doc is a transcription of this log, not a re-derivation. +func logMarkdownTable(t *testing.T, rows []benchRow) { + t.Log("markdown table (paste into phase0-bench.md):") + t.Log("| Config | VUs | Requests | Iterations | Elapsed | RPS | Error% | P50 | P95 | P99 |") + t.Log("|---|---|---|---|---|---|---|---|---|---|") + for _, r := range rows { + total := r.result.Report.Total + t.Logf("| %s | %d | %d | %d | %s | %.1f | %.1f%% | %s | %s | %s |", + r.config, r.vus, total.Count, r.result.Summary.Iterations, + r.result.Summary.Elapsed.Round(time.Millisecond), total.RPS, 100*r.errorRate, total.P50, total.P95, total.P99) + } +} + +// sanityCheckScaling pins the matrix's own precondition: at a fixed 5ms +// handler latency, RPS must scale with VUs, or these numbers do not measure +// what the doc claims they measure. A flat curve between VUs=1 and VUs=10 +// means this harness serialized somewhere - that would be a bug in this +// package, not a finding about the load engine, and it must be fixed before +// any number here is trusted. VUs=50 is deliberately excluded from this +// check: its expected port-exhaustion errors legitimately suppress RPS +// growth on this machine, which the check would otherwise misreport as a +// harness bug. +func sanityCheckScaling(t *testing.T, rows []benchRow) { + byConfig := make(map[string]map[int]float64) + for _, r := range rows { + if byConfig[r.config] == nil { + byConfig[r.config] = make(map[int]float64) + } + byConfig[r.config][r.vus] = r.result.Report.Total.RPS + } + + for cfg, byVU := range byConfig { + rps1, rps10 := byVU[1], byVU[10] + if rps1 <= 0 { + continue + } + const minScaleFactor = 3.0 + if rps10 < rps1*minScaleFactor { + t.Errorf("%s: RPS at VUs=10 (%.1f) is not substantially above VUs=1 (%.1f, x%.1f) - "+ + "expected at least x%.1f on a %s-latency target; investigate before trusting these numbers", + cfg, rps10, rps1, rps10/rps1, minScaleFactor, benchLatency) + } else { + t.Logf("sanity check ok: %s scaled x%.1f from VUs=1 (%.1f RPS) to VUs=10 (%.1f RPS)", + cfg, rps10/rps1, rps1, rps10) + } + } +} diff --git a/apps/cli/test/loadbench/loadbench_test.go b/apps/cli/test/loadbench/loadbench_test.go new file mode 100644 index 000000000..e183a8f03 --- /dev/null +++ b/apps/cli/test/loadbench/loadbench_test.go @@ -0,0 +1,432 @@ +// Package loadbench_test is a self-contained benchmark harness for the CLI's +// load mode (apps/cli/internal/loadrun): it measures sustained RPS against a +// local, in-process HTTP target with a fixed handler latency, so the numbers +// describe the load engine's own overhead ceiling rather than any real +// network or backend. +// +// It has no external network dependency whatsoever - the target server is +// httptest.NewServer, bound to loopback, and every request in this package +// stays on that connection. +// +// Two kinds of test live here, deliberately split by whether they belong in +// the default suite: +// +// - Fixture correctness (this file). Proves the chained-flow fixture used +// by the benchmark really chains - step N's request carries step N-1's +// response value, not a hardcoded one - and that the single-GET fixture +// hits the target exactly once per iteration. Both run in well under a +// second and run unconditionally, like any other test in this repo. +// - The RPS/percentile matrix (integration_loadbench_test.go). Minutes of +// wall-clock time, hardware-sensitive, gated behind the loadbench_integration +// build tag and RUN_LOADBENCH=true. See that file for how to run it and +// where its results end up. +package loadbench_test + +import ( + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/the-dev-tools/dev-tools/apps/cli/internal/common" + "github.com/the-dev-tools/dev-tools/apps/cli/internal/loadrun" + "github.com/the-dev-tools/dev-tools/apps/cli/internal/runner" + "github.com/the-dev-tools/dev-tools/packages/db/pkg/sqlitemem" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/flow/flowbuilder" + gqlresolver "github.com/the-dev-tools/dev-tools/packages/server/pkg/graphql/resolver" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/http/resolver" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/ioworkspace" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mflow" + "github.com/the-dev-tools/dev-tools/packages/server/pkg/service/scredential" + yamlflowsimplev2 "github.com/the-dev-tools/dev-tools/packages/server/pkg/translate/yamlflowsimplev2" +) + +// benchLatency (the target server's fixed per-request handler latency used by +// the actual benchmark matrix) lives in integration_loadbench_test.go, the +// only place that references it - the fixture-correctness tests in this file +// use zero latency instead (see newBenchTarget's latency parameter): they +// only care about wiring, and zero latency keeps them fast. Declaring it here +// anyway would make it unused whenever this package is built without the +// loadbench_integration tag, i.e. always, outside a benchmark run. + +// chainHeader is the response header the /chain endpoint uses to hand its +// issued token to whichever step reads it next. It deliberately has no +// hyphen: {{ }} interpolation runs the path through expr-lang +// (packages/server/pkg/expression), which parses a hyphenated segment as +// subtraction between two identifiers, not as a map key. +const chainHeader = "Chaintoken" + +// chainHop is one /chain request as the target server saw it: the token the +// requester claimed as "prev", and the token this response issues in return. +// A correctly wired chained flow produces prev[i] == issued[i-1] for every +// hop after the first. +type chainHop struct { + prev string + issued string +} + +// benchTarget is the local, in-process HTTP target every flow in this +// package drives. It has two routes: /single, for the single-GET config, and +// /chain, for the 5-step chained config. +type benchTarget struct { + *httptest.Server + requests atomic.Int64 + + tokens atomic.Int64 + + mu sync.Mutex + record bool + chainLog []chainHop +} + +// newBenchTarget starts a target server. latency is slept inside the handler +// before it responds, on every request. When record is true, every /chain +// request is appended to chainLog (guarded by mu) for a correctness test to +// inspect afterwards; the benchmark matrix always passes record=false so the +// measured path pays no bookkeeping cost beyond the plain atomic counters. +func newBenchTarget(t *testing.T, latency time.Duration, record bool) *benchTarget { + t.Helper() + + bt := &benchTarget{record: record} + + mux := http.NewServeMux() + mux.HandleFunc("/single", func(w http.ResponseWriter, r *http.Request) { + bt.requests.Add(1) + sleep(latency) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"ok":true}`)) + }) + mux.HandleFunc("/chain", func(w http.ResponseWriter, r *http.Request) { + bt.requests.Add(1) + sleep(latency) + + prev := r.URL.Query().Get("prev") + issued := fmt.Sprintf("tok-%d", bt.tokens.Add(1)) + + if bt.record { + bt.mu.Lock() + bt.chainLog = append(bt.chainLog, chainHop{prev: prev, issued: issued}) + bt.mu.Unlock() + } + + w.Header().Set(chainHeader, issued) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprintf(w, `{"token":%q}`, issued) + }) + + bt.Server = httptest.NewServer(mux) + t.Cleanup(bt.Close) + return bt +} + +func sleep(d time.Duration) { + if d > 0 { + time.Sleep(d) + } +} + +// hops returns a snapshot of the recorded /chain requests, safe to range +// over after the run that produced them has finished. +func (bt *benchTarget) hops() []chainHop { + bt.mu.Lock() + defer bt.mu.Unlock() + return append([]chainHop(nil), bt.chainLog...) +} + +// singleGetFlowYAML is config (a): one request node, no chaining. +func singleGetFlowYAML(baseURL string) string { + return fmt.Sprintf(` +workspace_name: LoadBench Workspace +flows: + - name: SingleGet + steps: + - manual_start: + name: Start + - request: + name: Step1 + depends_on: Start + method: GET + url: %s/single +`, baseURL) +} + +// chainedFlowYAML is config (b): five request nodes in a dependency chain, +// where step N sends the token step N-1's response issued. This is what +// TestChainedFlowReallyChains checks isn't a lie. +func chainedFlowYAML(baseURL string) string { + return fmt.Sprintf(` +workspace_name: LoadBench Workspace +flows: + - name: ChainedFlow + steps: + - manual_start: + name: Start + - request: + name: Step1 + depends_on: Start + method: GET + url: %[1]s/chain + - request: + name: Step2 + depends_on: Step1 + method: GET + url: %[1]s/chain + query_params: + prev: '{{ Step1.response.headers.%[2]s }}' + - request: + name: Step3 + depends_on: Step2 + method: GET + url: %[1]s/chain + query_params: + prev: '{{ Step2.response.headers.%[2]s }}' + - request: + name: Step4 + depends_on: Step3 + method: GET + url: %[1]s/chain + query_params: + prev: '{{ Step3.response.headers.%[2]s }}' + - request: + name: Step5 + depends_on: Step4 + method: GET + url: %[1]s/chain + query_params: + prev: '{{ Step4.response.headers.%[2]s }}' +`, baseURL, chainHeader) +} + +// chainedFlowSteps names the chained config's steps in dependency order, for +// callers that want a per-step breakdown. +var chainedFlowSteps = []string{"Step1", "Step2", "Step3", "Step4", "Step5"} + +// setupFlow imports a yamlflow document into a fresh in-memory workspace and +// returns the flow plus the services a load run needs. +// +// This duplicates apps/cli/internal/loadrun's own setupFlow test helper +// (which is unexported, so it cannot be imported) and, in turn, the CLI's +// setup in cmd/flow.go. Load-mode's docs already flag this as a known +// duplication of cmd wiring; for a benchmark, replicating a proven pattern is +// the right call over inventing a third way. +func setupFlow(t *testing.T, yamlDoc, flowName string) (*mflow.Flow, runner.RunnerServices) { + t.Helper() + + ctx := t.Context() + logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelError})) + + db, cleanup, err := sqlitemem.NewSQLiteMem(ctx) + if err != nil { + t.Fatalf("create in-memory db: %v", err) + } + t.Cleanup(cleanup) + + services, err := common.CreateServices(ctx, db, logger) + if err != nil { + t.Fatalf("create services: %v", err) + } + + workspaceID := idwrap.NewNow() + bundle, err := yamlflowsimplev2.ConvertSimplifiedYAML([]byte(yamlDoc), yamlflowsimplev2.ConvertOptionsV2{ + WorkspaceID: workspaceID, + }) + if err != nil { + t.Fatalf("convert yaml: %v", err) + } + + builder := flowbuilder.New( + &services.Node, &services.NodeRequest, &services.NodeFor, &services.NodeForEach, + &services.NodeIf, &services.NodeJS, &services.NodeAI, &services.NodeAiProvider, + &services.NodeMemory, &services.NodeGraphQL, &services.NodeWsConnection, + &services.NodeWsSend, &services.NodeWait, &services.NodeSubFlowTrigger, + &services.NodeSubFlowReturn, &services.NodeRunSubFlow, &services.WebSocket, + &services.WebSocketHeader, &services.GraphQL, &services.GraphQLHeader, + &services.Workspace, &services.Variable, &services.FlowVariable, + resolver.NewStandardResolver( + &services.HTTP, &services.HTTPHeader, services.HTTPSearchParam, + services.HTTPBodyRaw, services.HTTPBodyForm, services.HTTPBodyUrlEncoded, + services.HTTPAssert, + ), + gqlresolver.NewStandardResolver( + services.GraphQL.Reader(), &services.GraphQLHeader, &services.GraphQLAssert, + ), + services.Logger, + scredential.NewLLMProviderFactory(&services.Credential), + ) + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin tx: %v", err) + } + bundle.Workspace.ID = workspaceID + if err := services.Workspace.TX(tx).Create(ctx, &bundle.Workspace); err != nil { + _ = tx.Rollback() + t.Fatalf("create workspace: %v", err) + } + importOpts := ioworkspace.GetDefaultImportOptions(workspaceID) + importOpts.PreserveIDs = true + if _, err := ioworkspace.New(services.Queries, logger).Import(ctx, tx, bundle, importOpts); err != nil { + _ = tx.Rollback() + t.Fatalf("import bundle: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit import: %v", err) + } + + flows, err := services.Flow.GetFlowsByWorkspaceID(ctx, workspaceID) + if err != nil { + t.Fatalf("get flows: %v", err) + } + var flow *mflow.Flow + for i := range flows { + if flows[i].Name == flowName { + flow = &flows[i] + break + } + } + if flow == nil { + t.Fatalf("flow %q not found among %d imported flows", flowName, len(flows)) + } + + return flow, runner.RunnerServices{ + NodeService: services.Node, + EdgeService: services.FlowEdge, + FlowVariableService: services.FlowVariable, + Builder: builder, + } +} + +// TestChainedFlowReallyChains proves the fixture behind the "5-step chained +// flow" benchmark config is not five independent requests in a trenchcoat: +// each step's request must carry the exact token the previous step's +// response issued. +// +// It runs without RUN_LOADBENCH because it guards the thing that would make +// every chained-flow benchmark number meaningless if it silently broke: a +// static or mis-wired URL would still produce a report, just not one that +// measures chaining. VUs=1 with a single iteration keeps the target server's +// recorded request order unambiguous - concurrent VUs would interleave the +// log and this check would need per-chain correlation it doesn't have. +func TestChainedFlowReallyChains(t *testing.T) { + target := newBenchTarget(t, 0, true) + flow, services := setupFlow(t, chainedFlowYAML(target.URL), "ChainedFlow") + + result, err := loadrun.Run(t.Context(), loadrun.Config{ + Flow: flow, + VUs: 1, + MaxIterations: 1, + }, services, nil) + if err != nil { + t.Fatalf("Run failed: %v", err) + } + if result.Summary.Errors != 0 { + t.Fatalf("Summary.Errors = %d, want 0", result.Summary.Errors) + } + + hops := target.hops() + if len(hops) != len(chainedFlowSteps) { + t.Fatalf("server saw %d /chain requests, want %d", len(hops), len(chainedFlowSteps)) + } + + if hops[0].prev != "" { + t.Errorf("Step1 sent prev=%q, want empty (it is the first hop, nothing precedes it)", hops[0].prev) + } + for i := 1; i < len(hops); i++ { + if hops[i].prev != hops[i-1].issued { + t.Errorf("%s sent prev=%q, want %q (%s's issued token) - the chain is broken", + chainedFlowSteps[i], hops[i].prev, hops[i-1].issued, chainedFlowSteps[i-1]) + } + } + + // Every issued token must be distinct, or a flow that hardcodes (say) + // Step1's token into every subsequent request could pass the adjacency + // check above by accident. + seen := make(map[string]bool, len(hops)) + for _, h := range hops { + if seen[h.issued] { + t.Fatalf("token %q was issued more than once - the target stopped varying per request", h.issued) + } + seen[h.issued] = true + } +} + +// TestChainedFlowMultipleIterationsStillChain runs the same fixture for +// three iterations on a single VU, so the per-iteration hop count and the +// last iteration's chain are both checked - proving the chain resets cleanly +// each iteration rather than accidentally carrying a stale token forward. +func TestChainedFlowMultipleIterationsStillChain(t *testing.T) { + const iterations = 3 + + target := newBenchTarget(t, 0, true) + flow, services := setupFlow(t, chainedFlowYAML(target.URL), "ChainedFlow") + + result, err := loadrun.Run(t.Context(), loadrun.Config{ + Flow: flow, + VUs: 1, + MaxIterations: iterations, + }, services, nil) + if err != nil { + t.Fatalf("Run failed: %v", err) + } + if result.Summary.Errors != 0 { + t.Fatalf("Summary.Errors = %d, want 0", result.Summary.Errors) + } + + hops := target.hops() + wantHops := iterations * len(chainedFlowSteps) + if len(hops) != wantHops { + t.Fatalf("server saw %d /chain requests, want %d", len(hops), wantHops) + } + + for iter := range iterations { + start := iter * len(chainedFlowSteps) + iterHops := hops[start : start+len(chainedFlowSteps)] + + if iterHops[0].prev != "" { + t.Errorf("iteration %d: Step1 sent prev=%q, want empty", iter, iterHops[0].prev) + } + for i := 1; i < len(iterHops); i++ { + if iterHops[i].prev != iterHops[i-1].issued { + t.Errorf("iteration %d: %s sent prev=%q, want %q", + iter, chainedFlowSteps[i], iterHops[i].prev, iterHops[i-1].issued) + } + } + } +} + +// TestSingleGetFlowHitsTargetOncePerIteration is the single-GET config's +// equivalent sanity check: every iteration is exactly one request, and the +// report agrees with what the server actually saw. +func TestSingleGetFlowHitsTargetOncePerIteration(t *testing.T) { + const iterations = 3 + + target := newBenchTarget(t, 0, false) + flow, services := setupFlow(t, singleGetFlowYAML(target.URL), "SingleGet") + + result, err := loadrun.Run(t.Context(), loadrun.Config{ + Flow: flow, + VUs: 1, + MaxIterations: iterations, + }, services, nil) + if err != nil { + t.Fatalf("Run failed: %v", err) + } + + if got := target.requests.Load(); got != iterations { + t.Errorf("server saw %d requests, want %d", got, iterations) + } + if result.Report.Total.Count != iterations { + t.Errorf("Report.Total.Count = %d, want %d", result.Report.Total.Count, iterations) + } + if result.Summary.Errors != 0 { + t.Errorf("Summary.Errors = %d, want 0", result.Summary.Errors) + } +} diff --git a/docs/superpowers/specs/phase0-bench.md b/docs/superpowers/specs/phase0-bench.md new file mode 100644 index 000000000..883bf9508 --- /dev/null +++ b/docs/superpowers/specs/phase0-bench.md @@ -0,0 +1,244 @@ +# Phase 0 benchmark: RPS/worker + +Status: **dev-hardware only. Fly shared-cpu-machine numbers are not yet measured.** + +This is Phase 0 work item 7 (`docs/superpowers/plans/2026-08-08-stresseur-phase0-1.md` +§4.1.7): measure what one load worker (one CLI process, one `load run`) can actually +generate, using the real `loadrun` engine (`apps/cli/internal/loadrun`), not a +reimplementation of its scheduling. It answers "RPS per worker" for two reference +flows, which is the input spec §3.5 ("workers are sized by benchmark ... 500 VUs ≈ a +handful of machines, not 500") and spec §6 ("benchmark finalizes the unit economics") +both name as a prerequisite. + +**These numbers gate §3.5 capacity math and §6 pricing only as upper-bound local +numbers.** They were produced on a laptop against an in-process target with no network, +no TLS, no real backend work, and no other tenants competing for the CPU. A Fly +shared-cpu machine (the actual worker in production) has different CPU/network +characteristics, runs one worker per machine (fewer noisy neighbors than a laptop, but +weaker single-core performance than an M-series Mac), and any real target has +non-trivial latency of its own. Until that number exists, treat everything below as +"the engine's own overhead ceiling, measured somewhere convenient" - a ceiling real +capacity math must stay under, not a floor it can plan against. + +## Environment + +| | | +|---|---| +| Date | 2026-08-08 | +| Machine | Apple M2 Max, 12 logical CPUs (`runtime.NumCPU()`), 32 GB RAM | +| OS/Arch | darwin/arm64 (`GOOS`/`GOARCH`) | +| Go | go1.25.5 (`runtime.Version()`) | +| Lean mode | on (always on for load runs - see `loadrun.go` package doc) | +| Target | local, in-process `httptest.NewServer`, loopback only, fixed 5ms handler latency (`time.Sleep(5*time.Millisecond)`), no external network dependency of any kind | +| Engine | `apps/cli/internal/loadrun.Run`, driven in-process exactly as `apps/cli/internal/loadrun/loadrun_test.go`'s `setupFlow` helper does (yamlflow import → in-memory SQLite workspace → `flowbuilder` → `loadrun.Run`) | + +## Methodology + +- **Harness**: `apps/cli/test/loadbench/`. `loadbench_test.go` holds the target server, + the two flow fixtures, and fixture-correctness tests that run unconditionally (no + build tag, no env guard - they run in `cli:test` like any other test). + `integration_loadbench_test.go` holds the actual RPS/percentile matrix, gated by both + `//go:build loadbench_integration` and `RUN_LOADBENCH=true`, following this repo's + integration-test convention (paired build tag + env guard, e.g. + `packages/server/pkg/flow/node/nai`'s `ai_integration` / `RUN_AI_INTEGRATION_TESTS`). +- **Configs**: + - **(a) single-get** - one request node (`GET /single`). + - **(b) chained-5-step** - five request nodes in a real dependency chain. Each step + reads a token the previous step's response issued (via a response header, + `Chaintoken` - see "why a header, not the body" below) and sends it back as a query + parameter on the next request. `TestChainedFlowReallyChains` and + `TestChainedFlowMultipleIterationsStillChain` (both unguarded, run in `cli:test`) + assert step N's request carries exactly step N-1's issued token, so this is + real chaining, not five independent requests wearing a trenchcoat. +- **VUs**: 1, 10, 50, per the task contract. +- **Stop condition**: `Duration`, not `MaxIterations` - simpler to reason about and + guarantees the ≥5s floor unconditionally. A 1s warmup run (discarded) precedes every + measured window, so Go's connection pool and the scenario's goroutines aren't + measured mid-ramp. Measured-window length is tiered by VU level, since this + fixed-5ms-latency target caps one VU at ~200 req/s and low-VU cells need more wall + time to accumulate a comparable sample: 10s at VUs=1, 8s at VUs=10, 6s at VUs=50. All + tiers clear well over a thousand requests; see the table for exact counts. +- **Isolation between cells**: each (config, VUs) cell runs in its own `t.Run` subtest, + with its own target server and its own in-memory SQLite workspace, torn down before + the next cell starts. Cells run VUs-major (all VUs=1 first, then all VUs=10, then + both VUs=50 cells last) and are separated by a 35s cooldown - both exist specifically + to stop one cell's numbers from contaminating another's; see "Anomaly investigated" + below for why that isolation turned out to matter. +- **What "RPS" means here**: `loadmetrics.Report.Total.RPS` is + `(requests recorded)/(wall time covered)`, and a request is recorded whether it + succeeded or failed (see `vuWorker.record` in `loadrun.go`) - so it is an *attempt* + rate, not a success rate. At VUs=1 and VUs=10 every attempt succeeded, so the + distinction is moot there. At VUs=50 it is not; see below. +- **Why a header, not the body, carries the chained value**: load runs always run in + lean mode, which replaces the entire decoded response body with a fixed placeholder + string once assertions run (`nrequest.LeanBodyPlaceholder`) - a flow that tried to + chain via `response.body.someField` would silently chain the placeholder, not a real + per-response value. Response *headers* are untouched by lean mode, so the target + returns its issued token via a `Chaintoken` response header instead. (It has no + hyphen: `{{ }}` interpolation runs through `expr-lang`, which parses a hyphenated + segment as subtraction between two identifiers rather than as a map key.) + +## Results + +RPS is requests/sec (attempts, per above). P50/P95/P99 are per-request latency +(HDR histogram, includes both successes and failures - a fast-failing dial attempt at +VUs=50 pulls percentiles in a different direction than a slow one, see below). + +| Config | VUs | Requests | Iterations | Elapsed | RPS | Error % | P50 | P95 | P99 | Max | +|---|---|---|---|---|---|---|---|---|---|---| +| single-get | 1 | 1,554 | 1,554 | 10.005s | 155.3 | 0.0% | 6.503ms | 6.835ms | 7.091ms | 8.343ms | +| chained-5-step | 1 | 1,560 | 312 | 10.029s | 155.5 | 0.0% | 6.515ms | 6.875ms | 7.071ms | 8.543ms | +| single-get | 10 | 12,516 | 12,516 | 8.002s | 1,564.1 | 0.0% | 6.323ms | 6.687ms | 7.327ms | 18.927ms | +| chained-5-step | 10 | 12,600 | 2,520 | 8.026s | 1,569.8 | 0.0% | 6.359ms | 6.651ms | 7.215ms | 14.975ms | +| single-get | 50 | 25,544 | 25,544 | 6.014s | 4,247.6 | 12.9% | 5.879ms | 11.375ms | 156.927ms | 860.159ms | +| chained-5-step | 50 | 26,807 | 7,778 | 6.055s | 4,427.0 | 42.0%\* | 5.907ms | 7.355ms | 195.199ms | 350.207ms | + +\* Iteration error rate. The chain aborts a step's dependents once that step fails +(`depends_on` never satisfies), so per-*iteration* failure overstates per-*request* +failure for the chained config: 3,266 of 26,807 individual requests failed (12.2%), +concentrated at Step1 (2,633 of 7,778 attempts, 33.9%) and falling steeply at each +later hop (Step2 8.0%, Step3 2.9%, Step4 1.0%, Step5 0.9%) as fewer iterations survive +to reach them. + +**This table is from the final run against the lint-clean committed code** (the run +this task's gates section also reports). An earlier run against functionally +identical code (before two cosmetic lint fixes - see "Files changed") measured +single-get/chained-5-step VUs=50 error rates of 14.1%/37.7% instead of this run's +12.9%/42.0%; VUs=1 and VUs=10 were stable to three figures across both runs. See +"run-to-run variability" under the anomaly section below - this spread is itself +part of the finding, not noise to average away. Raw command output for every run, +including the `t.Run` subtest breakdown and per-step DIAG lines, is preserved +verbatim in this task's report (`task-6-report.md`). + +### Scaling sanity check (VUs=1 → VUs=10) + +At a fixed 5ms handler latency the theoretical ceiling is linear in VUs +(`VUs / latency`). Measured: + +- single-get: 155.3 → 1,564.1 RPS = **x10.1** +- chained-5-step: 155.5 → 1,569.8 RPS = **x10.1** + +Both configs scale within noise of perfectly linear, and both are error-free at these +tiers. This is the harness's own precondition check (`sanityCheckScaling` in +`integration_loadbench_test.go`, run automatically): a flat curve here would mean the +harness was serializing somewhere, and the numbers would be measuring this package's +bugs instead of the engine. It passed on every run once cell isolation was fixed (see +below) - the curve is real. + +## Anomaly investigated: VUs=50 request failures + +The first full-matrix run showed VUs=50 failing 30-98% of iterations depending on +ordering, with a p99 in the *seconds*. That is exactly the "if it doesn't [scale], +investigate before writing the doc" case this task calls out, so before anything below +was accepted as real, it was root-caused rather than reported as-is. + +**Root cause**: `packages/server/pkg/httpclient.New()` (the CLI's actual HTTP client, +used unmodified by every `loadrun` VU worker) builds an `http.Client` with no custom +`Transport`, so it falls back to `http.DefaultTransport`, whose +`MaxIdleConnsPerHost` defaults to 2. At VUs=50 against a target this fast, the client +issues thousands of requests/sec to one host; with only 2 idle connections kept per +host, nearly every request redials instead of reusing a keep-alive connection. On +macOS the ephemeral port range is 49152-65535 (~16k ports, +`sysctl net.inet.ip.portrange.*`) and `TIME_WAIT` lasts 2×MSL ≈ 30s +(`sysctl net.inet.tcp.msl` = 15000ms here), so sustained redialing at thousands/sec +exhausts the range within single-digit seconds. This was confirmed independently of +the flow engine with a ~60-line raw `net/http` repro (50 goroutines, a fresh +`http.Client` each, hammering a local `httptest.Server` for 6s): 21.4% of requests +failed with `dial tcp 127.0.0.1:PORT: connect: can't assign requested address` - +textbook ephemeral port exhaustion, not a flow-engine or flake. + +**Why the first matrix run was worse than that repro (30-98% vs ~15-21%)**: the first +version of this harness ran all six cells back-to-back with only a 2s gap and no +resource teardown between them. A VUs=10 cell alone opens ~12,500 short-lived +connections; with a 30s `TIME_WAIT`, a 2s gap starts the next cell with most of the +ephemeral range still occupied by the *previous* cell's sockets - so the reported +"VUs=50" failure rate was really measuring leftover port pressure from VUs=10, not +VUs=50 itself, and got worse the more cells had already run. Fixed by: running cells +VUs-major so nothing runs after a VUs=50 cell, wrapping every cell in its own `t.Run` +so its server and DB are torn down before the next cell starts, and - the part that +actually mattered - a 35s cooldown before every cell, longer than this machine's +`TIME_WAIT`. The numbers in the table above are from the run after that fix; they +matched the isolated single-cell repro's error rate (~14-15% for single-get) far more +closely than the contaminated run did, which is itself evidence the fix addressed the +right thing. + +**Is this a load-engine bug?** Not one this task should fix - `httpclient.New()` is +shared production code with call sites well beyond load mode, and retuning its +connection pool is a separate, deliberate change with its own review, not a +benchmark-harness side effect. It is a real, reproducible characteristic worth +recording for whoever does own that decision: **a single worker sustaining +several-thousand req/s against one host, for more than a few seconds, can exhaust +local ephemeral ports on this class of machine** without an explicit +`MaxIdleConnsPerHost`/`MaxConnsPerHost` tuned for the expected VU count. Two mitigating +factors for how much this matters in practice: + +1. It is a *rate* problem, not strictly a *VUs* problem - it happens because 50 VUs + against a 5ms target reach ~4,200-4,500 req/s. A real target with realistic latency + (tens to hundreds of ms) would let the same 50 VUs reach only a fraction of that + rate, buying comparable headroom before hitting the same wall. This benchmark's + target is deliberately unrealistically fast (that is the point - it isolates engine + overhead), and that is exactly what makes it also the case most likely to trip this. +2. Effective *successful* throughput at VUs=50 was still far above VUs=10, not flat: + in the reported run, single-get did 22,240 successful requests in 6.014s (3,698.0 + successful req/s); chained-5-step did 23,541 successful individual requests in the + same window (3,887.9 successful req/s), or 4,512 fully-completed 5-step iterations + (3,725.8 successful req/s expressed the same way - iterations x steps / elapsed). + All three land within ~5% of each other despite two different flow shapes - + consistent with one shared underlying ceiling (port cycling rate), not something + specific to chaining. (The earlier run showed the same pattern at slightly higher + absolute numbers: ~3,850-4,050 successful req/s - see below.) + +**Run-to-run variability**: two complete matrix runs against functionally identical +code (see "Files changed" for the no-op lint diff between them) produced consistent +VUs=1/VUs=10 numbers (RPS within 1% of each other, both runs zero errors, both scaled +~x10.0-10.1) but noticeably different VUs=50 error rates: + +| Run | single-get VUs=50 error% | chained-5-step VUs=50 error% | +|---|---|---| +| Pre lint-fix (same logic, cosmetic diff only) | 14.1% | 37.7% | +| Post lint-fix - **this is the run reported in the Results table above** | 12.9% | 42.0% | + +This is expected, not a measurement bug: ephemeral port availability at the moment a +cell starts depends on exactly how much of the previous cooldown's `TIME_WAIT` backlog +has drained, which is a real-clock race against the kernel, not something either this +harness or the load engine controls. Treat any single VUs=50 error-rate figure as "this +class of machine, under this specific artifact, lands somewhere in the low-teens to +low-40s percent" rather than a precise number - the imprecision is itself the finding. + +**Capacity-math implication**: do not read "4,247.6 RPS at VUs=50" (or "4,481.6" from +the other run) as a clean 50-VU number the way the VUs=1/VUs=10 rows can be read. Read +it as "on this machine, against this unrealistically fast target, roughly 3,700-4,050 +req/s got through across two runs, and 13-42% of attempts hit a client +connection-pooling limit that a real target's latency would likely mask." §3.5's +"workers are sized by benchmark" should size against the VUs=10 row (clean, linear, +zero errors, stable across repeated runs) until either a Fly-hardware run or a +realistic-latency target confirms whether the VUs=50 ceiling is a laptop-and-loopback +artifact or something that also shows up in production. + +## What this does and doesn't tell you + +- **Tells you**: the engine's own per-request overhead above a bare 5ms floor is + small and stable - roughly 1.3-1.5ms at P50 (6.3-6.5ms measured vs 5ms configured) + across every error-free cell, whether chained or not, whether at 1 VU or 10. Building + the request, running it through `expr-lang` interpolation, evaluating assertions, and + folding the result into `loadmetrics` costs about that much per request on this + hardware. That overhead is flat with VU count and with chain position (all 5 steps of + the chained flow show essentially the same P50), which is the shape you want: it + means the *engine* isn't the bottleneck at these VU levels, whatever else is. +- **Does not tell you**: what a Fly shared-cpu machine can sustain (not yet measured - + the explicit gap this doc flags), what a realistic-latency target changes about the + VUs=50 picture (not yet measured), or where the true per-worker VU ceiling sits once + connection pooling is tuned for the expected load-testing use case (out of scope for + this task; a note for whoever picks up `httpclient.go` next). + +## Reproducing this + +```bash +# Fixture correctness (fast, runs in the default suite too): +go test ./apps/cli/test/loadbench/... + +# The full matrix (~5 minutes, mostly cooldown; see the `cooldown` doc comment +# in integration_loadbench_test.go for why it can't be shorter on this OS): +RUN_LOADBENCH=true go test -tags loadbench_integration \ + -run TestLoadBenchMatrix ./apps/cli/test/loadbench/ -v -timeout 900s +``` From bd393dd808f8bbcc9d41cf5bab6ce17facad309c Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 22:49:46 +0300 Subject: [PATCH 32/38] style: prettier-format phase0 bench doc --- docs/superpowers/specs/phase0-bench.md | 58 +++++++++++++------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/superpowers/specs/phase0-bench.md b/docs/superpowers/specs/phase0-bench.md index 883bf9508..1cf6d270f 100644 --- a/docs/superpowers/specs/phase0-bench.md +++ b/docs/superpowers/specs/phase0-bench.md @@ -22,15 +22,15 @@ capacity math must stay under, not a floor it can plan against. ## Environment -| | | -|---|---| -| Date | 2026-08-08 | -| Machine | Apple M2 Max, 12 logical CPUs (`runtime.NumCPU()`), 32 GB RAM | -| OS/Arch | darwin/arm64 (`GOOS`/`GOARCH`) | -| Go | go1.25.5 (`runtime.Version()`) | -| Lean mode | on (always on for load runs - see `loadrun.go` package doc) | -| Target | local, in-process `httptest.NewServer`, loopback only, fixed 5ms handler latency (`time.Sleep(5*time.Millisecond)`), no external network dependency of any kind | -| Engine | `apps/cli/internal/loadrun.Run`, driven in-process exactly as `apps/cli/internal/loadrun/loadrun_test.go`'s `setupFlow` helper does (yamlflow import → in-memory SQLite workspace → `flowbuilder` → `loadrun.Run`) | +| | | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Date | 2026-08-08 | +| Machine | Apple M2 Max, 12 logical CPUs (`runtime.NumCPU()`), 32 GB RAM | +| OS/Arch | darwin/arm64 (`GOOS`/`GOARCH`) | +| Go | go1.25.5 (`runtime.Version()`) | +| Lean mode | on (always on for load runs - see `loadrun.go` package doc) | +| Target | local, in-process `httptest.NewServer`, loopback only, fixed 5ms handler latency (`time.Sleep(5*time.Millisecond)`), no external network dependency of any kind | +| Engine | `apps/cli/internal/loadrun.Run`, driven in-process exactly as `apps/cli/internal/loadrun/loadrun_test.go`'s `setupFlow` helper does (yamlflow import → in-memory SQLite workspace → `flowbuilder` → `loadrun.Run`) | ## Methodology @@ -66,14 +66,14 @@ capacity math must stay under, not a floor it can plan against. below for why that isolation turned out to matter. - **What "RPS" means here**: `loadmetrics.Report.Total.RPS` is `(requests recorded)/(wall time covered)`, and a request is recorded whether it - succeeded or failed (see `vuWorker.record` in `loadrun.go`) - so it is an *attempt* + succeeded or failed (see `vuWorker.record` in `loadrun.go`) - so it is an _attempt_ rate, not a success rate. At VUs=1 and VUs=10 every attempt succeeded, so the distinction is moot there. At VUs=50 it is not; see below. - **Why a header, not the body, carries the chained value**: load runs always run in lean mode, which replaces the entire decoded response body with a fixed placeholder string once assertions run (`nrequest.LeanBodyPlaceholder`) - a flow that tried to chain via `response.body.someField` would silently chain the placeholder, not a real - per-response value. Response *headers* are untouched by lean mode, so the target + per-response value. Response _headers_ are untouched by lean mode, so the target returns its issued token via a `Chaintoken` response header instead. (It has no hyphen: `{{ }}` interpolation runs through `expr-lang`, which parses a hyphenated segment as subtraction between two identifiers rather than as a map key.) @@ -84,17 +84,17 @@ RPS is requests/sec (attempts, per above). P50/P95/P99 are per-request latency (HDR histogram, includes both successes and failures - a fast-failing dial attempt at VUs=50 pulls percentiles in a different direction than a slow one, see below). -| Config | VUs | Requests | Iterations | Elapsed | RPS | Error % | P50 | P95 | P99 | Max | -|---|---|---|---|---|---|---|---|---|---|---| -| single-get | 1 | 1,554 | 1,554 | 10.005s | 155.3 | 0.0% | 6.503ms | 6.835ms | 7.091ms | 8.343ms | -| chained-5-step | 1 | 1,560 | 312 | 10.029s | 155.5 | 0.0% | 6.515ms | 6.875ms | 7.071ms | 8.543ms | -| single-get | 10 | 12,516 | 12,516 | 8.002s | 1,564.1 | 0.0% | 6.323ms | 6.687ms | 7.327ms | 18.927ms | -| chained-5-step | 10 | 12,600 | 2,520 | 8.026s | 1,569.8 | 0.0% | 6.359ms | 6.651ms | 7.215ms | 14.975ms | -| single-get | 50 | 25,544 | 25,544 | 6.014s | 4,247.6 | 12.9% | 5.879ms | 11.375ms | 156.927ms | 860.159ms | -| chained-5-step | 50 | 26,807 | 7,778 | 6.055s | 4,427.0 | 42.0%\* | 5.907ms | 7.355ms | 195.199ms | 350.207ms | +| Config | VUs | Requests | Iterations | Elapsed | RPS | Error % | P50 | P95 | P99 | Max | +| -------------- | --- | -------- | ---------- | ------- | ------- | ------- | ------- | -------- | --------- | --------- | +| single-get | 1 | 1,554 | 1,554 | 10.005s | 155.3 | 0.0% | 6.503ms | 6.835ms | 7.091ms | 8.343ms | +| chained-5-step | 1 | 1,560 | 312 | 10.029s | 155.5 | 0.0% | 6.515ms | 6.875ms | 7.071ms | 8.543ms | +| single-get | 10 | 12,516 | 12,516 | 8.002s | 1,564.1 | 0.0% | 6.323ms | 6.687ms | 7.327ms | 18.927ms | +| chained-5-step | 10 | 12,600 | 2,520 | 8.026s | 1,569.8 | 0.0% | 6.359ms | 6.651ms | 7.215ms | 14.975ms | +| single-get | 50 | 25,544 | 25,544 | 6.014s | 4,247.6 | 12.9% | 5.879ms | 11.375ms | 156.927ms | 860.159ms | +| chained-5-step | 50 | 26,807 | 7,778 | 6.055s | 4,427.0 | 42.0%\* | 5.907ms | 7.355ms | 195.199ms | 350.207ms | \* Iteration error rate. The chain aborts a step's dependents once that step fails -(`depends_on` never satisfies), so per-*iteration* failure overstates per-*request* +(`depends_on` never satisfies), so per-_iteration_ failure overstates per-_request_ failure for the chained config: 3,266 of 26,807 individual requests failed (12.2%), concentrated at Step1 (2,633 of 7,778 attempts, 33.9%) and falling steeply at each later hop (Step2 8.0%, Step3 2.9%, Step4 1.0%, Step5 0.9%) as fewer iterations survive @@ -128,7 +128,7 @@ below) - the curve is real. ## Anomaly investigated: VUs=50 request failures The first full-matrix run showed VUs=50 failing 30-98% of iterations depending on -ordering, with a p99 in the *seconds*. That is exactly the "if it doesn't [scale], +ordering, with a p99 in the _seconds_. That is exactly the "if it doesn't [scale], investigate before writing the doc" case this task calls out, so before anything below was accepted as real, it was root-caused rather than reported as-is. @@ -151,7 +151,7 @@ textbook ephemeral port exhaustion, not a flow-engine or flake. version of this harness ran all six cells back-to-back with only a 2s gap and no resource teardown between them. A VUs=10 cell alone opens ~12,500 short-lived connections; with a 30s `TIME_WAIT`, a 2s gap starts the next cell with most of the -ephemeral range still occupied by the *previous* cell's sockets - so the reported +ephemeral range still occupied by the _previous_ cell's sockets - so the reported "VUs=50" failure rate was really measuring leftover port pressure from VUs=10, not VUs=50 itself, and got worse the more cells had already run. Fixed by: running cells VUs-major so nothing runs after a VUs=50 cell, wrapping every cell in its own `t.Run` @@ -172,13 +172,13 @@ local ephemeral ports on this class of machine** without an explicit `MaxIdleConnsPerHost`/`MaxConnsPerHost` tuned for the expected VU count. Two mitigating factors for how much this matters in practice: -1. It is a *rate* problem, not strictly a *VUs* problem - it happens because 50 VUs +1. It is a _rate_ problem, not strictly a _VUs_ problem - it happens because 50 VUs against a 5ms target reach ~4,200-4,500 req/s. A real target with realistic latency (tens to hundreds of ms) would let the same 50 VUs reach only a fraction of that rate, buying comparable headroom before hitting the same wall. This benchmark's target is deliberately unrealistically fast (that is the point - it isolates engine overhead), and that is exactly what makes it also the case most likely to trip this. -2. Effective *successful* throughput at VUs=50 was still far above VUs=10, not flat: +2. Effective _successful_ throughput at VUs=50 was still far above VUs=10, not flat: in the reported run, single-get did 22,240 successful requests in 6.014s (3,698.0 successful req/s); chained-5-step did 23,541 successful individual requests in the same window (3,887.9 successful req/s), or 4,512 fully-completed 5-step iterations @@ -193,10 +193,10 @@ code (see "Files changed" for the no-op lint diff between them) produced consist VUs=1/VUs=10 numbers (RPS within 1% of each other, both runs zero errors, both scaled ~x10.0-10.1) but noticeably different VUs=50 error rates: -| Run | single-get VUs=50 error% | chained-5-step VUs=50 error% | -|---|---|---| -| Pre lint-fix (same logic, cosmetic diff only) | 14.1% | 37.7% | -| Post lint-fix - **this is the run reported in the Results table above** | 12.9% | 42.0% | +| Run | single-get VUs=50 error% | chained-5-step VUs=50 error% | +| ----------------------------------------------------------------------- | ------------------------ | ---------------------------- | +| Pre lint-fix (same logic, cosmetic diff only) | 14.1% | 37.7% | +| Post lint-fix - **this is the run reported in the Results table above** | 12.9% | 42.0% | This is expected, not a measurement bug: ephemeral port availability at the moment a cell starts depends on exactly how much of the previous cooldown's `TIME_WAIT` backlog @@ -224,7 +224,7 @@ artifact or something that also shows up in production. folding the result into `loadmetrics` costs about that much per request on this hardware. That overhead is flat with VU count and with chain position (all 5 steps of the chained flow show essentially the same P50), which is the shape you want: it - means the *engine* isn't the bottleneck at these VU levels, whatever else is. + means the _engine_ isn't the bottleneck at these VU levels, whatever else is. - **Does not tell you**: what a Fly shared-cpu machine can sustain (not yet measured - the explicit gap this doc flags), what a realistic-latency target changes about the VUs=50 picture (not yet measured), or where the true per-worker VU ceiling sits once From 207ae79b040d70fc27fc69653df906fe2164f8fb Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 23:25:40 +0300 Subject: [PATCH 33/38] fix: close out load-testing final review wave (8 items) Root-causes the two version-string defects together: version.go's version becomes a var so -ldflags -X can inject it, and taskfile.yaml's build:release now does so, so `devtools version` identifies the build instead of always printing v0.1.0. Also: design-doc dates the "current state" table to branch base 36d63065 and documents the run: skip-and-continue semantics; adds direct unit tests for the run: topological sort (declaration-order tie-break, self-dependency cycle, duplicate-flow-name error) plus integration coverage for the transitive skip cascade and unrelated-flow continuation; fixes a duplicate flow name in a run: block producing a blank "dependency cycle in run block: " error instead of naming the duplicate; guards the yamlflowsimplev2 golden test against silently passing on an emptied corpus; adds a zero-request FormatLoadTable case (the GraphQL-only-flow shape); documents that JUnit output carries no load data; fixes goimports ordering in importer_load_test.go; and gives finalize.sh the same "why no set -e" rationale write-summary.sh already has. No engine behavior change except the duplicate-flow-name error message (explicitly scoped: parseRunEntries only, topoSortRunEntries untouched). Goldens untouched. --- actions/run-flows/scripts/finalize.sh | 6 +- apps/cli/cmd/flow.go | 6 +- apps/cli/cmd/version.go | 7 +- apps/cli/internal/reporter/load_test.go | 33 +++- apps/cli/internal/runner/run_order.go | 16 ++ apps/cli/internal/runner/run_order_test.go | 144 ++++++++++++++++++ apps/cli/internal/runner/runner_test.go | 111 ++++++++++++++ apps/cli/taskfile.yaml | 4 +- .../specs/2026-08-08-load-testing-design.md | 14 +- .../pkg/ioworkspace/importer_load_test.go | 2 +- .../translate/yamlflowsimplev2/golden_test.go | 7 +- 11 files changed, 336 insertions(+), 14 deletions(-) create mode 100644 apps/cli/internal/runner/run_order_test.go diff --git a/actions/run-flows/scripts/finalize.sh b/actions/run-flows/scripts/finalize.sh index 0a7c417e0..412026dcb 100755 --- a/actions/run-flows/scripts/finalize.sh +++ b/actions/run-flows/scripts/finalize.sh @@ -1,7 +1,11 @@ #!/usr/bin/env bash # Sets the action's outputs (json-report, junit-report, success) and enforces # the fail-on-error input. Runs with `if: always()` in action.yml so outputs -# are always set, even when the flow run failed. +# are always set, even when the flow run failed. Deliberately does not use +# `set -e`: a failure here should degrade gracefully rather than abort the +# composite action before outputs get set, and the `[[ ... ]] &&` idiom at +# the success-assignment line below is a `set -e` footgun (a false test +# exits the whole expression non-zero, which -e treats as fatal). # # Env in: # REPORT_DIR - directory containing report.json/junit.xml (inputs.report-dir) diff --git a/apps/cli/cmd/flow.go b/apps/cli/cmd/flow.go index 7f41feab0..8669fb95e 100644 --- a/apps/cli/cmd/flow.go +++ b/apps/cli/cmd/flow.go @@ -88,7 +88,11 @@ Load mode Only HTTP request steps are measured. GraphQL, WebSocket and sub-flow steps still execute, but they are neither counted in the report nor covered by the lean execution mode that keeps memory flat, so a flow built from them - can grow its memory use over a long run.`, + can grow its memory use over a long run. + + JUnit output carries no load data. Load results go to the console table + and the JSON report's additive load_report field only; --report junit + during a load run still writes a file, but as an empty test suite.`, Args: cobra.RangeArgs(1, 2), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() diff --git a/apps/cli/cmd/version.go b/apps/cli/cmd/version.go index 3f5bb9969..6447c36df 100644 --- a/apps/cli/cmd/version.go +++ b/apps/cli/cmd/version.go @@ -10,7 +10,12 @@ func init() { rootCmd.AddCommand(versionCmd) } -const version = "v0.1.0" +// version is overwritten at build time via -ldflags -X (see +// apps/cli/taskfile.yaml's build:release task). The literal below is only +// what a plain `go build` without that flag produces (e.g. local dev +// builds), so it intentionally stays a placeholder rather than tracking the +// package version. +var version = "v0.1.0" var versionCmd = &cobra.Command{ Use: "version", diff --git a/apps/cli/internal/reporter/load_test.go b/apps/cli/internal/reporter/load_test.go index 05e18fa15..9913c61dd 100644 --- a/apps/cli/internal/reporter/load_test.go +++ b/apps/cli/internal/reporter/load_test.go @@ -50,7 +50,7 @@ func sampleLoadReport() LoadReport { }, {Step: "ConfirmOrder"}: { Count: 512, - P50: 70 * time.Millisecond, P95: 200 * time.Millisecond, P99: 300 * time.Millisecond, + P50: 70 * time.Millisecond, P95: 200 * time.Millisecond, P99: 300 * time.Millisecond, RPS: 143.1, }, }, @@ -75,6 +75,35 @@ func TestFormatLoadTable(t *testing.T) { } } +// TestFormatLoadTableZeroRequests covers the report shape a GraphQL-only (or +// WebSocket-only / sub-flow-only) flow produces under load mode: load +// metrics only count HTTP request steps (see LoadMetricsScope), so a +// scenario built entirely from other step kinds finishes with zero requests +// recorded anywhere. The table must still render its header and a TOTAL row +// - all zeros, no NaN, no panic - rather than coming out empty or divide-by +// -zero garbage. +func TestFormatLoadTableZeroRequests(t *testing.T) { + report := LoadReport{ + ByStep: loadmetrics.Report{ + Total: loadmetrics.Stats{}, + PerStep: map[loadmetrics.Key]loadmetrics.Stats{}, + }, + } + + got := FormatLoadTable(report) + + want := "" + + "Step p50 p95 p99 RPS Err%\n" + + "TOTAL 0s 0s 0s 0.0 0.0\n" + + if got != want { + t.Errorf("table mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, want) + } + if strings.Contains(got, "NaN") { + t.Errorf("table contains NaN:\n%s", got) + } +} + // TestFormatLoadTableWidensForLongStepNames keeps the columns aligned when a // step name outgrows the default column, instead of truncating it. func TestFormatLoadTableWidensForLongStepNames(t *testing.T) { @@ -266,7 +295,7 @@ func TestJSONReporterWithLoadReport(t *testing.T) { WorkerVersion string `json:"worker_version"` Report struct { Total struct { - Count string `json:"count"` + Count string `json:"count"` Rps float64 `json:"rps"` } `json:"total"` PerStep []struct { diff --git a/apps/cli/internal/runner/run_order.go b/apps/cli/internal/runner/run_order.go index 51d6a3651..45cf961aa 100644 --- a/apps/cli/internal/runner/run_order.go +++ b/apps/cli/internal/runner/run_order.go @@ -45,6 +45,22 @@ func parseRunEntries(fileData []byte) ([]runEntry, error) { return nil, fmt.Errorf("no run field found in workflow") } + // Reject a repeated flow name outright, before it ever reaches + // topoSortRunEntries. That function's byName/inDegree/declOrder maps are + // keyed by name, so a duplicate silently collapses onto the + // last-declared entry there; depending on the dependency shape, the + // resulting count mismatch can even make findCycle's "remaining" set + // land empty, producing a bare "dependency cycle in run block: " error + // that names nothing. Naming the duplicate here keeps the topological + // sort itself untouched. + seen := make(map[string]bool, len(entries)) + for _, e := range entries { + if seen[e.flowName] { + return nil, fmt.Errorf("duplicate flow %q in run block", e.flowName) + } + seen[e.flowName] = true + } + return entries, nil } diff --git a/apps/cli/internal/runner/run_order_test.go b/apps/cli/internal/runner/run_order_test.go new file mode 100644 index 000000000..44ca924e6 --- /dev/null +++ b/apps/cli/internal/runner/run_order_test.go @@ -0,0 +1,144 @@ +package runner + +import ( + "strings" + "testing" +) + +// namesOf extracts the flow names from a sorted run: block, for compact +// order assertions. +func namesOf(entries []runEntry) []string { + names := make([]string, len(entries)) + for i, e := range entries { + names[i] = e.flowName + } + return names +} + +// TestTopoSortRunEntries exercises topoSortRunEntries directly against +// hand-built runEntry values: no YAML, no flow execution, no I/O. The +// integration-shaped coverage (actual RunMultipleFlows calls against real +// flows and a mock HTTP server) lives in runner_test.go; these cases pin the +// sort/cycle-detection algorithm itself, which was previously only exercised +// indirectly through that heavier seam. +func TestTopoSortRunEntries(t *testing.T) { + tests := []struct { + name string + entries []runEntry + // wantOrder is checked when wantErrSubstrs is empty. + wantOrder []string + // wantErrSubstrs are all required to appear in the returned error; + // when non-empty, an error is required and wantOrder is ignored. + wantErrSubstrs []string + }{ + { + // Kahn's algorithm processes the initial "ready" set (every + // flow with no dependencies) in a FIFO queue seeded in + // declaration order, so with no edges to reorder anything, the + // output is exactly the input order. This is the tie-break + // that makes the sort deterministic across runs of the same + // file. + name: "3+ dependency-free flows preserve declaration order", + entries: []runEntry{ + {flowName: "Charlie"}, + {flowName: "Alpha"}, + {flowName: "Bravo"}, + }, + wantOrder: []string{"Charlie", "Alpha", "Bravo"}, + }, + { + name: "self-dependency is reported as a cycle naming the flow", + entries: []runEntry{ + {flowName: "A", dependsOn: []string{"A"}}, + }, + wantErrSubstrs: []string{"dependency cycle in run block", "A"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := topoSortRunEntries(tt.entries) + + if len(tt.wantErrSubstrs) > 0 { + if err == nil { + t.Fatalf("expected an error, got order %v", namesOf(got)) + } + for _, substr := range tt.wantErrSubstrs { + if !strings.Contains(err.Error(), substr) { + t.Errorf("error %q does not contain %q", err.Error(), substr) + } + } + return + } + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + gotOrder := namesOf(got) + if len(gotOrder) != len(tt.wantOrder) { + t.Fatalf("order = %v, want %v", gotOrder, tt.wantOrder) + } + for i, want := range tt.wantOrder { + if gotOrder[i] != want { + t.Errorf("order = %v, want %v", gotOrder, tt.wantOrder) + break + } + } + }) + } +} + +// TestParseRunEntries_DuplicateFlowName covers a defect found in review: +// topoSortRunEntries's byName/inDegree/declOrder maps are keyed by flow +// name, so a run: block that declares the same flow twice silently +// collapses onto the last-declared entry in those maps while `names` (a +// plain slice) still holds both occurrences. Depending on the dependency +// shape this either silently drops one declared run (no error at all, the +// entries collapsing case) or - the case pinned here - makes findCycle's +// "remaining" set land empty, producing a bare "dependency cycle in run +// block: " error that names nothing useful. +// +// parseRunEntries now rejects a duplicate flow name outright, before +// topoSortRunEntries ever runs, so every duplicate produces the same clear, +// named error instead of either failure mode. +func TestParseRunEntries_DuplicateFlowName(t *testing.T) { + tests := []struct { + name string + yaml string + }{ + { + name: "plain duplicate, no dependencies", + yaml: "run:\n - flow: A\n - flow: A\n", + }, + { + // The specific repro from review: both "A" entries depend on + // X, so dependents["X"] lists "A" twice for an inDegree + // counter that only starts at 1 (the last-declared "A" + // entry's dependency count). Resolving X decrements it twice, + // overshooting to -1 - inDegree never reads as "still + // pending" (>0), so findCycle's remaining set is empty and the + // old code produced "dependency cycle in run block: " with + // nothing after the colon. + name: "duplicate that used to blank out the cycle message", + yaml: "run:\n - flow: X\n - flow: A\n depends_on: X\n - flow: A\n depends_on: X\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := parseRunEntries([]byte(tt.yaml)) + if err == nil { + t.Fatal("expected an error for a duplicate flow name, got nil") + } + if !strings.Contains(err.Error(), "duplicate") { + t.Errorf("error %q does not mention 'duplicate'", err.Error()) + } + if !strings.Contains(err.Error(), `"A"`) { + t.Errorf("error %q does not name the duplicated flow %q", err.Error(), "A") + } + if strings.Contains(err.Error(), "dependency cycle in run block:") { + t.Errorf("error regressed to the blank cycle message: %q", err.Error()) + } + }) + } +} diff --git a/apps/cli/internal/runner/runner_test.go b/apps/cli/internal/runner/runner_test.go index 9ac82a0f3..f89ebc1d6 100644 --- a/apps/cli/internal/runner/runner_test.go +++ b/apps/cli/internal/runner/runner_test.go @@ -1265,6 +1265,117 @@ flows: } } +// TestRunMultipleFlows_TransitiveSkipCascade verifies two things about the +// same run in one pass, since they are two observable facets of a single +// execution: (1) a skip cascades transitively - A fails, B (depends on A) is +// skipped, and C (depends on B) is skipped too, each carrying its own +// explicit reason naming its own failed/skipped dependency rather than +// re-deriving "A failed" for every downstream flow; (2) an unrelated flow D +// with no dependency on any of them still executes. (2) is what pins the +// register row 2 continuation semantics: a failed dependency skips only its +// dependents, the run continues for everything else, where the old code +// aborted the entire run at the first failure. +func TestRunMultipleFlows_TransitiveSkipCascade(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + fixture := newFlowTestFixture(t) + + var mu sync.Mutex + var requestOrder []string + fixture.mockServer.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + requestOrder = append(requestOrder, r.URL.Path) + mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]interface{}{"status": "ok"}) + }) + + // A's request targets an address nothing listens on (port 1 is + // reserved), so it fails fast and deterministically - the same seam + // TestRunMultipleFlows_FailedDependencySkipsDependent uses. B depends on + // A, C depends on B, and D depends on nothing. + yamlContent := fmt.Sprintf(`workspace_name: Cascade Test +run: + - flow: A + - flow: B + depends_on: A + - flow: C + depends_on: B + - flow: D +flows: + - name: A + steps: + - manual_start: + name: Start + - request: + name: RequestA + method: GET + url: http://127.0.0.1:1/unreachable + depends_on: Start + - name: B + steps: + - manual_start: + name: Start + - request: + name: RequestB + method: GET + url: %s/b + depends_on: Start + - name: C + steps: + - manual_start: + name: Start + - request: + name: RequestC + method: GET + url: %s/c + depends_on: Start + - name: D + steps: + - manual_start: + name: Start + - request: + name: RequestD + method: GET + url: %s/d + depends_on: Start +`, fixture.mockServer.URL, fixture.mockServer.URL, fixture.mockServer.URL) + + flows := setupMultiFlowFixture(t, fixture, yamlContent) + + ctx, cancel := context.WithTimeout(fixture.ctx, 15*time.Second) + defer cancel() + + err := runner.RunMultipleFlows(ctx, []byte(yamlContent), flows, fixture.getRunnerServices(nil), fixture.services.Logger, nil) + if err == nil { + t.Fatal("expected a non-nil error when A fails, got nil") + } + + // Each skip must name its own dependency, not just propagate "A" to + // every downstream flow: B names A (the flow that actually failed), and + // C names B (the flow it actually depends on, itself only skipped). + if !strings.Contains(err.Error(), `B: dependency "A" failed (skipped)`) { + t.Errorf("expected B's reported reason to name A, got: %v", err) + } + if !strings.Contains(err.Error(), `C: dependency "B" failed (skipped)`) { + t.Errorf("expected C's reported reason to name B, got: %v", err) + } + // D succeeded, so it must not appear in the failure summary at all. + if strings.Contains(err.Error(), "D:") { + t.Errorf("expected D to be absent from the failure summary (it succeeded), got: %v", err) + } + + mu.Lock() + defer mu.Unlock() + if len(requestOrder) != 1 || requestOrder[0] != "/d" { + t.Errorf("expected only D's request to be attempted (B and C skipped without running), got: %v", requestOrder) + } +} + // TestRunMultipleFlows_MalformedRunEntrySurfacesError pins delta #3: a run: // entry that is not a {flow: ...} mapping (a bare scalar here) used to be // silently dropped by the old hand-rolled map[string]interface{} parser (a diff --git a/apps/cli/taskfile.yaml b/apps/cli/taskfile.yaml index a91094e20..68ce98049 100644 --- a/apps/cli/taskfile.yaml +++ b/apps/cli/taskfile.yaml @@ -44,7 +44,9 @@ tasks: DIR: dist # Use `.` (package target) + `-tags cli` so mode_cli.go is included and runCLI is wired. # Building `main.go` as the target silently drops tagged companion files in the same package. - - go build -tags cli -o {{.BIN_DIR}}/devtools-cli-{{.VERSION | default .VERSION_DEFAULT}}-{{.PLATFORM | default (printf "%s-%s" .GOOS_DEFAULT .GOARCH_DEFAULT)}}{{.BINARY_SUFFIX}} . + # -ldflags -X injects the release version into cmd.version so `devtools version` + # identifies the build instead of always printing the source placeholder. + - go build -tags cli -ldflags "-X github.com/the-dev-tools/dev-tools/apps/cli/cmd.version=v{{.VERSION | default .VERSION_DEFAULT}}" -o {{.BIN_DIR}}/devtools-cli-{{.VERSION | default .VERSION_DEFAULT}}-{{.PLATFORM | default (printf "%s-%s" .GOOS_DEFAULT .GOARCH_DEFAULT)}}{{.BINARY_SUFFIX}} . clean: desc: Clean build artifacts diff --git a/docs/superpowers/specs/2026-08-08-load-testing-design.md b/docs/superpowers/specs/2026-08-08-load-testing-design.md index b2fe5d588..71ae50136 100644 --- a/docs/superpowers/specs/2026-08-08-load-testing-design.md +++ b/docs/superpowers/specs/2026-08-08-load-testing-design.md @@ -39,6 +39,8 @@ Stresseur sells the three things that structurally cannot be self-hosted freebie ## 2. Current state (verified 2026-08-08) +_State reflects branch base `36d63065`; Phase 0 (this branch) retires the ❌ rows below for YAML schema versioning, load/iteration levers, aggregate stats, and the GitHub Action, plus known-defect rows 1–2 (assertion drop, `run:` ordering)._ + | Capability | State | Evidence | | ------------------------- | ------------------------------------------------------------------------------------------------ | --------------------------------------------------------------- | | Portable YAML flow format | ✅ Self-contained by name; no DB IDs | `packages/server/pkg/translate/yamlflowsimplev2/types.go:16-27` | @@ -290,12 +292,12 @@ property instead of a hope. Every intentional change, its blast radius, and its comms. Nothing else may change observable behavior; golden tests enforce that. -| # | Change | Who feels it | Mitigation / comms | -| --- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | YAML assertions now enforced on import | Files whose assertions were silently ignored may now fail runs — **their tests start testing** | Minor version bump; changelog headline; release note shows how to delete/adjust assertions; docs page "why did my flow start failing" | -| 2 | `run:` executes in dependency order with strict failure modes: unknown flow/dep names abort **pre-flight** (nothing executes; previously flows before the bad entry ran first), malformed `run:` entries error instead of being silently skipped, dependency-failure skips are reported explicitly, and the aggregate failure message now includes the failing flow's name and status | Files listing flows out of dep order; typo'd flow/dep names that silently skipped flows; scripts parsing the old failure-message shape | Same release; error messages name the offending value and list valid flow names; changelog enumerates all four deltas | -| 3 | `version: 2` appears in exports | None (old parsers ignore unknown keys — verified) | Changelog note | -| 4 | New CLI flags / report fields | None (additive; JSON consumers ignoring unknown fields unaffected) | Changelog note | +| # | Change | Who feels it | Mitigation / comms | +| --- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | YAML assertions now enforced on import | Files whose assertions were silently ignored may now fail runs — **their tests start testing** | Minor version bump; changelog headline; release note shows how to delete/adjust assertions; docs page "why did my flow start failing" | +| 2 | `run:` executes in dependency order with strict failure modes: unknown flow/dep names abort **pre-flight** (nothing executes; previously flows before the bad entry ran first), malformed `run:` entries error instead of being silently skipped, dependency-failure skips are reported explicitly, and the aggregate failure message now includes the failing flow's name and status; a failed dependency now **skips** the dependent and **continues** the run — previously the entire run aborted at that point, so flows after the failure now execute where previously nothing did | Files listing flows out of dep order; typo'd flow/dep names that silently skipped flows; scripts parsing the old failure-message shape | Same release; error messages name the offending value and list valid flow names; changelog enumerates all four deltas | +| 3 | `version: 2` appears in exports | None (old parsers ignore unknown keys — verified) | Changelog note | +| 4 | New CLI flags / report fields | None (additive; JSON consumers ignoring unknown fields unaffected) | Changelog note | Release as **CLI/desktop minor** via Nx version plan (never manual bumps). Explicitly _not_ in Phase 0 (logged, tracked, separate hygiene PRs): Windows JS-node diff --git a/packages/server/pkg/ioworkspace/importer_load_test.go b/packages/server/pkg/ioworkspace/importer_load_test.go index 2817a548a..d31530088 100644 --- a/packages/server/pkg/ioworkspace/importer_load_test.go +++ b/packages/server/pkg/ioworkspace/importer_load_test.go @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/require" - "github.com/the-dev-tools/dev-tools/packages/db/pkg/sqlitemem" "github.com/the-dev-tools/dev-tools/packages/db/pkg/sqlc/gen" + "github.com/the-dev-tools/dev-tools/packages/db/pkg/sqlitemem" "github.com/the-dev-tools/dev-tools/packages/server/pkg/idwrap" "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mflow" "github.com/the-dev-tools/dev-tools/packages/server/pkg/model/mload" diff --git a/packages/server/pkg/translate/yamlflowsimplev2/golden_test.go b/packages/server/pkg/translate/yamlflowsimplev2/golden_test.go index 4c77b38b0..57245b109 100644 --- a/packages/server/pkg/translate/yamlflowsimplev2/golden_test.go +++ b/packages/server/pkg/translate/yamlflowsimplev2/golden_test.go @@ -125,7 +125,12 @@ func diff(want, got []byte) string { // 2. No unintentional drift: the stable output matches the committed // .golden snapshot. func TestGoldenRoundTrip(t *testing.T) { - for _, name := range goldenCases(t) { + cases := goldenCases(t) + if len(cases) == 0 { + t.Fatal("golden corpus is empty (testdata/golden has no *.yaml fixtures): the zero-behavior-change enforcement this test exists for is a no-op until the corpus is restored") + } + + for _, name := range cases { t.Run(name, func(t *testing.T) { in := readGoldenFile(t, filepath.Join(goldenDir, name+".yaml")) From d3e2e723e84e1f9ecaec31bc0b8396d6a6a2f827 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 23:41:35 +0300 Subject: [PATCH 34/38] docs: mark version defect fixed by phase 0 --- docs/superpowers/specs/2026-08-08-load-testing-design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/superpowers/specs/2026-08-08-load-testing-design.md b/docs/superpowers/specs/2026-08-08-load-testing-design.md index 71ae50136..c7024f957 100644 --- a/docs/superpowers/specs/2026-08-08-load-testing-design.md +++ b/docs/superpowers/specs/2026-08-08-load-testing-design.md @@ -69,7 +69,7 @@ _State reflects branch base `36d63065`; Phase 0 (this branch) retires the ❌ ro | `flow.timeout` / `flow.metadata` parsed but never read | Dead schema fields | `types.go:67-73` | | GraphQL-only / WS-only exports use ad-hoc shapes that cannot be re-imported | Round-trip broken for those exports | `rexportv2/export.go:257-342` | | CLI JS nodes broken on Windows | Go dials `unix`; worker binds named pipe on win32 | `jsrunner.go:63` vs `worker-js/src/main.ts:32-36` | -| `devtools version` prints v0.1.0 (package is 1.0.3) | Cosmetic; breaks support triage | `apps/cli/cmd/version.go:13` | +| `devtools version` prints v0.1.0 (package is 1.0.3) | **Fixed by Phase 0** (ldflags injection in `build:release`) | `apps/cli/cmd/version.go:13` | ## 3. Target architecture From 7038b6e6df8934cb6cc731df9c7e7e5324fdbe32 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sat, 8 Aug 2026 23:43:21 +0300 Subject: [PATCH 35/38] chore: ignore .context workspace scratch in prettier --- .prettierignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.prettierignore b/.prettierignore index 75ed3cb82..76b408f92 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,4 +1,5 @@ .ai/ +.context/ .golangci.yml .superpowers/ *.har From a0cf61aa5c97397159a8abb83810d2d3eb593ccf Mon Sep 17 00:00:00 2001 From: moosebay Date: Sun, 9 Aug 2026 00:26:25 +0300 Subject: [PATCH 36/38] ci: assert run-flows action via report.json, not step summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit $GITHUB_STEP_SUMMARY is a unique file per step, so the later assert step could never observe the summary the composite action appended inside its own step — the greps read a freshly created empty file and the check failed deterministically on both ubuntu-latest and macos-latest. Replace the three summary greps with jq assertions on the JSON report the summary is rendered from: both smoke flows present, status success, and a non-zero duration. Those are exactly the fields write-summary.sh reads, so summary data correctness is covered transitively. A comment records why the summary must not be grepped again. --- .github/workflows/action-test.yaml | 38 ++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/action-test.yaml b/.github/workflows/action-test.yaml index e7dedbb02..d8eb72ea0 100644 --- a/.github/workflows/action-test.yaml +++ b/.github/workflows/action-test.yaml @@ -28,20 +28,38 @@ jobs: file: actions/run-flows/testdata/smoke.yamlflow.yaml version: latest - - name: Assert outputs and summary + # Do NOT assert on $GITHUB_STEP_SUMMARY here: GitHub gives every step its + # own summary file, so what the action appended inside its own step is + # invisible to this one (it would read its own empty file and always + # fail). Assert on report.json instead — write-summary.sh renders its + # table from exactly the fields checked below, so the summary's data is + # covered transitively; its formatting is covered by the script's tests. + - name: Assert outputs and report contents shell: bash + env: + JSON_REPORT: ${{ steps.run-flows.outputs.json-report }} + JUNIT_REPORT: ${{ steps.run-flows.outputs.junit-report }} + SUCCESS: ${{ steps.run-flows.outputs.success }} run: | set -euo pipefail - echo "json-report=${{ steps.run-flows.outputs.json-report }}" - echo "junit-report=${{ steps.run-flows.outputs.junit-report }}" - echo "success=${{ steps.run-flows.outputs.success }}" + echo "json-report=$JSON_REPORT" + echo "junit-report=$JUNIT_REPORT" + echo "success=$SUCCESS" - test -f "${{ steps.run-flows.outputs.json-report }}" - test -f "${{ steps.run-flows.outputs.junit-report }}" + test -f "$JSON_REPORT" + test -f "$JUNIT_REPORT" - [ "${{ steps.run-flows.outputs.success }}" = "true" ] + [ "$SUCCESS" = "true" ] - grep -q "DevTools flow run" "$GITHUB_STEP_SUMMARY" - grep -q "fetch-user" "$GITHUB_STEP_SUMMARY" - grep -q "create-post" "$GITHUB_STEP_SUMMARY" + echo "report contents:" + jq -c '[.[] | {flow_name, status, duration}]' "$JSON_REPORT" + + # Both flows from testdata/smoke.yamlflow.yaml ran, succeeded, and + # recorded a non-zero duration (nanoseconds). + jq -e 'length == 2' "$JSON_REPORT" > /dev/null + for flow in fetch-user create-post; do + jq -e --arg flow "$flow" \ + 'any(.[]; .flow_name == $flow and .status == "success" and .duration > 0)' \ + "$JSON_REPORT" > /dev/null + done From 3da7833f2b8eaeef5762da38dd2bb8f8d1fe2659 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sun, 9 Aug 2026 00:32:43 +0300 Subject: [PATCH 37/38] fix(cli): warn instead of erroring on unknown run: dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit topoSortRunEntries hard-errored on any depends_on name that was not a flow in the same run: block. Shipped example files list cross-flow *step* names there — a pattern users copied because the pre-topological-sort code silently ignored anything it did not recognise — so the hard error broke 4 of 5 CLI integration fixtures and every user file following them. Unknown depends_on names now drop out with a warning on stderr that names the offending value, the valid flow names, and the fact that step-level dependencies in run: are unsupported and ignored; execution proceeds exactly as if the name had not been written. Cycles and duplicate flow names stay hard errors — they are structurally invalid, not a compatibility surface — and a run: entry whose flow: value names a nonexistent flow still aborts pre-flight. Example fixtures are deliberately left untouched: keeping them working unmodified is the point. Behavior-change register 4.2 row 2 updated. --- apps/cli/internal/runner/run_order.go | 59 ++++++++-- apps/cli/internal/runner/run_order_test.go | 81 ++++++++++++- apps/cli/internal/runner/runner.go | 11 +- apps/cli/internal/runner/runner_test.go | 110 ++++++++++++++++-- .../specs/2026-08-08-load-testing-design.md | 17 +-- 5 files changed, 244 insertions(+), 34 deletions(-) diff --git a/apps/cli/internal/runner/run_order.go b/apps/cli/internal/runner/run_order.go index 45cf961aa..cd2781e21 100644 --- a/apps/cli/internal/runner/run_order.go +++ b/apps/cli/internal/runner/run_order.go @@ -64,15 +64,52 @@ func parseRunEntries(fileData []byte) ([]runEntry, error) { return entries, nil } +// dropUnknownDependencies removes depends_on entries that do not name a flow +// declared in the same run: block, returning the cleaned entries plus one +// human-readable warning per dropped dependency. +// +// A run: block's depends_on has only ever ordered *flows* against each other. +// Shipped example files (apps/cli/test/yamlflow/simple_run_example.yaml and +// friends) nonetheless list cross-flow *step* names there, a pattern users +// have copied because the pre-topological-sort code silently ignored anything +// it did not recognise. Hard-erroring on those names would break every such +// file at once, so an unknown name keeps its old no-op meaning and merely +// becomes visible. Structurally invalid graphs - cycles, duplicate flow names +// - remain hard errors; they are bugs, not a compatibility surface. +func dropUnknownDependencies(entries []runEntry, byName map[string]runEntry, names []string) ([]runEntry, []string) { + var warnings []string + cleaned := make([]runEntry, 0, len(entries)) + + for _, e := range entries { + kept := make([]string, 0, len(e.dependsOn)) + for _, dep := range e.dependsOn { + if _, ok := byName[dep]; ok { + kept = append(kept, dep) + continue + } + warnings = append(warnings, fmt.Sprintf( + "warning: ignoring unknown dependency %q of flow %q in run block (known flows: %s); step-level dependencies are not supported in run: and are ignored", + dep, e.flowName, strings.Join(names, ", "), + )) + } + e.dependsOn = kept + cleaned = append(cleaned, e) + } + + return cleaned, warnings +} + // topoSortRunEntries orders run: block entries so that every flow appears // after all of its dependencies, using Kahn's algorithm. Ties (multiple // flows simultaneously ready to run) are broken by original run: block // order, so the result is deterministic for a given file. // -// Returns an error naming the offending dependency if depends_on references -// a flow that is not itself part of the run: block, or naming an example -// cycle if the dependency graph is not a DAG. -func topoSortRunEntries(entries []runEntry) ([]runEntry, error) { +// depends_on names that do not match a flow in the run: block are dropped +// with a warning (see dropUnknownDependencies) and the returned entries carry +// only the surviving dependencies, so execution proceeds exactly as if the +// unknown name had not been written. Returns an error naming an example cycle +// if the remaining dependency graph is not a DAG. +func topoSortRunEntries(entries []runEntry) ([]runEntry, []string, error) { byName := make(map[string]runEntry, len(entries)) declOrder := make(map[string]int, len(entries)) names := make([]string, 0, len(entries)) @@ -82,12 +119,12 @@ func topoSortRunEntries(entries []runEntry) ([]runEntry, error) { names = append(names, e.flowName) } + entries, warnings := dropUnknownDependencies(entries, byName, names) + // byName was populated from the pre-clean entries, so its values still + // carry the dropped dependencies. Re-point it at the cleaned ones: both + // the sorted output and findCycle read dependsOn through this map. for _, e := range entries { - for _, dep := range e.dependsOn { - if _, ok := byName[dep]; !ok { - return nil, fmt.Errorf("unknown dependency %q in run block (known flows: %s)", dep, strings.Join(names, ", ")) - } - } + byName[e.flowName] = e } // dependents[X] = flows that declare a dependency on X. @@ -137,10 +174,10 @@ func topoSortRunEntries(entries []runEntry) ([]runEntry, error) { } } cycle := findCycle(byName, remaining) - return nil, fmt.Errorf("dependency cycle in run block: %s", strings.Join(cycle, " → ")) + return nil, warnings, fmt.Errorf("dependency cycle in run block: %s", strings.Join(cycle, " → ")) } - return sorted, nil + return sorted, warnings, nil } // findCycle returns one dependency cycle among the given remaining flows, as diff --git a/apps/cli/internal/runner/run_order_test.go b/apps/cli/internal/runner/run_order_test.go index 44ca924e6..246de1f87 100644 --- a/apps/cli/internal/runner/run_order_test.go +++ b/apps/cli/internal/runner/run_order_test.go @@ -30,6 +30,13 @@ func TestTopoSortRunEntries(t *testing.T) { // wantErrSubstrs are all required to appear in the returned error; // when non-empty, an error is required and wantOrder is ignored. wantErrSubstrs []string + // wantWarnSubstrs are all required to appear in the concatenated + // warnings; wantNoWarnings asserts none were produced. + wantWarnSubstrs []string + wantNoWarnings bool + // wantDependsOn, when non-nil, pins each sorted entry's surviving + // dependencies by flow name. + wantDependsOn map[string][]string }{ { // Kahn's algorithm processes the initial "ready" set (every @@ -44,7 +51,8 @@ func TestTopoSortRunEntries(t *testing.T) { {flowName: "Alpha"}, {flowName: "Bravo"}, }, - wantOrder: []string{"Charlie", "Alpha", "Bravo"}, + wantOrder: []string{"Charlie", "Alpha", "Bravo"}, + wantNoWarnings: true, }, { name: "self-dependency is reported as a cycle naming the flow", @@ -53,11 +61,63 @@ func TestTopoSortRunEntries(t *testing.T) { }, wantErrSubstrs: []string{"dependency cycle in run block", "A"}, }, + { + // Compatibility case: shipped example files (e.g. + // apps/cli/test/yamlflow/simple_run_example.yaml) list a + // cross-flow *step* name in a run: block's depends_on. That has + // never ordered anything, so it is dropped with a warning + // instead of aborting the run, and the surviving flow-to-flow + // dependency still orders the sort. + name: "unknown dependency is dropped with a warning, not an error", + entries: []runEntry{ + {flowName: "FlowA"}, + {flowName: "FlowB", dependsOn: []string{"FlowA"}}, + {flowName: "FlowC", dependsOn: []string{"RequestA", "FlowB"}}, + }, + wantOrder: []string{"FlowA", "FlowB", "FlowC"}, + wantWarnSubstrs: []string{ + `ignoring unknown dependency "RequestA"`, + `of flow "FlowC"`, + "known flows: FlowA, FlowB, FlowC", + "step-level dependencies are not supported in run: and are ignored", + }, + wantDependsOn: map[string][]string{ + "FlowA": {}, + "FlowB": {"FlowA"}, + "FlowC": {"FlowB"}, + }, + }, + { + // Dropping every dependency of a flow must leave it ready + // immediately rather than stranding it with a stale in-degree, + // which would surface as a bogus "dependency cycle" error. + name: "flow whose only dependencies are unknown still runs", + entries: []runEntry{ + {flowName: "A", dependsOn: []string{"StepOne", "StepTwo"}}, + {flowName: "B"}, + }, + wantOrder: []string{"A", "B"}, + wantWarnSubstrs: []string{ + `ignoring unknown dependency "StepOne"`, + `ignoring unknown dependency "StepTwo"`, + }, + wantDependsOn: map[string][]string{"A": {}, "B": {}}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := topoSortRunEntries(tt.entries) + got, warnings, err := topoSortRunEntries(tt.entries) + allWarnings := strings.Join(warnings, "\n") + + for _, substr := range tt.wantWarnSubstrs { + if !strings.Contains(allWarnings, substr) { + t.Errorf("warnings %q do not contain %q", allWarnings, substr) + } + } + if tt.wantNoWarnings && len(warnings) > 0 { + t.Errorf("expected no warnings, got %v", warnings) + } if len(tt.wantErrSubstrs) > 0 { if err == nil { @@ -84,6 +144,23 @@ func TestTopoSortRunEntries(t *testing.T) { break } } + + for _, e := range got { + want, pinned := tt.wantDependsOn[e.flowName] + if !pinned { + continue + } + if len(e.dependsOn) != len(want) { + t.Errorf("%s dependsOn = %v, want %v", e.flowName, e.dependsOn, want) + continue + } + for i := range want { + if e.dependsOn[i] != want[i] { + t.Errorf("%s dependsOn = %v, want %v", e.flowName, e.dependsOn, want) + break + } + } + } }) } } diff --git a/apps/cli/internal/runner/runner.go b/apps/cli/internal/runner/runner.go index ceeb11c5e..f89a4afad 100644 --- a/apps/cli/internal/runner/runner.go +++ b/apps/cli/internal/runner/runner.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "log/slog" + "os" "strings" "sync" "time" @@ -61,7 +62,15 @@ func RunMultipleFlows(ctx context.Context, fileData []byte, allFlows []mflow.Flo } } - sorted, err := topoSortRunEntries(entries) + // Unknown depends_on names are dropped with a warning rather than + // aborting the run: they were silently ignored before the run: block + // gained a topological sort, and shipped example files rely on that. + // Cycles and duplicate flow names are still hard errors. + sorted, warnings, err := topoSortRunEntries(entries) + for _, w := range warnings { + fmt.Fprintln(os.Stderr, w) + logger.Warn(w) + } if err != nil { return err } diff --git a/apps/cli/internal/runner/runner_test.go b/apps/cli/internal/runner/runner_test.go index f89ebc1d6..5d503d9a8 100644 --- a/apps/cli/internal/runner/runner_test.go +++ b/apps/cli/internal/runner/runner_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "fmt" + "io" "log/slog" "net/http" "net/http/httptest" @@ -977,13 +978,37 @@ flows: } } -// TestRunMultipleFlows_UnknownDependencyError verifies that a depends_on -// entry naming a flow that is not part of the run: block produces a specific, -// actionable error rather than being silently ignored. -func TestRunMultipleFlows_UnknownDependencyError(t *testing.T) { +// TestRunMultipleFlows_UnknownDependencyWarns pins the compatibility +// semantics for a depends_on entry that does not name a flow in the run: +// block. Such a name has never ordered anything - depends_on relates flows to +// flows - but shipped example files under apps/cli/test/yamlflow/ list +// cross-flow *step* names there, a pattern users copied because the +// pre-topological-sort code silently ignored whatever it did not recognise. +// Hard-erroring would break every one of those files, so the unknown name +// keeps its old no-op meaning and gains a warning on stderr. Cycles and +// duplicate flow names stay hard errors (see the tests below / in +// run_order_test.go); a run: entry whose flow: value names a nonexistent flow +// also stays a pre-flight error (TestRunMultipleFlows_UnknownFlowInRunBlock). +func TestRunMultipleFlows_UnknownDependencyWarns(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + fixture := newFlowTestFixture(t) - yamlContent := `workspace_name: Unknown Dep Test + var mu sync.Mutex + var requestOrder []string + fixture.mockServer.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + requestOrder = append(requestOrder, r.URL.Path) + mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]interface{}{"status": "ok"}) + }) + + yamlContent := fmt.Sprintf(`workspace_name: Unknown Dep Test run: - flow: A - flow: B @@ -993,24 +1018,85 @@ flows: steps: - manual_start: name: Start + - request: + name: RequestA + method: GET + url: %s/a + depends_on: Start - name: B steps: - manual_start: name: Start -` + - request: + name: RequestB + method: GET + url: %s/b + depends_on: Start +`, fixture.mockServer.URL, fixture.mockServer.URL) + flows := setupMultiFlowFixture(t, fixture, yamlContent) - err := runner.RunMultipleFlows(fixture.ctx, []byte(yamlContent), flows, fixture.getRunnerServices(nil), fixture.services.Logger, nil) - if err == nil { - t.Fatal("expected an error for an unknown dependency, got nil") + ctx, cancel := context.WithTimeout(fixture.ctx, 15*time.Second) + defer cancel() + + stderr := captureStderr(t, func() { + if err := runner.RunMultipleFlows(ctx, []byte(yamlContent), flows, fixture.getRunnerServices(nil), fixture.services.Logger, nil); err != nil { + t.Errorf("unknown dependency must not abort the run, got error: %v", err) + } + }) + + const wantWarning = `warning: ignoring unknown dependency "Missing" of flow "B" in run block ` + + `(known flows: A, B); step-level dependencies are not supported in run: and are ignored` + if !strings.Contains(stderr, wantWarning) { + t.Errorf("stderr does not contain the expected warning:\n got: %q\n want: %q", stderr, wantWarning) } - const want = `unknown dependency "Missing" in run block (known flows: A, B)` - if err.Error() != want { - t.Errorf("unexpected error message:\n got: %q\n want: %q", err.Error(), want) + // The dropped dependency must not gate B: both flows run, and with no + // surviving edges the order is plain declaration order. + mu.Lock() + defer mu.Unlock() + if len(requestOrder) != 2 { + t.Fatalf("expected both flows to execute (2 requests), got %d: %v", len(requestOrder), requestOrder) + } + if requestOrder[0] != "/a" || requestOrder[1] != "/b" { + t.Errorf("expected declaration order [/a, /b] once the unknown dep is dropped, got %v", requestOrder) } } +// captureStderr redirects os.Stderr through a pipe for the duration of fn and +// returns everything written to it. The pipe is drained on a goroutine so a +// writer producing more than the pipe buffer cannot deadlock. +func captureStderr(t *testing.T, fn func()) string { + t.Helper() + + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("failed to create pipe: %v", err) + } + + orig := os.Stderr + os.Stderr = w + + done := make(chan string, 1) + go func() { + var buf strings.Builder + _, _ = io.Copy(&buf, r) + done <- buf.String() + }() + + func() { + defer func() { + os.Stderr = orig + _ = w.Close() + }() + fn() + }() + + out := <-done + _ = r.Close() + return out +} + // TestRunMultipleFlows_CyclicDependencyError verifies that a dependency cycle // in the run: block is rejected with a message naming the cycle, instead of // hanging or silently running flows in an arbitrary order. diff --git a/docs/superpowers/specs/2026-08-08-load-testing-design.md b/docs/superpowers/specs/2026-08-08-load-testing-design.md index c7024f957..bef069d09 100644 --- a/docs/superpowers/specs/2026-08-08-load-testing-design.md +++ b/docs/superpowers/specs/2026-08-08-load-testing-design.md @@ -268,8 +268,9 @@ property instead of a hope. `KnownFields` anywhere). 2. **Fix assertion import drop** (`converter_node.go`): populate `HTTPAsserts` the way GraphQL already does; add round-trip assertion tests. ⚠️ Behavior change — see 4.2. -3. **Fix `run:` ordering**: topological sort from `depends_on`; **error** on unknown - dep names instead of silent skip. ⚠️ Behavior change — see 4.2. +3. **Fix `run:` ordering**: topological sort from `depends_on`; **warn** on unknown dep + names (still ignored, as before) and **error** on cycles and duplicate flow names. + ⚠️ Behavior change — see 4.2. 4. **Engine levers**: `CreateFlowRunner` functional options (`WithMaxConcurrency`); RunProfile scheduler skeleton (constant-vus only in Phase 0); lean execution mode flag threaded through node request context. Defaults preserve current behavior @@ -292,12 +293,12 @@ property instead of a hope. Every intentional change, its blast radius, and its comms. Nothing else may change observable behavior; golden tests enforce that. -| # | Change | Who feels it | Mitigation / comms | -| --- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | YAML assertions now enforced on import | Files whose assertions were silently ignored may now fail runs — **their tests start testing** | Minor version bump; changelog headline; release note shows how to delete/adjust assertions; docs page "why did my flow start failing" | -| 2 | `run:` executes in dependency order with strict failure modes: unknown flow/dep names abort **pre-flight** (nothing executes; previously flows before the bad entry ran first), malformed `run:` entries error instead of being silently skipped, dependency-failure skips are reported explicitly, and the aggregate failure message now includes the failing flow's name and status; a failed dependency now **skips** the dependent and **continues** the run — previously the entire run aborted at that point, so flows after the failure now execute where previously nothing did | Files listing flows out of dep order; typo'd flow/dep names that silently skipped flows; scripts parsing the old failure-message shape | Same release; error messages name the offending value and list valid flow names; changelog enumerates all four deltas | -| 3 | `version: 2` appears in exports | None (old parsers ignore unknown keys — verified) | Changelog note | -| 4 | New CLI flags / report fields | None (additive; JSON consumers ignoring unknown fields unaffected) | Changelog note | +| # | Change | Who feels it | Mitigation / comms | +| --- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | YAML assertions now enforced on import | Files whose assertions were silently ignored may now fail runs — **their tests start testing** | Minor version bump; changelog headline; release note shows how to delete/adjust assertions; docs page "why did my flow start failing" | +| 2 | `run:` executes in dependency order with strict failure modes: an unknown `flow:` name aborts **pre-flight** (nothing executes; previously flows before the bad entry ran first), while an unknown name inside `depends_on` is **ignored with a warning** on stderr exactly as before — shipped example files put cross-flow _step_ names there and must keep working, so only cycles and duplicate flow names are hard errors; malformed `run:` entries error instead of being silently skipped, dependency-failure skips are reported explicitly, and the aggregate failure message now includes the failing flow's name and status; a failed dependency now **skips** the dependent and **continues** the run — previously the entire run aborted at that point, so flows after the failure now execute where previously nothing did | Files listing flows out of dep order; typo'd `flow:` names that silently skipped flows; typo'd `depends_on` names (unchanged behavior, now visible); scripts parsing the old failure-message shape | Same release; warnings and errors name the offending value and list valid flow names; changelog enumerates all four deltas | +| 3 | `version: 2` appears in exports | None (old parsers ignore unknown keys — verified) | Changelog note | +| 4 | New CLI flags / report fields | None (additive; JSON consumers ignoring unknown fields unaffected) | Changelog note | Release as **CLI/desktop minor** via Nx version plan (never manual bumps). Explicitly _not_ in Phase 0 (logged, tracked, separate hygiene PRs): Windows JS-node From af1ea705bc424b2b6bbb4fc62c740adf4647fce1 Mon Sep 17 00:00:00 2001 From: moosebay Date: Sun, 9 Aug 2026 00:37:35 +0300 Subject: [PATCH 38/38] fix(cli): cross-compile release binaries for the platform they are named after MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build:release interpolated PLATFORM into the output filename only. GOOS and GOARCH were never set, so every matrix row in release-go.yaml built for the architecture of its own runner and simply renamed the result. Both cross-compiled rows share a runner with their x64 sibling, so cli@1.0.3 published a linux-arm64 asset and a win32-ia32 asset that were byte-identical x86-64 binaries and could not run on the platforms they advertised. Derive GOOS/GOARCH from PLATFORM (darwin/linux pass through, win32 to windows; x64 to amd64, ia32 to 386, arm64 to arm64) and export them next to the existing CGO_ENABLED=0 — the CLI is pure Go, so every target cross-compiles. Also derive BINARY_SUFFIX=.exe for windows targets, which release-go.yaml never sets, so Windows assets stop shipping extensionless. An explicitly passed BINARY_SUFFIX still wins, and with no PLATFORM the build falls back to the host toolchain exactly as before. Verified locally: linux-arm64 is ELF ARM aarch64, linux-x64 is ELF x86-64, win32-ia32.exe is PE32 Intel 80386, all with distinct checksums. Existing 1.0.3 assets are left untouched; 1.1.0 ships correct ones. --- apps/cli/taskfile.yaml | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/apps/cli/taskfile.yaml b/apps/cli/taskfile.yaml index 68ce98049..dc6a361a0 100644 --- a/apps/cli/taskfile.yaml +++ b/apps/cli/taskfile.yaml @@ -29,8 +29,6 @@ tasks: build:release: desc: Build release binary with version and platform - env: - CGO_ENABLED: 0 vars: VERSION_DEFAULT: sh: node -e "console.log(require('./package.json').version)" @@ -38,6 +36,33 @@ tasks: sh: go env GOOS GOARCH_DEFAULT: sh: go env GOARCH + # PLATFORM arrives from release-go.yaml's build matrix as "-" + # in Node/Electron naming: darwin-x64, darwin-arm64, linux-x64, + # linux-arm64, win32-x64, win32-ia32. + # + # It used to be interpolated into the output *filename* only, so a + # matrix row always built for whatever architecture its runner happened + # to be and merely named the result after the platform it claimed. Both + # cross-compiled rows share a runner with their x64 sibling, so + # cli@1.0.3 published linux-arm64 and win32-ia32 assets that were + # byte-identical x86-64 binaries. Translate PLATFORM into GOOS/GOARCH so + # the binary matches the name it ships under. The CLI is pure Go with + # CGO_ENABLED=0, so every one of these targets cross-compiles. + # + # With no PLATFORM (local dev), both fall back to the host toolchain's + # own values and the build is byte-for-byte what it was before. + PLATFORM_OS: '{{if .PLATFORM}}{{splitList "-" .PLATFORM | first}}{{end}}' + PLATFORM_ARCH: '{{if .PLATFORM}}{{splitList "-" .PLATFORM | last}}{{end}}' + GOOS_TARGET: '{{if .PLATFORM_OS}}{{if eq .PLATFORM_OS "win32"}}windows{{else}}{{.PLATFORM_OS}}{{end}}{{else}}{{.GOOS_DEFAULT}}{{end}}' + GOARCH_TARGET: '{{if .PLATFORM_ARCH}}{{if eq .PLATFORM_ARCH "x64"}}amd64{{else if eq .PLATFORM_ARCH "ia32"}}386{{else}}{{.PLATFORM_ARCH}}{{end}}{{else}}{{.GOARCH_DEFAULT}}{{end}}' + # Windows assets need .exe or they publish as extensionless PE files + # that Windows refuses to execute until renamed. release-go.yaml never + # sets BINARY_SUFFIX, so derive it; an explicitly passed value wins. + SUFFIX: '{{if .BINARY_SUFFIX}}{{.BINARY_SUFFIX}}{{else if eq .GOOS_TARGET "windows"}}.exe{{end}}' + env: + CGO_ENABLED: 0 + GOOS: '{{.GOOS_TARGET}}' + GOARCH: '{{.GOARCH_TARGET}}' cmds: - task: ensure-dir vars: @@ -46,7 +71,7 @@ tasks: # Building `main.go` as the target silently drops tagged companion files in the same package. # -ldflags -X injects the release version into cmd.version so `devtools version` # identifies the build instead of always printing the source placeholder. - - go build -tags cli -ldflags "-X github.com/the-dev-tools/dev-tools/apps/cli/cmd.version=v{{.VERSION | default .VERSION_DEFAULT}}" -o {{.BIN_DIR}}/devtools-cli-{{.VERSION | default .VERSION_DEFAULT}}-{{.PLATFORM | default (printf "%s-%s" .GOOS_DEFAULT .GOARCH_DEFAULT)}}{{.BINARY_SUFFIX}} . + - go build -tags cli -ldflags "-X github.com/the-dev-tools/dev-tools/apps/cli/cmd.version=v{{.VERSION | default .VERSION_DEFAULT}}" -o {{.BIN_DIR}}/devtools-cli-{{.VERSION | default .VERSION_DEFAULT}}-{{.PLATFORM | default (printf "%s-%s" .GOOS_DEFAULT .GOARCH_DEFAULT)}}{{.SUFFIX}} . clean: desc: Clean build artifacts