From 9f38417b492e951fa03d1207f218c5bed15b2b69 Mon Sep 17 00:00:00 2001 From: Thomas Rooney Date: Thu, 8 Oct 2026 00:12:30 +0100 Subject: [PATCH 01/23] feat(go,terraform): mask sensitive body values in HTTP logs Values marked x-speakeasy-param-sensitive or format: password in request and response bodies could appear in HTTP debug logs and error messages. Value-based redaction only covers values known before the request, so a newly returned secret was logged before it could be recognised. Generate a per-operation schema graph of sensitive request and response body locations into a new public redact package. Operations with sensitive values tag their request context with the operation ID, so transports and custom HTTP clients can mask bodies by schema path. APIError messages and the Terraform provider's logging transport and diagnostics mask bodies with it. Methods of operations without sensitive values are unchanged. --- .changesets/1791413358-125ef0df.yaml | 12 ++ templates/perms.go | 5 + .../{{getDefaultErrorFileName}}.go.stmpl | 12 ++ .../{{getRedactLocation}}/redact.go.stmpl | 181 +++++++++++++++++ .../{{getUtilsLocation}}/utils.go.stmpl | 11 + templates/templates/go/includes/imports.ts | 5 + templates/templates/go/includes/includes.ts | 1 + templates/templates/go/includes/redact.ts | 190 ++++++++++++++++++ templates/templates/go/method.go.stmpl | 6 +- .../primary/redact_additional_test.go.stmpl | 93 +++++++++ .../internal/provider/utils.go.stmpl | 32 ++- .../provider/sensitive_body_test.go.stmpl | 71 +++++++ .../primary/sensitive-body-redaction.yaml | 166 +++++++++++++++ zSDKs/sdk-cli/.speakeasy/gen.lock | 2 + zSDKs/sdk-cli/internal/sdk/redact/redact.go | 179 +++++++++++++++++ zSDKs/sdk-go/.speakeasy/gen.lock | 2 + zSDKs/sdk-go/redact/redact.go | 179 +++++++++++++++++ .../.speakeasy/gen.lock | 12 +- .../internal/provider/sensitive_body_test.go | 73 +++++++ .../internal/provider/utils.go | 32 ++- .../internal/sdk/internal/utils/utils.go | 7 + .../internal/sdk/models/errors/sdkerror.go | 8 +- .../internal/sdk/redact/redact.go | 183 +++++++++++++++++ .../internal/sdk/sdk.go | 3 + 24 files changed, 1446 insertions(+), 19 deletions(-) create mode 100644 .changesets/1791413358-125ef0df.yaml create mode 100644 templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl create mode 100644 templates/templates/go/includes/redact.ts create mode 100644 templates/templates/go/tests/primary/redact_additional_test.go.stmpl create mode 100644 templates/templates/terraform/tests/review/internal/provider/sensitive_body_test.go.stmpl create mode 100644 tests/specs/fragments/primary/sensitive-body-redaction.yaml create mode 100644 zSDKs/sdk-cli/internal/sdk/redact/redact.go create mode 100644 zSDKs/sdk-go/redact/redact.go create mode 100644 zSDKs/terraform-provider-testing/internal/provider/sensitive_body_test.go create mode 100644 zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go diff --git a/.changesets/1791413358-125ef0df.yaml b/.changesets/1791413358-125ef0df.yaml new file mode 100644 index 000000000..ca7614aeb --- /dev/null +++ b/.changesets/1791413358-125ef0df.yaml @@ -0,0 +1,12 @@ +id: 1791413358-125ef0df +features: + - core +targets: + - cli + - go + - terraform +type: feat +bump: minor +description: mask sensitive request and response body values in HTTP logs and API error messages +author: ThomasRooney +date: "2026-10-07" diff --git a/templates/perms.go b/templates/perms.go index a68f0cd1f..26602cc53 100644 --- a/templates/perms.go +++ b/templates/perms.go @@ -552,6 +552,8 @@ var fileMode = map[string]fs.FileMode{ "templates/go/auxiliary/{{getPollingLocation}}/config.go.stmpl": 0o644, "templates/go/auxiliary/{{getPollingLocation}}/errors.go.stmpl": 0o644, "templates/go/auxiliary/{{getPollingLocation}}/option.go.stmpl": 0o644, + "templates/go/auxiliary/{{getRedactLocation}}": 0o20000000755, + "templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl": 0o644, "templates/go/auxiliary/{{getRetryLocation}}": 0o20000000755, "templates/go/auxiliary/{{getRetryLocation}}/config.go.stmpl": 0o644, "templates/go/auxiliary/{{getTypesLocation}}": 0o20000000755, @@ -613,6 +615,7 @@ var fileMode = map[string]fs.FileMode{ "templates/go/includes/operations.ts": 0o644, "templates/go/includes/options.ts": 0o644, "templates/go/includes/polling.ts": 0o644, + "templates/go/includes/redact.ts": 0o644, "templates/go/includes/responses.ts": 0o644, "templates/go/includes/retractions.ts": 0o644, "templates/go/includes/sanitization.ts": 0o644, @@ -686,6 +689,7 @@ var fileMode = map[string]fs.FileMode{ "templates/go/tests/primary/parameter_additional_test.go.stmpl": 0o644, "templates/go/tests/primary/parameter_encoding_additional_test.go.stmpl": 0o644, "templates/go/tests/primary/polling_additional_test.go.stmpl": 0o644, + "templates/go/tests/primary/redact_additional_test.go.stmpl": 0o644, "templates/go/tests/primary/requestbodies_additional_test.go.stmpl": 0o644, "templates/go/tests/primary/responsebodies_additional_test.go.stmpl": 0o644, "templates/go/tests/primary/retries_test.go.stmpl": 0o644, @@ -2146,6 +2150,7 @@ var fileMode = map[string]fs.FileMode{ "templates/terraform/tests/review/internal": 0o20000000755, "templates/terraform/tests/review/internal/provider": 0o20000000755, "templates/terraform/tests/review/internal/provider/http_client_test.go.stmpl": 0o644, + "templates/terraform/tests/review/internal/provider/sensitive_body_test.go.stmpl": 0o644, "templates/terraform/tests/review/internal/provider/sensitive_values_test.go.stmpl": 0o644, "templates/terraform/tests/review/internal/provider/utils_test.go.stmpl": 0o644, "templates/terraform/tsconfig.json": 0o644, diff --git a/templates/templates/go/auxiliary/{{getErrorsLocation}}/{{getDefaultErrorFileName}}.go.stmpl b/templates/templates/go/auxiliary/{{getErrorsLocation}}/{{getDefaultErrorFileName}}.go.stmpl index d22720a7a..11bd54f08 100644 --- a/templates/templates/go/auxiliary/{{getErrorsLocation}}/{{getDefaultErrorFileName}}.go.stmpl +++ b/templates/templates/go/auxiliary/{{getErrorsLocation}}/{{getDefaultErrorFileName}}.go.stmpl @@ -4,6 +4,10 @@ package {{sanitizeModelPackageName .Global.Config.Imports.GetErrorsPath false}} import ( "fmt" "net/http" + {{- if hasAnyRedactGraphs}} + + "{{getRootModulePath}}/{{getRedactLocation}}" + {{- end}} ) type {{$errorName}} struct { @@ -27,7 +31,15 @@ func New{{$errorName}}(message string, statusCode int, body string, httpRes *htt func (e *{{$errorName}}) Error() string { body := "" if len(e.Body) > 0 { + {{- if hasAnyRedactGraphs}} + redacted := e.Body + if e.RawResponse != nil && e.RawResponse.Request != nil { + redacted = string(redact.ResponseBody(e.RawResponse.Request.Context(), []byte(e.Body))) + } + body = fmt.Sprintf("\n%s", redacted) + {{- else}} body = fmt.Sprintf("\n%s", e.Body) + {{- end}} } return fmt.Sprintf("%s: Status %d%s", e.Message, e.StatusCode, body) diff --git a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl new file mode 100644 index 000000000..d277a9864 --- /dev/null +++ b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl @@ -0,0 +1,181 @@ +// Package redact masks values marked sensitive in the API description +// (x-speakeasy-param-sensitive or format: password) in HTTP bodies, so that +// HTTP clients and transports can log requests and responses safely. +// +// The SDK attaches the operation ID to the context of each request whose +// operation has sensitive request or response values. A transport reads it +// back from the request, or from a response's Request, to select the schema +// graph used for masking: +// +// body = redact.RequestBody(req.Context(), body) +// body = redact.ResponseBody(res.Request.Context(), body) +package redact + +import ( + "bytes" + "context" + "encoding/json" + "io" +) + +// Mask replaces each sensitive value. +const Mask = "(sensitive)" + +const maxDepth = 64 + +type operationKey struct{} + +// WithOperation returns ctx carrying operationID. +func WithOperation(ctx context.Context, operationID string) context.Context { + return context.WithValue(ctx, operationKey{}, operationID) +} + +// Operation returns the operation ID carried by ctx, or "". +func Operation(ctx context.Context) string { + if ctx == nil { + return "" + } + id, _ := ctx.Value(operationKey{}).(string) + return id +} + +// RequestBody masks the sensitive values of the request body of the +// operation carried by ctx. Bodies of other operations are returned as is. +func RequestBody(ctx context.Context, body []byte) []byte { + return operations[Operation(ctx)].request.redact(body) +} + +// ResponseBody masks the sensitive values of a response body of the +// operation carried by ctx, matching every response schema of the operation. +// Bodies of other operations are returned as is. +func ResponseBody(ctx context.Context, body []byte) []byte { + return operations[Operation(ctx)].response.redact(body) +} + +// A graph describes where sensitive values sit in a body. Node 0 is a +// sentinel for "nothing sensitive below here"; roots are the body schemas. +type graph struct { + nodes []node + roots []int +} + +type node struct { + sensitive bool + // fields maps JSON property names to their schema; a property that is not + // declared uses values (additional properties or map values). + fields map[string]int + item int + values int + variants []int +} + +// redact returns body unchanged when it has nothing to mask. A body that +// cannot be parsed as JSON is masked as a whole. +func (g *graph) redact(body []byte) []byte { + if g == nil || len(body) == 0 { + return body + } + for _, id := range g.expand(g.roots) { + if g.nodes[id].sensitive { + return []byte(Mask) + } + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var value any + if err := dec.Decode(&value); err != nil { + return []byte(Mask) + } + if _, err := dec.Token(); err != io.EOF { + return []byte(Mask) + } + masked, changed := g.mask(value, g.roots, 0) + if !changed { + return body + } + var out bytes.Buffer + enc := json.NewEncoder(&out) + enc.SetEscapeHTML(false) + if err := enc.Encode(masked); err != nil { + return []byte(Mask) + } + return bytes.TrimSuffix(out.Bytes(), []byte("\n")) +} + +func (g *graph) mask(value any, roots []int, depth int) (any, bool) { + roots = g.expand(roots) + if len(roots) == 0 { + return value, false + } + for _, id := range roots { + if g.nodes[id].sensitive { + return Mask, true + } + } + if depth >= maxDepth { + return Mask, true + } + changed := false + switch v := value.(type) { + case map[string]any: + for key, child := range v { + var childRoots []int + for _, id := range roots { + n := g.nodes[id] + childID, declared := n.fields[key] + if !declared { + childID = n.values + } + if childID != 0 { + childRoots = append(childRoots, childID) + } + } + if masked, ok := g.mask(child, childRoots, depth+1); ok { + v[key] = masked + changed = true + } + } + case []any: + var items []int + for _, id := range roots { + if item := g.nodes[id].item; item != 0 { + items = append(items, item) + } + } + for i, child := range v { + if masked, ok := g.mask(child, items, depth+1); ok { + v[i] = masked + changed = true + } + } + } + return value, changed +} + +// expand adds the union variants of roots, since the variant a value matches +// is not known. +func (g *graph) expand(roots []int) []int { + var expanded []int + seen := make(map[int]bool) + var visit func(int) + visit = func(id int) { + if id <= 0 || id >= len(g.nodes) || seen[id] { + return + } + seen[id] = true + expanded = append(expanded, id) + for _, variant := range g.nodes[id].variants { + visit(variant) + } + } + for _, id := range roots { + visit(id) + } + return expanded +} + +var operations = map[string]struct{ request, response *graph }{ + {{- range templateRedactGraphs}} + {{.OperationID}}: { {{- if .Request}}request: {{.Request}},{{end}}{{if .Response}} response: {{.Response}},{{end -}} }, + {{- end}} +} diff --git a/templates/templates/go/auxiliary/{{getUtilsLocation}}/utils.go.stmpl b/templates/templates/go/auxiliary/{{getUtilsLocation}}/utils.go.stmpl index 8187439f8..a304fedf0 100644 --- a/templates/templates/go/auxiliary/{{getUtilsLocation}}/utils.go.stmpl +++ b/templates/templates/go/auxiliary/{{getUtilsLocation}}/utils.go.stmpl @@ -20,6 +20,9 @@ import ( "github.com/ericlagergren/decimal" {{end}} "{{getRootModulePath}}/{{getNullableLocation}}" + {{- if hasAnyRedactGraphs}} + "{{getRootModulePath}}/{{getRedactLocation}}" + {{- end}} ) const ( @@ -28,6 +31,14 @@ const ( pathParamTagKey = "pathParam" ) +{{- if hasAnyRedactGraphs}} +// WithOperation marks requests built from ctx as belonging to operationID, so +// that transports can mask the operation's sensitive values when logging. +func WithOperation(ctx context.Context, operationID string) context.Context { + return redact.WithOperation(ctx, operationID) +} +{{end}} + var ( paramRegex = regexp.MustCompile(`({.*?})`) SerializationMethodToContentType = map[string]string{ diff --git a/templates/templates/go/includes/imports.ts b/templates/templates/go/includes/imports.ts index 60a11ddc4..484862d88 100644 --- a/templates/templates/go/includes/imports.ts +++ b/templates/templates/go/includes/imports.ts @@ -56,6 +56,10 @@ function addImport( imp = getRetryLocation(); local = true; break; + case "redact": + imp = getRedactLocation(); + local = true; + break; case "config": imp = `${getInternalPackageName()}/config`; local = true; @@ -208,6 +212,7 @@ function resolveNamespaceAlias(outputLocation: string): string { getUtilsLocation(), getPollingLocation(), getRetryLocation(), + getRedactLocation(), getNullableLocation(), ] .filter(Boolean) diff --git a/templates/templates/go/includes/includes.ts b/templates/templates/go/includes/includes.ts index 26a0c7e09..3e0635a3e 100644 --- a/templates/templates/go/includes/includes.ts +++ b/templates/templates/go/includes/includes.ts @@ -35,3 +35,4 @@ require("includes/polling.ts"); require("includes/options.ts"); require("includes/open-unions.ts"); require("includes/unions.ts"); +require("includes/redact.ts"); diff --git a/templates/templates/go/includes/redact.ts b/templates/templates/go/includes/redact.ts new file mode 100644 index 000000000..b9bab7cff --- /dev/null +++ b/templates/templates/go/includes/redact.ts @@ -0,0 +1,190 @@ +function getRedactLocation() { + return getTypesLocation().replace(/types$/, "redact"); +} +registerTemplateFunc("getRedactLocation", getRedactLocation); + +function isSensitiveBodyType(type: TypeDef): boolean { + const marker = type.Extensions?.All?.["x-speakeasy-param-sensitive"]; + return marker === true || marker === "true" || type.Format === "password"; +} + +interface RedactNode { + Sensitive?: boolean; + // A stream body is masked as a whole when any of its items holds a + // sensitive value, since a stream is not parsed item by item. + Stream?: boolean; + Fields?: Map; + Item?: number; + Values?: number; + Variants?: number[]; +} + +interface RedactGraph { + Nodes: RedactNode[]; + Roots: number[]; +} + +function newRedactGraph(): RedactGraph { + return { Nodes: [{}], Roots: [] }; +} + +function addRedactRoot(graph: RedactGraph, type: TypeDef | undefined) { + const walked = new Map(); + const walk = (type: TypeDef | undefined): number => { + if (!type) return 0; + const typeID = type.IsCustomType() ? getUniqueID(type) : ""; + if (typeID && walked.has(typeID)) return walked.get(typeID)!; + const id = graph.Nodes.length; + const node: RedactNode = {}; + graph.Nodes.push(node); + if (typeID) walked.set(typeID, id); + if (isSensitiveBodyType(type)) { + node.Sensitive = true; + return id; + } + switch (type.Type.toString()) { + case "class": + case "error": + node.Fields = new Map(); + for (const field of type.Fields || []) { + if (field.IsAdditionalProperties) + node.Values = walk(field.Type.ItemType); + else + node.Fields.set( + originalFieldName(field), + field.Const ? 0 : walk(field.Type), + ); + } + break; + case "array": + case "set": + node.Item = walk(type.ItemType); + break; + case "map": + node.Values = walk(type.ItemType); + break; + case "event-stream": + case "jsonl": + node.Stream = true; + node.Item = walk(type.ItemType); + break; + case "union": + node.Variants = (type.AssociatedTypes || []).map(walk); + break; + } + return id; + }; + graph.Roots.push(walk(type)); +} + +function renderRedactGraph(graph: RedactGraph): string | undefined { + const reachable = new Set(); + graph.Nodes.forEach((node, i) => { + if (node.Sensitive) reachable.add(i); + }); + let changed = true; + while (changed) { + changed = false; + graph.Nodes.forEach((node, i) => { + if (reachable.has(i)) return; + const children = [ + ...(node.Fields?.values() || []), + ...(node.Variants || []), + node.Item || 0, + node.Values || 0, + ]; + if (children.some((child) => reachable.has(child))) { + reachable.add(i); + changed = true; + } + }); + } + const roots = graph.Roots.filter((root) => reachable.has(root)); + if (roots.length === 0) return undefined; + const indices = [...reachable].sort((a, b) => a - b); + const remap = new Map(indices.map((id, i) => [id, i + 1])); + const nodes = indices.map((id) => { + const node = graph.Nodes[id]; + if (node.Stream) return "{ sensitive: true }"; + const parts: string[] = []; + if (node.Sensitive) parts.push("sensitive: true"); + const values = node.Values && reachable.has(node.Values) ? node.Values : 0; + // With sensitive map values, declared fields are listed even when plain so + // that they never fall back to the map-value schema. + const fields = [...(node.Fields || [])] + .filter(([, child]) => values || reachable.has(child)) + .map( + ([name, child]) => `${JSON.stringify(name)}: ${remap.get(child) || 0}`, + ); + if (fields.length) + parts.push(`fields: map[string]int{ ${fields.join(", ")} }`); + if (node.Item && reachable.has(node.Item)) + parts.push(`item: ${remap.get(node.Item)}`); + if (values) parts.push(`values: ${remap.get(values)}`); + const variants = (node.Variants || []).filter((child) => + reachable.has(child), + ); + if (variants.length) + parts.push( + `variants: []int{ ${variants + .map((child) => remap.get(child)) + .join(", ")} }`, + ); + return `{ ${parts.join(", ")} }`; + }); + return `&graph{ nodes: []node{ {}, ${nodes.join( + ", ", + )} }, roots: []int{ ${roots.map((id) => remap.get(id)).join(", ")} } }`; +} + +function redactGraphs(op: Operation): { + Request?: string; + Response?: string; +} { + const request = newRedactGraph(); + const body = getRequestBody(op); + addRedactRoot( + request, + body?.Type === "request" ? body.Value.RequestBody?.Type : body?.Value.Type, + ); + const response = newRedactGraph(); + for (const sub of op.Response?.Responses || []) { + for (const content of sub.Content || []) { + addRedactRoot(response, content.Content?.Type); + } + } + return { + Request: renderRedactGraph(request), + Response: renderRedactGraph(response), + }; +} + +function hasRedactGraphs(op: Operation): boolean { + const graphs = redactGraphs(op); + return !!(graphs.Request || graphs.Response); +} +registerTemplateFunc("hasRedactGraphs", hasRedactGraphs); + +function templateRedactGraphs(): { + OperationID: string; + Request?: string; + Response?: string; +}[] { + const entries: { + OperationID: string; + Request?: string; + Response?: string; + }[] = []; + for (const op of allOperations()) { + const graphs = redactGraphs(op); + if (graphs.Request || graphs.Response) + entries.push({ OperationID: JSON.stringify(op.ID), ...graphs }); + } + return entries.sort((a, b) => a.OperationID.localeCompare(b.OperationID)); +} +registerTemplateFunc("templateRedactGraphs", templateRedactGraphs); + +function hasAnyRedactGraphs(): boolean { + return templateRedactGraphs().length > 0; +} +registerTemplateFunc("hasAnyRedactGraphs", hasAnyRedactGraphs); diff --git a/templates/templates/go/method.go.stmpl b/templates/templates/go/method.go.stmpl index 67b2eb318..40ce30c7b 100644 --- a/templates/templates/go/method.go.stmpl +++ b/templates/templates/go/method.go.stmpl @@ -171,7 +171,8 @@ func (s *{{sanitizeClassName $op.OwningSDK.Type.Name}}) {{$methodName}}(ctx cont {{- end}} } - req, err := http.NewRequestWithContext(ctx, "{{upper $op.Method}}", opURL, {{if and $op.Request $op.Request.RequestBody}}bodyReader{{else}}nil{{end}}) + {{if hasRedactGraphs $op}}{{addImport "utils"}}ctx = utils.WithOperation(ctx, "{{$op.ID}}") + {{end}}req, err := http.NewRequestWithContext(ctx, "{{upper $op.Method}}", opURL, {{if and $op.Request $op.Request.RequestBody}}bodyReader{{else}}nil{{end}}) if err != nil { {{templateErrorReturn $op "fmt.Errorf(\"error creating request: %w\", err)"}} } @@ -780,7 +781,8 @@ func (s *{{sanitizeClassName $op.OwningSDK.Type.Name}}) {{$methodName}}(ctx cont {{- end}} } - req, err := http.NewRequestWithContext(ctx, "{{upper $op.Method}}", opURL, {{if and $op.Request $op.Request.RequestBody}}bodyReader{{else}}nil{{end}}) + {{if hasRedactGraphs $op}}{{addImport "utils"}}ctx = utils.WithOperation(ctx, "{{$op.ID}}") + {{end}}req, err := http.NewRequestWithContext(ctx, "{{upper $op.Method}}", opURL, {{if and $op.Request $op.Request.RequestBody}}bodyReader{{else}}nil{{end}}) if err != nil { {{templateErrorReturn $op "fmt.Errorf(\"error creating request: %w\", err)"}} } diff --git a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl new file mode 100644 index 000000000..19f0a1d79 --- /dev/null +++ b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl @@ -0,0 +1,93 @@ +package tests + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + + sdk "{{getRootModulePath}}" + "{{getRootModulePath}}/{{getRedactLocation}}" + "{{getRootModulePath}}/pkg/models/apierrors" + "{{getRootModulePath}}/pkg/models/shared" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const sensitiveBodyPayload = `{ + "name": "visible-name", + "password": "secret-password", + "token": "secret-token", + "nested": {"label": "visible-nested", "secret": "secret-nested"}, + "items": [{"label": "visible-item", "secret": "secret-item"}], + "tokens": ["secret-tokens-0", "secret-tokens-1"], + "labels": {"visible": "visible-label", "extra": "secret-label"}, + "credential": {"kind": "password", "password": "secret-credential"}, + "tree": {"label": "visible-root", "secret": "secret-root", "children": [{"label": "visible-child", "secret": "secret-child", "children": []}]}, + "whole": {"value": "secret-whole"}, + "unrelated": {"secret": "visible-unrelated"} +}` + +type redactCaptureClient struct { + status int + operation string +} + +func (c *redactCaptureClient) Do(req *http.Request) (*http.Response, error) { + c.operation = redact.Operation(req.Context()) + return &http.Response{ + StatusCode: c.status, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(sensitiveBodyPayload)), + Request: req, + }, nil +} + +func assertSensitiveBodyMasked(t *testing.T, output string) { + t.Helper() + for _, secret := range []string{"secret-password", "secret-token", "secret-nested", "secret-item", "secret-tokens-0", "secret-tokens-1", "secret-label", "secret-credential", "secret-root", "secret-child", "secret-whole"} { + assert.NotContains(t, output, secret) + } + for _, visible := range []string{"visible-name", "visible-nested", "visible-item", "visible-label", "visible-root", "visible-child", "visible-unrelated"} { + assert.Contains(t, output, visible) + } +} + +func TestRedactSensitiveBodies(t *testing.T) { + ctx := redact.WithOperation(context.Background(), "sensitiveBodyRedaction") + + assertSensitiveBodyMasked(t, string(redact.RequestBody(ctx, []byte(sensitiveBodyPayload)))) + assertSensitiveBodyMasked(t, string(redact.ResponseBody(ctx, []byte(sensitiveBodyPayload)))) + + plain := []byte(`{"name": "visible-name"}`) + assert.Equal(t, plain, redact.ResponseBody(ctx, plain), "bodies without sensitive values are returned unchanged") + assert.Equal(t, redact.Mask, string(redact.ResponseBody(ctx, []byte("secret-text"))), "unparseable bodies are masked as a whole") + assert.Equal(t, []byte(sensitiveBodyPayload), redact.ResponseBody(context.Background(), []byte(sensitiveBodyPayload)), "requests without an operation are not masked") + + errorBody := string(redact.ResponseBody(ctx, []byte(`{"message": "visible-message", "secret": "secret-error"}`))) + assert.Contains(t, errorBody, "visible-message") + assert.NotContains(t, errorBody, "secret-error", "error response schemas are masked") + + streamCtx := redact.WithOperation(context.Background(), "sensitiveStreamRedaction") + assert.Equal(t, redact.Mask, string(redact.ResponseBody(streamCtx, []byte(`{"label": "visible", "secret": "secret-stream"}`))), "streams holding sensitive values are masked as a whole") +} + +func TestRedactOperationOnRequestContext(t *testing.T) { + client := &redactCaptureClient{status: http.StatusOK} + s := sdk.New(sdk.WithServerURL("http://localhost"), sdk.WithClient(client)) + + res, err := s.SensitiveBodies.Redaction(context.Background(), shared.SensitiveBodyPayload{Name: "visible-name"}) + require.NoError(t, err) + assert.Equal(t, "sensitiveBodyRedaction", client.operation) + require.NotNil(t, res.SensitiveBodyPayload) + assert.Equal(t, "secret-password", *res.SensitiveBodyPayload.Password, "decoded values are not masked") + + client.status = http.StatusInternalServerError + _, err = s.SensitiveBodies.Redaction(context.Background(), shared.SensitiveBodyPayload{Name: "visible-name"}) + var apiErr *apierrors.APIError + require.True(t, errors.As(err, &apiErr)) + assert.Contains(t, apiErr.Body, "secret-password", "the error body is not masked") + assertSensitiveBodyMasked(t, apiErr.Error()) +} diff --git a/templates/templates/terraform/auxiliary/internal/provider/utils.go.stmpl b/templates/templates/terraform/auxiliary/internal/provider/utils.go.stmpl index 5de82f435..f9814597d 100644 --- a/templates/templates/terraform/auxiliary/internal/provider/utils.go.stmpl +++ b/templates/templates/terraform/auxiliary/internal/provider/utils.go.stmpl @@ -20,6 +20,7 @@ import ( "github.com/hashicorp/terraform-plugin-log/tflog" tfReflect "{{getRootPackage}}/internal/provider/reflect" + "{{getSDKPackage}}/redact" ) func debugResponse(response *http.Response) string { @@ -31,14 +32,30 @@ func debugResponse(response *http.Response) string { return err.Error() } } - dumpRes, err := httputil.DumpResponse(response, true) + // The body is read directly rather than from a full dump, which would + // re-encode a chunked body and so prevent it from being parsed. + dumpRes, err := httputil.DumpResponse(response, false) if err != nil { - dumpRes, err = httputil.DumpResponse(response, false) - if err != nil { - return err.Error() + return err.Error() + } + ctx := response.Request.Context() + if response.Body != nil { + if body, err := io.ReadAll(response.Body); err == nil { + response.Body = io.NopCloser(bytes.NewReader(body)) + dumpRes = append(dumpRes, redact.ResponseBody(ctx, body)...) } } - return redactSensitiveValues(response.Request.Context(), fmt.Sprintf("**Request**:\n%s\n**Response**:\n%s", string(dumpReq), string(dumpRes))) + dumpReq = redactDumpBody(dumpReq, func(body []byte) []byte { return redact.RequestBody(ctx, body) }) + return redactSensitiveValues(ctx, fmt.Sprintf("**Request**:\n%s\n**Response**:\n%s", string(dumpReq), string(dumpRes))) +} + +// redactDumpBody applies redactBody to the body of an HTTP message dump. +func redactDumpBody(dump []byte, redactBody func([]byte) []byte) []byte { + head, body, found := bytes.Cut(dump, []byte("\r\n\r\n")) + if !found { + return dump + } + return bytes.Join([][]byte{head, redactBody(body)}, []byte("\r\n\r\n")) } func merge(ctx context.Context, req resource.UpdateRequest, resp *resource.UpdateResponse, target interface{}) { @@ -207,7 +224,7 @@ func decomposeRequestForLogging(req *http.Request) (map[string]interface{}, erro } // Read the rest of the body content - fields[FieldHttpRequestBody] = bodyFromRestOfRequestReader(reqReader) + fields[FieldHttpRequestBody] = string(redact.RequestBody(req.Context(), []byte(bodyFromRestOfRequestReader(reqReader)))) redactSensitiveFields(req, fields) return fields, nil } @@ -282,6 +299,9 @@ func decomposeResponseForLogging(res *http.Response) (map[string]interface{}, er // http.Client res.Body = io.NopCloser(bytes.NewBuffer(resBody)) + if res.Request != nil { + resBody = redact.ResponseBody(res.Request.Context(), resBody) + } fields[FieldHttpResponseBody] = string(resBody) redactSensitiveFields(res.Request, fields) diff --git a/templates/templates/terraform/tests/review/internal/provider/sensitive_body_test.go.stmpl b/templates/templates/terraform/tests/review/internal/provider/sensitive_body_test.go.stmpl new file mode 100644 index 000000000..3c5ce1710 --- /dev/null +++ b/templates/templates/terraform/tests/review/internal/provider/sensitive_body_test.go.stmpl @@ -0,0 +1,71 @@ +package provider + +import ( + "bytes" + stderrors "errors" + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/hashicorp/terraform-plugin-log/tflogtest" + + "{{getSDKPackage}}" + "{{getSDKPackage}}/models/errors" + "{{getSDKPackage}}/models/operations" +) + +// A computed sensitive value is unknown before the response arrives, so only +// the response schema can identify it in the transport log and diagnostics. +func TestSensitiveResponseValuesRedactedFromLogsAndErrors(t *testing.T) { + status := http.StatusOK + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"id":"visible-id","string_request":"visible-request","string_response":"computed-secret","string_request_and_response":"both-secret"}`)) + })) + defer server.Close() + + var logs bytes.Buffer + ctx := tflogtest.RootLogger(context.Background(), &logs) + client := sdk.New( + sdk.WithServerURL(server.URL), + sdk.WithClient(&http.Client{Transport: NewProviderHTTPTransport(ProviderHTTPTransportOpts{Transport: http.DefaultTransport})}), + ) + + res, err := client.GetXParamSensitive(ctx, operations.GetXParamSensitiveRequest{ID: "visible-id"}) + if err != nil { + t.Fatal(err) + } + if got := res.XParamSensitiveResponse.GetStringResponse(); got == nil || *got != "computed-secret" { + t.Errorf("SDK decoded string_response as %v, want the original value", got) + } + + status = http.StatusBadRequest + _, err = client.GetXParamSensitive(ctx, operations.GetXParamSensitiveRequest{ID: "visible-id"}) + var apiErr *errors.SDKError + if !stderrors.As(err, &apiErr) { + t.Fatalf("expected an SDK API error, got %v", err) + } + if !strings.Contains(apiErr.Body, "computed-secret") { + t.Errorf("SDKError.Body should keep the original body: %s", apiErr.Body) + } + + for name, output := range map[string]string{ + "transport log": logs.String(), + "error message": err.Error(), + "debug response": debugResponse(apiErr.RawResponse), + } { + for _, secret := range []string{"computed-secret", "both-secret"} { + if strings.Contains(output, secret) { + t.Errorf("%s leaked %q: %s", name, secret, output) + } + } + for _, visible := range []string{"visible-id", "visible-request"} { + if !strings.Contains(output, visible) { + t.Errorf("%s dropped non-sensitive %q: %s", name, visible, output) + } + } + } +} diff --git a/tests/specs/fragments/primary/sensitive-body-redaction.yaml b/tests/specs/fragments/primary/sensitive-body-redaction.yaml new file mode 100644 index 000000000..646b2d1dd --- /dev/null +++ b/tests/specs/fragments/primary/sensitive-body-redaction.yaml @@ -0,0 +1,166 @@ +openapi: 3.1.0 +security: + - apiKeyAuth: [] + - oauth2: [] + - clientCredentials: [read, write] + - customSchemeAppId: [] + - basicHttp: [] + - accessToken: [] + - {} +paths: + /anything/sensitiveBodyRedaction: + post: + operationId: sensitiveBodyRedaction + x-speakeasy-group: sensitiveBodies + x-speakeasy-name-override: redaction + description: >- + Exercises locating values marked sensitive (x-speakeasy-param-sensitive + or format password) in request and response bodies by schema path, so + that they can be masked when bodies are logged. Sensitive values appear + at the top level, in nested objects, in array items, as array items, as + additionalProperties values beside a declared plain property, in union + variants, through a recursive schema, and as a whole object. An + unmarked property sharing a name with a sensitive one stays readable. + The error response schema also holds a sensitive value. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/SensitiveBodyPayload" + responses: + "200": + description: The payload, including newly returned sensitive values. + content: + application/json: + schema: + $ref: "#/components/schemas/SensitiveBodyPayload" + "400": + description: An error whose body holds a sensitive value. + content: + application/json: + schema: + $ref: "#/components/schemas/SensitiveBodyError" + /anything/sensitiveStreamRedaction: + get: + operationId: sensitiveStreamRedaction + x-speakeasy-group: sensitiveBodies + x-speakeasy-name-override: streamRedaction + description: >- + Exercises a streamed JSON Lines response whose items hold a value marked + sensitive. A stream body is not parsed item by item, so it is masked as + a whole when logged. + responses: + "200": + description: A stream of items holding sensitive values. + content: + application/jsonl: + schema: + $ref: "#/components/schemas/SensitiveBodyNested" +components: + schemas: + SensitiveBodyPayload: + type: object + required: + - name + properties: + name: + type: string + password: + type: string + format: password + token: + type: string + x-speakeasy-param-sensitive: true + nested: + $ref: "#/components/schemas/SensitiveBodyNested" + items: + type: array + items: + $ref: "#/components/schemas/SensitiveBodyNested" + tokens: + type: array + items: + type: string + x-speakeasy-param-sensitive: true + labels: + $ref: "#/components/schemas/SensitiveBodyLabels" + credential: + oneOf: + - $ref: "#/components/schemas/SensitiveBodyAPIKeyCredential" + - $ref: "#/components/schemas/SensitiveBodyPasswordCredential" + tree: + $ref: "#/components/schemas/SensitiveBodyTree" + whole: + type: object + x-speakeasy-param-sensitive: true + properties: + value: + type: string + unrelated: + $ref: "#/components/schemas/SensitiveBodyUnrelated" + SensitiveBodyError: + type: object + properties: + message: + type: string + secret: + type: string + x-speakeasy-param-sensitive: true + SensitiveBodyNested: + type: object + properties: + label: + type: string + secret: + type: string + x-speakeasy-param-sensitive: true + SensitiveBodyLabels: + type: object + properties: + visible: + type: string + additionalProperties: + type: string + x-speakeasy-param-sensitive: true + SensitiveBodyAPIKeyCredential: + type: object + required: + - kind + - key + properties: + kind: + type: string + const: api_key + key: + type: string + x-speakeasy-param-sensitive: true + SensitiveBodyPasswordCredential: + type: object + required: + - kind + - password + properties: + kind: + type: string + const: password + password: + type: string + format: password + SensitiveBodyTree: + type: object + properties: + label: + type: string + secret: + type: string + x-speakeasy-param-sensitive: true + children: + type: array + items: + $ref: "#/components/schemas/SensitiveBodyTree" + SensitiveBodyUnrelated: + type: object + properties: + secret: + type: string diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index ba46d959c..61f3de679 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -563,6 +563,8 @@ trackedFiles: last_write_checksum: sha1:b828c94c45116e632925027e4e2c211e971e04e2 internal/sdk/optionalnullable/optionalnullable_test.go: last_write_checksum: sha1:d3991667754ed67db803aeab4a6239f6f9a1be37 + internal/sdk/redact/redact.go: + last_write_checksum: sha1:8090b7e04d1365758dde8e2807b14ecb697b5257 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/sdk-cli/internal/sdk/redact/redact.go b/zSDKs/sdk-cli/internal/sdk/redact/redact.go new file mode 100644 index 000000000..98b5dfe1b --- /dev/null +++ b/zSDKs/sdk-cli/internal/sdk/redact/redact.go @@ -0,0 +1,179 @@ +// Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + +// Package redact masks values marked sensitive in the API description +// (x-speakeasy-param-sensitive or format: password) in HTTP bodies, so that +// HTTP clients and transports can log requests and responses safely. +// +// The SDK attaches the operation ID to the context of each request whose +// operation has sensitive request or response values. A transport reads it +// back from the request, or from a response's Request, to select the schema +// graph used for masking: +// +// body = redact.RequestBody(req.Context(), body) +// body = redact.ResponseBody(res.Request.Context(), body) +package redact + +import ( + "bytes" + "context" + "encoding/json" + "io" +) + +// Mask replaces each sensitive value. +const Mask = "(sensitive)" + +const maxDepth = 64 + +type operationKey struct{} + +// WithOperation returns ctx carrying operationID. +func WithOperation(ctx context.Context, operationID string) context.Context { + return context.WithValue(ctx, operationKey{}, operationID) +} + +// Operation returns the operation ID carried by ctx, or "". +func Operation(ctx context.Context) string { + if ctx == nil { + return "" + } + id, _ := ctx.Value(operationKey{}).(string) + return id +} + +// RequestBody masks the sensitive values of the request body of the +// operation carried by ctx. Bodies of other operations are returned as is. +func RequestBody(ctx context.Context, body []byte) []byte { + return operations[Operation(ctx)].request.redact(body) +} + +// ResponseBody masks the sensitive values of a response body of the +// operation carried by ctx, matching every response schema of the operation. +// Bodies of other operations are returned as is. +func ResponseBody(ctx context.Context, body []byte) []byte { + return operations[Operation(ctx)].response.redact(body) +} + +// A graph describes where sensitive values sit in a body. Node 0 is a +// sentinel for "nothing sensitive below here"; roots are the body schemas. +type graph struct { + nodes []node + roots []int +} + +type node struct { + sensitive bool + // fields maps JSON property names to their schema; a property that is not + // declared uses values (additional properties or map values). + fields map[string]int + item int + values int + variants []int +} + +// redact returns body unchanged when it has nothing to mask. A body that +// cannot be parsed as JSON is masked as a whole. +func (g *graph) redact(body []byte) []byte { + if g == nil || len(body) == 0 { + return body + } + for _, id := range g.expand(g.roots) { + if g.nodes[id].sensitive { + return []byte(Mask) + } + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var value any + if err := dec.Decode(&value); err != nil { + return []byte(Mask) + } + if _, err := dec.Token(); err != io.EOF { + return []byte(Mask) + } + masked, changed := g.mask(value, g.roots, 0) + if !changed { + return body + } + var out bytes.Buffer + enc := json.NewEncoder(&out) + enc.SetEscapeHTML(false) + if err := enc.Encode(masked); err != nil { + return []byte(Mask) + } + return bytes.TrimSuffix(out.Bytes(), []byte("\n")) +} + +func (g *graph) mask(value any, roots []int, depth int) (any, bool) { + roots = g.expand(roots) + if len(roots) == 0 { + return value, false + } + for _, id := range roots { + if g.nodes[id].sensitive { + return Mask, true + } + } + if depth >= maxDepth { + return Mask, true + } + changed := false + switch v := value.(type) { + case map[string]any: + for key, child := range v { + var childRoots []int + for _, id := range roots { + n := g.nodes[id] + childID, declared := n.fields[key] + if !declared { + childID = n.values + } + if childID != 0 { + childRoots = append(childRoots, childID) + } + } + if masked, ok := g.mask(child, childRoots, depth+1); ok { + v[key] = masked + changed = true + } + } + case []any: + var items []int + for _, id := range roots { + if item := g.nodes[id].item; item != 0 { + items = append(items, item) + } + } + for i, child := range v { + if masked, ok := g.mask(child, items, depth+1); ok { + v[i] = masked + changed = true + } + } + } + return value, changed +} + +// expand adds the union variants of roots, since the variant a value matches +// is not known. +func (g *graph) expand(roots []int) []int { + var expanded []int + seen := make(map[int]bool) + var visit func(int) + visit = func(id int) { + if id <= 0 || id >= len(g.nodes) || seen[id] { + return + } + seen[id] = true + expanded = append(expanded, id) + for _, variant := range g.nodes[id].variants { + visit(variant) + } + } + for _, id := range roots { + visit(id) + } + return expanded +} + +var operations = map[string]struct{ request, response *graph }{} diff --git a/zSDKs/sdk-go/.speakeasy/gen.lock b/zSDKs/sdk-go/.speakeasy/gen.lock index ad322fc15..b6a56f5e2 100644 --- a/zSDKs/sdk-go/.speakeasy/gen.lock +++ b/zSDKs/sdk-go/.speakeasy/gen.lock @@ -782,6 +782,8 @@ trackedFiles: last_write_checksum: sha1:f33d36ba2e0ca6811572cd08ce53fe5f3c5664aa recursiveformfield.go: last_write_checksum: sha1:2e06532b8c5445500bd2ae8ae8586d24e911a3e2 + redact/redact.go: + last_write_checksum: sha1:8090b7e04d1365758dde8e2807b14ecb697b5257 renderassetop.go: last_write_checksum: sha1:a788e389967e88581337759b330af30c4e562bf8 requesttimeouterror.go: diff --git a/zSDKs/sdk-go/redact/redact.go b/zSDKs/sdk-go/redact/redact.go new file mode 100644 index 000000000..98b5dfe1b --- /dev/null +++ b/zSDKs/sdk-go/redact/redact.go @@ -0,0 +1,179 @@ +// Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + +// Package redact masks values marked sensitive in the API description +// (x-speakeasy-param-sensitive or format: password) in HTTP bodies, so that +// HTTP clients and transports can log requests and responses safely. +// +// The SDK attaches the operation ID to the context of each request whose +// operation has sensitive request or response values. A transport reads it +// back from the request, or from a response's Request, to select the schema +// graph used for masking: +// +// body = redact.RequestBody(req.Context(), body) +// body = redact.ResponseBody(res.Request.Context(), body) +package redact + +import ( + "bytes" + "context" + "encoding/json" + "io" +) + +// Mask replaces each sensitive value. +const Mask = "(sensitive)" + +const maxDepth = 64 + +type operationKey struct{} + +// WithOperation returns ctx carrying operationID. +func WithOperation(ctx context.Context, operationID string) context.Context { + return context.WithValue(ctx, operationKey{}, operationID) +} + +// Operation returns the operation ID carried by ctx, or "". +func Operation(ctx context.Context) string { + if ctx == nil { + return "" + } + id, _ := ctx.Value(operationKey{}).(string) + return id +} + +// RequestBody masks the sensitive values of the request body of the +// operation carried by ctx. Bodies of other operations are returned as is. +func RequestBody(ctx context.Context, body []byte) []byte { + return operations[Operation(ctx)].request.redact(body) +} + +// ResponseBody masks the sensitive values of a response body of the +// operation carried by ctx, matching every response schema of the operation. +// Bodies of other operations are returned as is. +func ResponseBody(ctx context.Context, body []byte) []byte { + return operations[Operation(ctx)].response.redact(body) +} + +// A graph describes where sensitive values sit in a body. Node 0 is a +// sentinel for "nothing sensitive below here"; roots are the body schemas. +type graph struct { + nodes []node + roots []int +} + +type node struct { + sensitive bool + // fields maps JSON property names to their schema; a property that is not + // declared uses values (additional properties or map values). + fields map[string]int + item int + values int + variants []int +} + +// redact returns body unchanged when it has nothing to mask. A body that +// cannot be parsed as JSON is masked as a whole. +func (g *graph) redact(body []byte) []byte { + if g == nil || len(body) == 0 { + return body + } + for _, id := range g.expand(g.roots) { + if g.nodes[id].sensitive { + return []byte(Mask) + } + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var value any + if err := dec.Decode(&value); err != nil { + return []byte(Mask) + } + if _, err := dec.Token(); err != io.EOF { + return []byte(Mask) + } + masked, changed := g.mask(value, g.roots, 0) + if !changed { + return body + } + var out bytes.Buffer + enc := json.NewEncoder(&out) + enc.SetEscapeHTML(false) + if err := enc.Encode(masked); err != nil { + return []byte(Mask) + } + return bytes.TrimSuffix(out.Bytes(), []byte("\n")) +} + +func (g *graph) mask(value any, roots []int, depth int) (any, bool) { + roots = g.expand(roots) + if len(roots) == 0 { + return value, false + } + for _, id := range roots { + if g.nodes[id].sensitive { + return Mask, true + } + } + if depth >= maxDepth { + return Mask, true + } + changed := false + switch v := value.(type) { + case map[string]any: + for key, child := range v { + var childRoots []int + for _, id := range roots { + n := g.nodes[id] + childID, declared := n.fields[key] + if !declared { + childID = n.values + } + if childID != 0 { + childRoots = append(childRoots, childID) + } + } + if masked, ok := g.mask(child, childRoots, depth+1); ok { + v[key] = masked + changed = true + } + } + case []any: + var items []int + for _, id := range roots { + if item := g.nodes[id].item; item != 0 { + items = append(items, item) + } + } + for i, child := range v { + if masked, ok := g.mask(child, items, depth+1); ok { + v[i] = masked + changed = true + } + } + } + return value, changed +} + +// expand adds the union variants of roots, since the variant a value matches +// is not known. +func (g *graph) expand(roots []int) []int { + var expanded []int + seen := make(map[int]bool) + var visit func(int) + visit = func(id int) { + if id <= 0 || id >= len(g.nodes) || seen[id] { + return + } + seen[id] = true + expanded = append(expanded, id) + for _, variant := range g.nodes[id].variants { + visit(variant) + } + } + for _, id := range roots { + visit(id) + } + return expanded +} + +var operations = map[string]struct{ request, response *graph }{} diff --git a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock index 618b8b520..e96f7b249 100644 --- a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock +++ b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock @@ -1183,6 +1183,8 @@ trackedFiles: last_write_checksum: sha1:90486b28f731a812a5a29ea27474c3d1c250b05c internal/provider/rootunionwriteonly_resource_sdk.go: last_write_checksum: sha1:3877a6339d3c954216c275907f26f5f8f606a7ac + internal/provider/sensitive_body_test.go: + last_write_checksum: sha1:77d5081775c1662a561e65eae2cf506d23a67597 internal/provider/sensitive_values.go: last_write_checksum: sha1:81d8e01d37718f9af451660d4ebc5f6126569c60 internal/provider/sensitive_values_test.go: @@ -2076,7 +2078,7 @@ trackedFiles: internal/provider/unsoundreadonlysingleop_resource_sdk.go: last_write_checksum: sha1:9ebe7ce0a238c2f58ec4fab7e587fc8d91f1b5bd internal/provider/utils.go: - last_write_checksum: sha1:b7eb39b516cfeced2367ff75019fca2b7f91d6c3 + last_write_checksum: sha1:e21456c199e9497aeb41d297f4cae588185a5bc8 internal/provider/utils_test.go: last_write_checksum: sha1:40da24e32eac29a8aac14e772e5fce44044f2ae0 internal/provider/xadditionalpropertiesname_data_source.go: @@ -2422,11 +2424,11 @@ trackedFiles: internal/sdk/internal/utils/union_test.go: last_write_checksum: sha1:c55e5e44c26dfae67b83359851996122a3dfc9cf internal/sdk/internal/utils/utils.go: - last_write_checksum: sha1:2b7f8f2bd131693f165c80adb2e47f404819d65d + last_write_checksum: sha1:76e9fbc958c040371954894ddb2d99a63f0331d1 internal/sdk/models/errors/responsevalidationerror.go: last_write_checksum: sha1:aa7f69462e3a38d05371225adeb8c87bb433e6c4 internal/sdk/models/errors/sdkerror.go: - last_write_checksum: sha1:5d65fce5049df7c5113e11c9ccb1e4c46b91901c + last_write_checksum: sha1:08765f9bb829f73021d99c4951c5d340cf1a7b07 internal/sdk/models/operations/closexentityoperationclosemultipleop1.go: last_write_checksum: sha1:7664dbc7a028e9f840a833fd5a24ade28c6bded3 internal/sdk/models/operations/closexentityoperationclosemultipleop2.go: @@ -3571,10 +3573,12 @@ trackedFiles: last_write_checksum: sha1:9695f32fc3b427c945b3aa34c3ada4dab0ba9ac1 internal/sdk/polling/option.go: last_write_checksum: sha1:69d4557bf43a1a97b2d3735c442d2d25be53e56f + internal/sdk/redact/redact.go: + last_write_checksum: sha1:73715057b336608296e6ea2900255ae2f5c2a42a internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: - last_write_checksum: sha1:224db2187527a941b29d524d57531f33cfc70f48 + last_write_checksum: sha1:a2997f8af41cead3493768284c9cc8c950f8f57c internal/sdk/types/bigint.go: last_write_checksum: sha1:49b004005d0461fb04b846eca062b070b0360b31 internal/sdk/types/date.go: diff --git a/zSDKs/terraform-provider-testing/internal/provider/sensitive_body_test.go b/zSDKs/terraform-provider-testing/internal/provider/sensitive_body_test.go new file mode 100644 index 000000000..1c9a49722 --- /dev/null +++ b/zSDKs/terraform-provider-testing/internal/provider/sensitive_body_test.go @@ -0,0 +1,73 @@ +// Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + +package provider + +import ( + "bytes" + "context" + stderrors "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/hashicorp/terraform-plugin-log/tflogtest" + + "github.com/hashicorp/terraform-provider-testing/internal/sdk" + "github.com/hashicorp/terraform-provider-testing/internal/sdk/models/errors" + "github.com/hashicorp/terraform-provider-testing/internal/sdk/models/operations" +) + +// A computed sensitive value is unknown before the response arrives, so only +// the response schema can identify it in the transport log and diagnostics. +func TestSensitiveResponseValuesRedactedFromLogsAndErrors(t *testing.T) { + status := http.StatusOK + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"id":"visible-id","string_request":"visible-request","string_response":"computed-secret","string_request_and_response":"both-secret"}`)) + })) + defer server.Close() + + var logs bytes.Buffer + ctx := tflogtest.RootLogger(context.Background(), &logs) + client := sdk.New( + sdk.WithServerURL(server.URL), + sdk.WithClient(&http.Client{Transport: NewProviderHTTPTransport(ProviderHTTPTransportOpts{Transport: http.DefaultTransport})}), + ) + + res, err := client.GetXParamSensitive(ctx, operations.GetXParamSensitiveRequest{ID: "visible-id"}) + if err != nil { + t.Fatal(err) + } + if got := res.XParamSensitiveResponse.GetStringResponse(); got == nil || *got != "computed-secret" { + t.Errorf("SDK decoded string_response as %v, want the original value", got) + } + + status = http.StatusBadRequest + _, err = client.GetXParamSensitive(ctx, operations.GetXParamSensitiveRequest{ID: "visible-id"}) + var apiErr *errors.SDKError + if !stderrors.As(err, &apiErr) { + t.Fatalf("expected an SDK API error, got %v", err) + } + if !strings.Contains(apiErr.Body, "computed-secret") { + t.Errorf("SDKError.Body should keep the original body: %s", apiErr.Body) + } + + for name, output := range map[string]string{ + "transport log": logs.String(), + "error message": err.Error(), + "debug response": debugResponse(apiErr.RawResponse), + } { + for _, secret := range []string{"computed-secret", "both-secret"} { + if strings.Contains(output, secret) { + t.Errorf("%s leaked %q: %s", name, secret, output) + } + } + for _, visible := range []string{"visible-id", "visible-request"} { + if !strings.Contains(output, visible) { + t.Errorf("%s dropped non-sensitive %q: %s", name, visible, output) + } + } + } +} diff --git a/zSDKs/terraform-provider-testing/internal/provider/utils.go b/zSDKs/terraform-provider-testing/internal/provider/utils.go index 9defadc19..a31cf523e 100644 --- a/zSDKs/terraform-provider-testing/internal/provider/utils.go +++ b/zSDKs/terraform-provider-testing/internal/provider/utils.go @@ -22,6 +22,7 @@ import ( "github.com/hashicorp/terraform-plugin-log/tflog" tfReflect "github.com/hashicorp/terraform-provider-testing/internal/provider/reflect" + "github.com/hashicorp/terraform-provider-testing/internal/sdk/redact" ) func debugResponse(response *http.Response) string { @@ -44,14 +45,30 @@ func debugResponse(response *http.Response) string { return err.Error() } } - dumpRes, err := httputil.DumpResponse(response, true) + // The body is read directly rather than from a full dump, which would + // re-encode a chunked body and so prevent it from being parsed. + dumpRes, err := httputil.DumpResponse(response, false) if err != nil { - dumpRes, err = httputil.DumpResponse(response, false) - if err != nil { - return err.Error() + return err.Error() + } + ctx := response.Request.Context() + if response.Body != nil { + if body, err := io.ReadAll(response.Body); err == nil { + response.Body = io.NopCloser(bytes.NewReader(body)) + dumpRes = append(dumpRes, redact.ResponseBody(ctx, body)...) } } - return redactSensitiveValues(response.Request.Context(), fmt.Sprintf("**Request**:\n%s\n**Response**:\n%s", string(dumpReq), string(dumpRes))) + dumpReq = redactDumpBody(dumpReq, func(body []byte) []byte { return redact.RequestBody(ctx, body) }) + return redactSensitiveValues(ctx, fmt.Sprintf("**Request**:\n%s\n**Response**:\n%s", string(dumpReq), string(dumpRes))) +} + +// redactDumpBody applies redactBody to the body of an HTTP message dump. +func redactDumpBody(dump []byte, redactBody func([]byte) []byte) []byte { + head, body, found := bytes.Cut(dump, []byte("\r\n\r\n")) + if !found { + return dump + } + return bytes.Join([][]byte{head, redactBody(body)}, []byte("\r\n\r\n")) } func merge(ctx context.Context, req resource.UpdateRequest, resp *resource.UpdateResponse, target interface{}) { @@ -220,7 +237,7 @@ func decomposeRequestForLogging(req *http.Request) (map[string]interface{}, erro } // Read the rest of the body content - fields[FieldHttpRequestBody] = bodyFromRestOfRequestReader(reqReader) + fields[FieldHttpRequestBody] = string(redact.RequestBody(req.Context(), []byte(bodyFromRestOfRequestReader(reqReader)))) redactSensitiveFields(req, fields) return fields, nil } @@ -306,6 +323,9 @@ func decomposeResponseForLogging(res *http.Response) (map[string]interface{}, er // http.Client res.Body = io.NopCloser(bytes.NewBuffer(resBody)) + if res.Request != nil { + resBody = redact.ResponseBody(res.Request.Context(), resBody) + } fields[FieldHttpResponseBody] = string(resBody) redactSensitiveFields(res.Request, fields) diff --git a/zSDKs/terraform-provider-testing/internal/sdk/internal/utils/utils.go b/zSDKs/terraform-provider-testing/internal/sdk/internal/utils/utils.go index f239efa3c..0ffc2fe6d 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/internal/utils/utils.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/internal/utils/utils.go @@ -17,6 +17,7 @@ import ( "time" "github.com/hashicorp/terraform-provider-testing/internal/sdk/optionalnullable" + "github.com/hashicorp/terraform-provider-testing/internal/sdk/redact" ) const ( @@ -25,6 +26,12 @@ const ( pathParamTagKey = "pathParam" ) +// WithOperation marks requests built from ctx as belonging to operationID, so +// that transports can mask the operation's sensitive values when logging. +func WithOperation(ctx context.Context, operationID string) context.Context { + return redact.WithOperation(ctx, operationID) +} + var ( paramRegex = regexp.MustCompile(`({.*?})`) SerializationMethodToContentType = map[string]string{ diff --git a/zSDKs/terraform-provider-testing/internal/sdk/models/errors/sdkerror.go b/zSDKs/terraform-provider-testing/internal/sdk/models/errors/sdkerror.go index c633d568d..6ae2da4cb 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/models/errors/sdkerror.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/models/errors/sdkerror.go @@ -5,6 +5,8 @@ package errors import ( "fmt" "net/http" + + "github.com/hashicorp/terraform-provider-testing/internal/sdk/redact" ) type SDKError struct { @@ -28,7 +30,11 @@ func NewSDKError(message string, statusCode int, body string, httpRes *http.Resp func (e *SDKError) Error() string { body := "" if len(e.Body) > 0 { - body = fmt.Sprintf("\n%s", e.Body) + redacted := e.Body + if e.RawResponse != nil && e.RawResponse.Request != nil { + redacted = string(redact.ResponseBody(e.RawResponse.Request.Context(), []byte(e.Body))) + } + body = fmt.Sprintf("\n%s", redacted) } return fmt.Sprintf("%s: Status %d%s", e.Message, e.StatusCode, body) diff --git a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go new file mode 100644 index 000000000..0422e4e60 --- /dev/null +++ b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go @@ -0,0 +1,183 @@ +// Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + +// Package redact masks values marked sensitive in the API description +// (x-speakeasy-param-sensitive or format: password) in HTTP bodies, so that +// HTTP clients and transports can log requests and responses safely. +// +// The SDK attaches the operation ID to the context of each request whose +// operation has sensitive request or response values. A transport reads it +// back from the request, or from a response's Request, to select the schema +// graph used for masking: +// +// body = redact.RequestBody(req.Context(), body) +// body = redact.ResponseBody(res.Request.Context(), body) +package redact + +import ( + "bytes" + "context" + "encoding/json" + "io" +) + +// Mask replaces each sensitive value. +const Mask = "(sensitive)" + +const maxDepth = 64 + +type operationKey struct{} + +// WithOperation returns ctx carrying operationID. +func WithOperation(ctx context.Context, operationID string) context.Context { + return context.WithValue(ctx, operationKey{}, operationID) +} + +// Operation returns the operation ID carried by ctx, or "". +func Operation(ctx context.Context) string { + if ctx == nil { + return "" + } + id, _ := ctx.Value(operationKey{}).(string) + return id +} + +// RequestBody masks the sensitive values of the request body of the +// operation carried by ctx. Bodies of other operations are returned as is. +func RequestBody(ctx context.Context, body []byte) []byte { + return operations[Operation(ctx)].request.redact(body) +} + +// ResponseBody masks the sensitive values of a response body of the +// operation carried by ctx, matching every response schema of the operation. +// Bodies of other operations are returned as is. +func ResponseBody(ctx context.Context, body []byte) []byte { + return operations[Operation(ctx)].response.redact(body) +} + +// A graph describes where sensitive values sit in a body. Node 0 is a +// sentinel for "nothing sensitive below here"; roots are the body schemas. +type graph struct { + nodes []node + roots []int +} + +type node struct { + sensitive bool + // fields maps JSON property names to their schema; a property that is not + // declared uses values (additional properties or map values). + fields map[string]int + item int + values int + variants []int +} + +// redact returns body unchanged when it has nothing to mask. A body that +// cannot be parsed as JSON is masked as a whole. +func (g *graph) redact(body []byte) []byte { + if g == nil || len(body) == 0 { + return body + } + for _, id := range g.expand(g.roots) { + if g.nodes[id].sensitive { + return []byte(Mask) + } + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var value any + if err := dec.Decode(&value); err != nil { + return []byte(Mask) + } + if _, err := dec.Token(); err != io.EOF { + return []byte(Mask) + } + masked, changed := g.mask(value, g.roots, 0) + if !changed { + return body + } + var out bytes.Buffer + enc := json.NewEncoder(&out) + enc.SetEscapeHTML(false) + if err := enc.Encode(masked); err != nil { + return []byte(Mask) + } + return bytes.TrimSuffix(out.Bytes(), []byte("\n")) +} + +func (g *graph) mask(value any, roots []int, depth int) (any, bool) { + roots = g.expand(roots) + if len(roots) == 0 { + return value, false + } + for _, id := range roots { + if g.nodes[id].sensitive { + return Mask, true + } + } + if depth >= maxDepth { + return Mask, true + } + changed := false + switch v := value.(type) { + case map[string]any: + for key, child := range v { + var childRoots []int + for _, id := range roots { + n := g.nodes[id] + childID, declared := n.fields[key] + if !declared { + childID = n.values + } + if childID != 0 { + childRoots = append(childRoots, childID) + } + } + if masked, ok := g.mask(child, childRoots, depth+1); ok { + v[key] = masked + changed = true + } + } + case []any: + var items []int + for _, id := range roots { + if item := g.nodes[id].item; item != 0 { + items = append(items, item) + } + } + for i, child := range v { + if masked, ok := g.mask(child, items, depth+1); ok { + v[i] = masked + changed = true + } + } + } + return value, changed +} + +// expand adds the union variants of roots, since the variant a value matches +// is not known. +func (g *graph) expand(roots []int) []int { + var expanded []int + seen := make(map[int]bool) + var visit func(int) + visit = func(id int) { + if id <= 0 || id >= len(g.nodes) || seen[id] { + return + } + seen[id] = true + expanded = append(expanded, id) + for _, variant := range g.nodes[id].variants { + visit(variant) + } + } + for _, id := range roots { + visit(id) + } + return expanded +} + +var operations = map[string]struct{ request, response *graph }{ + "create-x-param-sensitive": {request: &graph{nodes: []node{{}, {fields: map[string]int{"string_request_and_response": 2, "string_request": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}, response: &graph{nodes: []node{{}, {fields: map[string]int{"string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, + "get-x-param-sensitive": {response: &graph{nodes: []node{{}, {fields: map[string]int{"string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, + "update-x-param-sensitive": {request: &graph{nodes: []node{{}, {fields: map[string]int{"string_request_and_response": 2, "string_request": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}, response: &graph{nodes: []node{{}, {fields: map[string]int{"string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, +} diff --git a/zSDKs/terraform-provider-testing/internal/sdk/sdk.go b/zSDKs/terraform-provider-testing/internal/sdk/sdk.go index 4dc8e6c7f..461079173 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/sdk.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/sdk.go @@ -32302,6 +32302,7 @@ func (s *SDK) CreateXParamSensitive(ctx context.Context, request shared.XParamSe defer cancel() } + ctx = utils.WithOperation(ctx, "create-x-param-sensitive") req, err := http.NewRequestWithContext(ctx, "POST", opURL, bodyReader) if err != nil { return nil, fmt.Errorf("error creating request: %w", err) @@ -32546,6 +32547,7 @@ func (s *SDK) GetXParamSensitive(ctx context.Context, request operations.GetXPar defer cancel() } + ctx = utils.WithOperation(ctx, "get-x-param-sensitive") req, err := http.NewRequestWithContext(ctx, "GET", opURL, nil) if err != nil { return nil, fmt.Errorf("error creating request: %w", err) @@ -32681,6 +32683,7 @@ func (s *SDK) UpdateXParamSensitive(ctx context.Context, request operations.Upda defer cancel() } + ctx = utils.WithOperation(ctx, "update-x-param-sensitive") req, err := http.NewRequestWithContext(ctx, "PATCH", opURL, bodyReader) if err != nil { return nil, fmt.Errorf("error creating request: %w", err) From 0938bf518f6ba3f2d91c870087e2b1e5f64c5aab Mon Sep 17 00:00:00 2001 From: Thomas Rooney Date: Thu, 8 Oct 2026 11:19:33 +0100 Subject: [PATCH 02/23] refactor(go): compute the sensitive body graph in the generator AST Move the schema walk from the Go templates into Operation methods (SensitiveRequestBodyGraph, SensitiveResponseBodyGraph, HasSensitiveBodies) so that any target can render the same language-agnostic graph. The Go templates now only render it. --- internal/ast/sensitive_body.go | 254 ++++++++++++++++++ internal/ast/sensitive_body_test.go | 190 +++++++++++++ .../templates/common/common/typeDefs.d.ts | 37 +++ templates/templates/go/includes/redact.ts | 196 ++------------ .../.speakeasy/gen.lock | 2 +- .../internal/sdk/redact/redact.go | 4 +- 6 files changed, 513 insertions(+), 170 deletions(-) create mode 100644 internal/ast/sensitive_body.go create mode 100644 internal/ast/sensitive_body_test.go diff --git a/internal/ast/sensitive_body.go b/internal/ast/sensitive_body.go new file mode 100644 index 000000000..1ef773120 --- /dev/null +++ b/internal/ast/sensitive_body.go @@ -0,0 +1,254 @@ +package ast + +import "sort" + +// SensitiveBodyGraph locates the values marked sensitive (x-speakeasy-param-sensitive +// or format: password) in the bodies of an operation, by their JSON location, +// so that generated code can mask them when bodies are logged. +// +// Nodes[0] is a sentinel meaning "nothing sensitive below here"; a node +// reference of 0 can be skipped. Roots are the body schemas: one per request +// content type, or one per response status code and content type. Only nodes +// that can reach a sensitive value are kept. +type SensitiveBodyGraph struct { + Nodes []*SensitiveBodyNode + Roots []int +} + +// SensitiveBodyNode is a schema in a SensitiveBodyGraph. +type SensitiveBodyNode struct { + // Sensitive masks the whole value. Streams are masked as a whole when any + // of their items holds a sensitive value. + Sensitive bool + // Fields are object properties by JSON name, sorted by name. A property + // that is not listed uses Values. When Values is set, declared properties + // are listed even when they hold nothing sensitive (with Node 0), so that + // they never fall back to Values. + Fields []SensitiveBodyField + // Item is the schema of array items. + Item int + // Values is the schema of map values and additional properties. + Values int + // Variants are union members; the member a value matches is not known. + Variants []int +} + +// SensitiveBodyField is an object property of a SensitiveBodyNode. +type SensitiveBodyField struct { + Name string + Node int +} + +// SensitiveRequestBodyGraph returns the sensitive value locations of the +// operation's request body, or nil when it holds none. +func (o *Operation) SensitiveRequestBodyGraph() *SensitiveBodyGraph { + b := newSensitiveBodyBuilder() + b.addRoot(o.requestBodyType()) + return b.build() +} + +// SensitiveResponseBodyGraph returns the sensitive value locations of all the +// operation's response bodies, including errors, or nil when they hold none. +func (o *Operation) SensitiveResponseBodyGraph() *SensitiveBodyGraph { + b := newSensitiveBodyBuilder() + if o.Response != nil { + for _, response := range o.Response.Responses { + for _, content := range response.Content { + if content != nil && content.Content != nil { + b.addRoot(content.Content.Type) + } + } + } + } + return b.build() +} + +// HasSensitiveBodies reports whether the operation's request or response +// bodies hold values marked sensitive. +func (o *Operation) HasSensitiveBodies() bool { + return o.SensitiveRequestBodyGraph() != nil || o.SensitiveResponseBodyGraph() != nil +} + +func (o *Operation) requestBodyType() *TypeDef { + if o.Request == nil { + return nil + } + if o.Request.Field != nil && o.Request.Field.Type != nil { + for _, field := range o.Request.Field.Type.Fields { + if field.Annotations.Has(AnnotationTypeRequest) { + return field.Type + } + } + } + if o.Request.IsRequestBody && o.Request.RequestBody != nil { + return o.Request.RequestBody.Type + } + return nil +} + +func isSensitiveBodyType(t *TypeDef) bool { + if t.Format == "password" { + return true + } + if t.Extensions == nil { + return false + } + marker := t.Extensions.All["x-speakeasy-param-sensitive"] + return marker == true || marker == "true" +} + +type sensitiveBodyBuilder struct { + nodes []*sensitiveBodyBuildNode + roots []int +} + +type sensitiveBodyBuildNode struct { + sensitive bool + stream bool + fields []SensitiveBodyField + item int + values int + variants []int +} + +func newSensitiveBodyBuilder() *sensitiveBodyBuilder { + return &sensitiveBodyBuilder{nodes: []*sensitiveBodyBuildNode{{}}} +} + +// addRoot walks a body schema. Custom types are walked once per root, by +// identity and by registration ID (which also matches the truncated copies +// that break recursive references), so that recursive schemas terminate. +func (b *sensitiveBodyBuilder) addRoot(t *TypeDef) { + walkedTypes := map[*TypeDef]int{} + walkedIDs := map[string]int{} + var walk func(t *TypeDef) int + walk = func(t *TypeDef) int { + if t == nil { + return 0 + } + if id, ok := walkedTypes[t]; ok { + return id + } + typeID := "" + if t.IsCustomType() && t.ContextStack != nil { + typeID = t.GetRegistrationID() + " name:" + t.Name + if id, ok := walkedIDs[typeID]; ok { + return id + } + } + id := len(b.nodes) + node := &sensitiveBodyBuildNode{} + b.nodes = append(b.nodes, node) + walkedTypes[t] = id + if typeID != "" { + walkedIDs[typeID] = id + } + if isSensitiveBodyType(t) { + node.sensitive = true + return id + } + switch t.Type { + case DataTypeClass, DataTypeError: + for _, field := range t.Fields { + if field.IsAdditionalProperties { + if field.Type != nil { + node.values = walk(field.Type.ItemType) + } + continue + } + name := field.OriginalName + if name == "" { + name = field.Name + } + child := 0 + if field.Const == nil { + child = walk(field.Type) + } + node.fields = append(node.fields, SensitiveBodyField{Name: name, Node: child}) + } + case DataTypeArray, DataTypeSet: + node.item = walk(t.ItemType) + case DataTypeMap: + node.values = walk(t.ItemType) + case DataTypeEventStream, DataTypeJsonL: + node.stream = true + node.item = walk(t.ItemType) + case DataTypeUnion: + for _, variant := range t.AssociatedTypes { + node.variants = append(node.variants, walk(variant)) + } + } + return id + } + b.roots = append(b.roots, walk(t)) +} + +// build prunes the nodes that cannot reach a sensitive value and renumbers +// the rest. +func (b *sensitiveBodyBuilder) build() *SensitiveBodyGraph { + reachable := make([]bool, len(b.nodes)) + for i, node := range b.nodes { + reachable[i] = node.sensitive + } + for changed := true; changed; { + changed = false + for i, node := range b.nodes { + if reachable[i] { + continue + } + children := append([]int{node.item, node.values}, node.variants...) + for _, field := range node.fields { + children = append(children, field.Node) + } + for _, child := range children { + if child != 0 && reachable[child] { + reachable[i] = true + changed = true + break + } + } + } + } + + remap := make([]int, len(b.nodes)) + graph := &SensitiveBodyGraph{Nodes: []*SensitiveBodyNode{{}}} + for i := 1; i < len(b.nodes); i++ { + if reachable[i] { + remap[i] = len(graph.Nodes) + graph.Nodes = append(graph.Nodes, &SensitiveBodyNode{}) + } + } + for _, root := range b.roots { + if root != 0 && reachable[root] { + graph.Roots = append(graph.Roots, remap[root]) + } + } + if len(graph.Roots) == 0 { + return nil + } + + for i, node := range b.nodes { + if i == 0 || !reachable[i] { + continue + } + out := graph.Nodes[remap[i]] + if node.sensitive || node.stream { + out.Sensitive = true + continue + } + out.Values = remap[node.values] + for _, field := range node.fields { + if out.Values != 0 || remap[field.Node] != 0 { + out.Fields = append(out.Fields, SensitiveBodyField{Name: field.Name, Node: remap[field.Node]}) + } + } + sort.SliceStable(out.Fields, func(a, b int) bool { return out.Fields[a].Name < out.Fields[b].Name }) + out.Item = remap[node.item] + for _, variant := range node.variants { + if remap[variant] != 0 { + out.Variants = append(out.Variants, remap[variant]) + } + } + } + return graph +} diff --git a/internal/ast/sensitive_body_test.go b/internal/ast/sensitive_body_test.go new file mode 100644 index 000000000..24ca9258c --- /dev/null +++ b/internal/ast/sensitive_body_test.go @@ -0,0 +1,190 @@ +package ast + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func sensitiveString() *TypeDef { + return &TypeDef{Type: DataTypeString, Extensions: &TypeDefExtensions{All: map[string]any{"x-speakeasy-param-sensitive": true}}} +} + +func plainString() *TypeDef { + return &TypeDef{Type: DataTypeString} +} + +func class(name string, fields ...*FieldDef) *TypeDef { + return &TypeDef{Type: DataTypeClass, Name: name, Fields: fields} +} + +func field(name string, typ *TypeDef) *FieldDef { + return &FieldDef{Name: name, OriginalName: name, Type: typ} +} + +func responseOperation(types ...*TypeDef) *Operation { + response := &SubResponse{Code: []string{"200"}} + for _, typ := range types { + response.Content = append(response.Content, &ResponseBodyContent{Content: &FieldDef{Type: typ}}) + } + return &Operation{BaseOperation: BaseOperation{Response: &Response{Responses: SubResponses{response}}}} +} + +// sensitiveChild follows a JSON property from the nodes, as a masker would. +func sensitiveChild(g *SensitiveBodyGraph, nodes []int, name string) []int { + var children []int + for _, id := range sensitiveVariants(g, nodes) { + node := g.Nodes[id] + child := node.Values + for _, f := range node.Fields { + if f.Name == name { + child = f.Node + } + } + if child != 0 { + children = append(children, child) + } + } + return children +} + +func sensitiveItems(g *SensitiveBodyGraph, nodes []int) []int { + var items []int + for _, id := range sensitiveVariants(g, nodes) { + if item := g.Nodes[id].Item; item != 0 { + items = append(items, item) + } + } + return items +} + +func sensitiveVariants(g *SensitiveBodyGraph, nodes []int) []int { + var out []int + seen := map[int]bool{} + var visit func(int) + visit = func(id int) { + if id == 0 || seen[id] { + return + } + seen[id] = true + out = append(out, id) + for _, v := range g.Nodes[id].Variants { + visit(v) + } + } + for _, id := range nodes { + visit(id) + } + return out +} + +func isSensitiveAt(g *SensitiveBodyGraph, nodes []int) bool { + for _, id := range sensitiveVariants(g, nodes) { + if g.Nodes[id].Sensitive { + return true + } + } + return false +} + +func TestSensitiveResponseBodyGraph(t *testing.T) { + t.Parallel() + + tree := class("Tree", field("label", plainString()), field("secret", sensitiveString())) + tree.Fields = append(tree.Fields, field("children", &TypeDef{Type: DataTypeArray, ItemType: tree})) + labels := class("Labels", field("visible", plainString()), &FieldDef{Name: "AdditionalProperties", IsAdditionalProperties: true, Type: &TypeDef{Type: DataTypeMap, ItemType: sensitiveString()}}) + password := &TypeDef{Type: DataTypeString, Format: "password"} + payload := class("Payload", + field("name", plainString()), + field("token", sensitiveString()), + &FieldDef{Name: "Password", OriginalName: "pass_word", Type: password}, + field("nested", class("Nested", field("label", plainString()), field("secret", sensitiveString()))), + field("tokens", &TypeDef{Type: DataTypeArray, ItemType: sensitiveString()}), + field("labels", labels), + field("credential", &TypeDef{Type: DataTypeUnion, Name: "Credential", AssociatedTypes: TypeDefs{ + class("APIKey", field("key", sensitiveString())), + class("Basic", field("user", plainString())), + }}), + field("tree", tree), + field("unrelated", class("Unrelated", field("secret", plainString()))), + ) + errorType := &TypeDef{Type: DataTypeError, Name: "Error", Fields: Fields{field("message", plainString()), field("api_key", sensitiveString())}} + + g := responseOperation(payload, errorType).SensitiveResponseBodyGraph() + require.NotNil(t, g) + require.Len(t, g.Roots, 2) + root := g.Roots + + at := func(nodes []int, path ...string) []int { + for _, step := range path { + if step == "[]" { + nodes = sensitiveItems(g, nodes) + } else { + nodes = sensitiveChild(g, nodes, step) + } + } + return nodes + } + + for _, path := range [][]string{ + {"token"}, + {"pass_word"}, + {"nested", "secret"}, + {"tokens", "[]"}, + {"labels", "undeclared"}, + {"credential", "key"}, + {"tree", "secret"}, + {"tree", "children", "[]", "children", "[]", "secret"}, + {"api_key"}, + } { + assert.True(t, isSensitiveAt(g, at(root, path...)), "%v should be sensitive", path) + } + for _, path := range [][]string{ + {"name"}, + {"Password"}, + {"nested", "label"}, + {"labels", "visible"}, + {"credential", "user"}, + {"tree", "label"}, + {"unrelated", "secret"}, + {"message"}, + } { + assert.False(t, isSensitiveAt(g, at(root, path...)), "%v should not be sensitive", path) + } +} + +func TestSensitiveBodyGraphStreamsAndAbsence(t *testing.T) { + t.Parallel() + + stream := &TypeDef{Type: DataTypeJsonL, ItemType: class("Event", field("token", sensitiveString()))} + g := responseOperation(stream).SensitiveResponseBodyGraph() + require.NotNil(t, g) + assert.True(t, isSensitiveAt(g, g.Roots), "a stream holding sensitive values is sensitive as a whole") + + plain := class("Plain", field("secret", plainString())) + assert.Nil(t, responseOperation(plain).SensitiveResponseBodyGraph()) + assert.Nil(t, responseOperation(&TypeDef{Type: DataTypeEventStream, ItemType: plain}).SensitiveResponseBodyGraph()) + assert.False(t, responseOperation(plain).HasSensitiveBodies()) +} + +func TestSensitiveRequestBodyGraph(t *testing.T) { + t.Parallel() + + body := class("Body", field("name", plainString()), field("secret", sensitiveString())) + direct := &Operation{BaseOperation: BaseOperation{Request: &Request{IsRequestBody: true, RequestBody: &FieldDef{Type: body}}}} + g := direct.SensitiveRequestBodyGraph() + require.NotNil(t, g) + assert.True(t, isSensitiveAt(g, sensitiveChild(g, g.Roots, "secret"))) + assert.True(t, direct.HasSensitiveBodies()) + + wrapped := &Operation{BaseOperation: BaseOperation{Request: &Request{Field: &FieldDef{Type: class("Request", + field("id", plainString()), + &FieldDef{Name: "Body", Type: body, Annotations: Annotations{&RequestAnnotation{}}}, + )}}}} + g = wrapped.SensitiveRequestBodyGraph() + require.NotNil(t, g) + assert.True(t, isSensitiveAt(g, sensitiveChild(g, g.Roots, "secret"))) + + assert.Nil(t, (&Operation{}).SensitiveRequestBodyGraph()) +} diff --git a/templates/templates/common/common/typeDefs.d.ts b/templates/templates/common/common/typeDefs.d.ts index d1b543c8a..46b82efb0 100644 --- a/templates/templates/common/common/typeDefs.d.ts +++ b/templates/templates/common/common/typeDefs.d.ts @@ -2096,6 +2096,43 @@ declare global { GetID(): string; GetAcceptTypes(): string[]; GetExampleSeed(): number; + /** + * Returns the locations of values marked sensitive in the request body, + * or null when it holds none. + */ + SensitiveRequestBodyGraph(): SensitiveBodyGraph | null; + /** + * Returns the locations of values marked sensitive in all response + * bodies, including errors, or null when they hold none. + */ + SensitiveResponseBodyGraph(): SensitiveBodyGraph | null; + HasSensitiveBodies(): boolean; + }; + + /** + * Locations of values marked sensitive in an operation's bodies. Nodes[0] + * is a sentinel for "nothing sensitive below here"; a reference of 0 can be + * skipped. Roots are the body schemas. + */ + type SensitiveBodyGraph = { + Nodes: SensitiveBodyNode[]; + Roots: number[]; + }; + + type SensitiveBodyNode = { + /** Masks the whole value. */ + Sensitive: boolean; + /** + * Object properties by JSON name, sorted by name. Unlisted properties use + * Values. + */ + Fields?: { Name: string; Node: number }[]; + /** Schema of array items. */ + Item: number; + /** Schema of map values and additional properties. */ + Values: number; + /** Union members. */ + Variants?: number[]; }; type OpenAPILocation = { diff --git a/templates/templates/go/includes/redact.ts b/templates/templates/go/includes/redact.ts index b9bab7cff..43ad84509 100644 --- a/templates/templates/go/includes/redact.ts +++ b/templates/templates/go/includes/redact.ts @@ -3,188 +3,50 @@ function getRedactLocation() { } registerTemplateFunc("getRedactLocation", getRedactLocation); -function isSensitiveBodyType(type: TypeDef): boolean { - const marker = type.Extensions?.All?.["x-speakeasy-param-sensitive"]; - return marker === true || marker === "true" || type.Format === "password"; -} - -interface RedactNode { - Sensitive?: boolean; - // A stream body is masked as a whole when any of its items holds a - // sensitive value, since a stream is not parsed item by item. - Stream?: boolean; - Fields?: Map; - Item?: number; - Values?: number; - Variants?: number[]; -} - -interface RedactGraph { - Nodes: RedactNode[]; - Roots: number[]; -} - -function newRedactGraph(): RedactGraph { - return { Nodes: [{}], Roots: [] }; -} - -function addRedactRoot(graph: RedactGraph, type: TypeDef | undefined) { - const walked = new Map(); - const walk = (type: TypeDef | undefined): number => { - if (!type) return 0; - const typeID = type.IsCustomType() ? getUniqueID(type) : ""; - if (typeID && walked.has(typeID)) return walked.get(typeID)!; - const id = graph.Nodes.length; - const node: RedactNode = {}; - graph.Nodes.push(node); - if (typeID) walked.set(typeID, id); - if (isSensitiveBodyType(type)) { - node.Sensitive = true; - return id; - } - switch (type.Type.toString()) { - case "class": - case "error": - node.Fields = new Map(); - for (const field of type.Fields || []) { - if (field.IsAdditionalProperties) - node.Values = walk(field.Type.ItemType); - else - node.Fields.set( - originalFieldName(field), - field.Const ? 0 : walk(field.Type), - ); - } - break; - case "array": - case "set": - node.Item = walk(type.ItemType); - break; - case "map": - node.Values = walk(type.ItemType); - break; - case "event-stream": - case "jsonl": - node.Stream = true; - node.Item = walk(type.ItemType); - break; - case "union": - node.Variants = (type.AssociatedTypes || []).map(walk); - break; - } - return id; - }; - graph.Roots.push(walk(type)); -} - -function renderRedactGraph(graph: RedactGraph): string | undefined { - const reachable = new Set(); - graph.Nodes.forEach((node, i) => { - if (node.Sensitive) reachable.add(i); - }); - let changed = true; - while (changed) { - changed = false; - graph.Nodes.forEach((node, i) => { - if (reachable.has(i)) return; - const children = [ - ...(node.Fields?.values() || []), - ...(node.Variants || []), - node.Item || 0, - node.Values || 0, - ]; - if (children.some((child) => reachable.has(child))) { - reachable.add(i); - changed = true; - } - }); - } - const roots = graph.Roots.filter((root) => reachable.has(root)); - if (roots.length === 0) return undefined; - const indices = [...reachable].sort((a, b) => a - b); - const remap = new Map(indices.map((id, i) => [id, i + 1])); - const nodes = indices.map((id) => { - const node = graph.Nodes[id]; - if (node.Stream) return "{ sensitive: true }"; +function renderRedactGraph(graph: SensitiveBodyGraph | null): string { + if (!graph) return ""; + const nodes = graph.Nodes.map((node) => { const parts: string[] = []; if (node.Sensitive) parts.push("sensitive: true"); - const values = node.Values && reachable.has(node.Values) ? node.Values : 0; - // With sensitive map values, declared fields are listed even when plain so - // that they never fall back to the map-value schema. - const fields = [...(node.Fields || [])] - .filter(([, child]) => values || reachable.has(child)) - .map( - ([name, child]) => `${JSON.stringify(name)}: ${remap.get(child) || 0}`, - ); - if (fields.length) - parts.push(`fields: map[string]int{ ${fields.join(", ")} }`); - if (node.Item && reachable.has(node.Item)) - parts.push(`item: ${remap.get(node.Item)}`); - if (values) parts.push(`values: ${remap.get(values)}`); - const variants = (node.Variants || []).filter((child) => - reachable.has(child), - ); - if (variants.length) + if (node.Fields?.length) parts.push( - `variants: []int{ ${variants - .map((child) => remap.get(child)) - .join(", ")} }`, + `fields: map[string]int{ ${node.Fields.map( + (field) => `${JSON.stringify(field.Name)}: ${field.Node}`, + ).join(", ")} }`, ); + if (node.Item) parts.push(`item: ${node.Item}`); + if (node.Values) parts.push(`values: ${node.Values}`); + if (node.Variants?.length) + parts.push(`variants: []int{ ${node.Variants.join(", ")} }`); return `{ ${parts.join(", ")} }`; }); - return `&graph{ nodes: []node{ {}, ${nodes.join( + return `&graph{ nodes: []node{ ${nodes.join( ", ", - )} }, roots: []int{ ${roots.map((id) => remap.get(id)).join(", ")} } }`; -} - -function redactGraphs(op: Operation): { - Request?: string; - Response?: string; -} { - const request = newRedactGraph(); - const body = getRequestBody(op); - addRedactRoot( - request, - body?.Type === "request" ? body.Value.RequestBody?.Type : body?.Value.Type, - ); - const response = newRedactGraph(); - for (const sub of op.Response?.Responses || []) { - for (const content of sub.Content || []) { - addRedactRoot(response, content.Content?.Type); - } - } - return { - Request: renderRedactGraph(request), - Response: renderRedactGraph(response), - }; + )} }, roots: []int{ ${graph.Roots.join(", ")} } }`; } -function hasRedactGraphs(op: Operation): boolean { - const graphs = redactGraphs(op); - return !!(graphs.Request || graphs.Response); -} -registerTemplateFunc("hasRedactGraphs", hasRedactGraphs); - function templateRedactGraphs(): { OperationID: string; - Request?: string; - Response?: string; + Request: string; + Response: string; }[] { - const entries: { - OperationID: string; - Request?: string; - Response?: string; - }[] = []; - for (const op of allOperations()) { - const graphs = redactGraphs(op); - if (graphs.Request || graphs.Response) - entries.push({ OperationID: JSON.stringify(op.ID), ...graphs }); - } - return entries.sort((a, b) => a.OperationID.localeCompare(b.OperationID)); + return allOperations() + .filter((op) => op.HasSensitiveBodies()) + .map((op) => ({ + OperationID: JSON.stringify(op.ID), + Request: renderRedactGraph(op.SensitiveRequestBodyGraph()), + Response: renderRedactGraph(op.SensitiveResponseBodyGraph()), + })) + .sort((a, b) => a.OperationID.localeCompare(b.OperationID)); } registerTemplateFunc("templateRedactGraphs", templateRedactGraphs); +function hasRedactGraphs(op: Operation): boolean { + return op.HasSensitiveBodies(); +} +registerTemplateFunc("hasRedactGraphs", hasRedactGraphs); + function hasAnyRedactGraphs(): boolean { - return templateRedactGraphs().length > 0; + return allOperations().some((op) => op.HasSensitiveBodies()); } registerTemplateFunc("hasAnyRedactGraphs", hasAnyRedactGraphs); diff --git a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock index e96f7b249..0327ab076 100644 --- a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock +++ b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock @@ -3574,7 +3574,7 @@ trackedFiles: internal/sdk/polling/option.go: last_write_checksum: sha1:69d4557bf43a1a97b2d3735c442d2d25be53e56f internal/sdk/redact/redact.go: - last_write_checksum: sha1:73715057b336608296e6ea2900255ae2f5c2a42a + last_write_checksum: sha1:ac466b2aeac1fbe0a7436e14cf2ab877794388a8 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go index 0422e4e60..5bc4ee54c 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go @@ -177,7 +177,7 @@ func (g *graph) expand(roots []int) []int { } var operations = map[string]struct{ request, response *graph }{ - "create-x-param-sensitive": {request: &graph{nodes: []node{{}, {fields: map[string]int{"string_request_and_response": 2, "string_request": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}, response: &graph{nodes: []node{{}, {fields: map[string]int{"string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, + "create-x-param-sensitive": {request: &graph{nodes: []node{{}, {fields: map[string]int{"string_request": 3, "string_request_and_response": 2}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}, response: &graph{nodes: []node{{}, {fields: map[string]int{"string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, "get-x-param-sensitive": {response: &graph{nodes: []node{{}, {fields: map[string]int{"string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, - "update-x-param-sensitive": {request: &graph{nodes: []node{{}, {fields: map[string]int{"string_request_and_response": 2, "string_request": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}, response: &graph{nodes: []node{{}, {fields: map[string]int{"string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, + "update-x-param-sensitive": {request: &graph{nodes: []node{{}, {fields: map[string]int{"string_request": 3, "string_request_and_response": 2}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}, response: &graph{nodes: []node{{}, {fields: map[string]int{"string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, } From 915ea6197c9c12fd11d303e379da9abd3ea2f1f7 Mon Sep 17 00:00:00 2001 From: Thomas Rooney Date: Thu, 8 Oct 2026 18:21:34 +0100 Subject: [PATCH 03/23] feat(go): mask sensitive values in form and multipart bodies The redact package now takes the body's Content-Type and masks application/x-www-form-urlencoded fields and multipart parts by name, including JSON-encoded fields, keeping the body valid. The sensitive body graph is now built from TypeDef.Children, with TypeDef.Walk skipping schemas that hold nothing sensitive. --- internal/ast/sensitive_body.go | 64 +++-- .../{{getDefaultErrorFileName}}.go.stmpl | 2 +- .../{{getRedactLocation}}/redact.go.stmpl | 229 ++++++++++++++---- .../primary/redact_additional_test.go.stmpl | 136 ++++++++++- .../internal/provider/utils.go.stmpl | 10 +- .../primary/sensitive-body-redaction.yaml | 127 ++++++++++ zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/sdk/redact/redact.go | 229 ++++++++++++++---- zSDKs/sdk-go/.speakeasy/gen.lock | 2 +- zSDKs/sdk-go/redact/redact.go | 229 ++++++++++++++---- .../.speakeasy/gen.lock | 6 +- .../internal/provider/utils.go | 10 +- .../internal/sdk/models/errors/sdkerror.go | 2 +- .../internal/sdk/redact/redact.go | 229 ++++++++++++++---- 14 files changed, 1056 insertions(+), 221 deletions(-) diff --git a/internal/ast/sensitive_body.go b/internal/ast/sensitive_body.go index 1ef773120..81a09f979 100644 --- a/internal/ast/sensitive_body.go +++ b/internal/ast/sensitive_body.go @@ -115,10 +115,14 @@ func newSensitiveBodyBuilder() *sensitiveBodyBuilder { return &sensitiveBodyBuilder{nodes: []*sensitiveBodyBuildNode{{}}} } -// addRoot walks a body schema. Custom types are walked once per root, by -// identity and by registration ID (which also matches the truncated copies -// that break recursive references), so that recursive schemas terminate. +// addRoot walks a body schema along the edges given by TypeDef.Children. +// Custom types are walked once per root, by identity and by registration ID +// (which also matches the truncated copies that break recursive references), +// so that recursive schemas terminate. func (b *sensitiveBodyBuilder) addRoot(t *TypeDef) { + if !containsSensitiveBodyType(t) { + return + } walkedTypes := map[*TypeDef]int{} walkedIDs := map[string]int{} var walk func(t *TypeDef) int @@ -147,35 +151,34 @@ func (b *sensitiveBodyBuilder) addRoot(t *TypeDef) { node.sensitive = true return id } - switch t.Type { - case DataTypeClass, DataTypeError: - for _, field := range t.Fields { - if field.IsAdditionalProperties { - if field.Type != nil { - node.values = walk(field.Type.ItemType) - } - continue + for field, child := range t.Children { + switch { + case field == nil: + switch t.Type { + case DataTypeArray, DataTypeSet: + node.item = walk(child) + case DataTypeMap: + node.values = walk(child) + case DataTypeEventStream, DataTypeJsonL: + node.stream = true + node.item = walk(child) + case DataTypeUnion: + node.variants = append(node.variants, walk(child)) + } + case t.Type != DataTypeClass && t.Type != DataTypeError: + case field.IsAdditionalProperties: + if child != nil { + node.values = walk(child.ItemType) } + default: name := field.OriginalName if name == "" { name = field.Name } - child := 0 + node.fields = append(node.fields, SensitiveBodyField{Name: name}) if field.Const == nil { - child = walk(field.Type) + node.fields[len(node.fields)-1].Node = walk(child) } - node.fields = append(node.fields, SensitiveBodyField{Name: name, Node: child}) - } - case DataTypeArray, DataTypeSet: - node.item = walk(t.ItemType) - case DataTypeMap: - node.values = walk(t.ItemType) - case DataTypeEventStream, DataTypeJsonL: - node.stream = true - node.item = walk(t.ItemType) - case DataTypeUnion: - for _, variant := range t.AssociatedTypes { - node.variants = append(node.variants, walk(variant)) } } return id @@ -183,6 +186,17 @@ func (b *sensitiveBodyBuilder) addRoot(t *TypeDef) { b.roots = append(b.roots, walk(t)) } +// containsSensitiveBodyType reports whether t or any type below it is marked +// sensitive, so that schemas without sensitive values are not walked further. +func containsSensitiveBodyType(t *TypeDef) bool { + for _, typ := range t.Walk() { + if isSensitiveBodyType(typ) { + return true + } + } + return false +} + // build prunes the nodes that cannot reach a sensitive value and renumbers // the rest. func (b *sensitiveBodyBuilder) build() *SensitiveBodyGraph { diff --git a/templates/templates/go/auxiliary/{{getErrorsLocation}}/{{getDefaultErrorFileName}}.go.stmpl b/templates/templates/go/auxiliary/{{getErrorsLocation}}/{{getDefaultErrorFileName}}.go.stmpl index 11bd54f08..0b22c5efe 100644 --- a/templates/templates/go/auxiliary/{{getErrorsLocation}}/{{getDefaultErrorFileName}}.go.stmpl +++ b/templates/templates/go/auxiliary/{{getErrorsLocation}}/{{getDefaultErrorFileName}}.go.stmpl @@ -34,7 +34,7 @@ func (e *{{$errorName}}) Error() string { {{- if hasAnyRedactGraphs}} redacted := e.Body if e.RawResponse != nil && e.RawResponse.Request != nil { - redacted = string(redact.ResponseBody(e.RawResponse.Request.Context(), []byte(e.Body))) + redacted = string(redact.ResponseBody(e.RawResponse.Request.Context(), e.RawResponse.Header.Get("Content-Type"), []byte(e.Body))) } body = fmt.Sprintf("\n%s", redacted) {{- else}} diff --git a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl index d277a9864..0ab74539b 100644 --- a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl +++ b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl @@ -7,8 +7,12 @@ // back from the request, or from a response's Request, to select the schema // graph used for masking: // -// body = redact.RequestBody(req.Context(), body) -// body = redact.ResponseBody(res.Request.Context(), body) +// body = redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), body) +// body = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), body) +// +// JSON, application/x-www-form-urlencoded and multipart bodies are masked +// value by value. Other bodies of an operation with sensitive values are +// masked as a whole. package redact import ( @@ -16,6 +20,10 @@ import ( "context" "encoding/json" "io" + "mime" + "mime/multipart" + "net/url" + "strings" ) // Mask replaces each sensitive value. @@ -40,16 +48,18 @@ func Operation(ctx context.Context) string { } // RequestBody masks the sensitive values of the request body of the -// operation carried by ctx. Bodies of other operations are returned as is. -func RequestBody(ctx context.Context, body []byte) []byte { - return operations[Operation(ctx)].request.redact(body) +// operation carried by ctx, serialized as contentType. Bodies of other +// operations are returned as is. +func RequestBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].request.redact(contentType, body) } // ResponseBody masks the sensitive values of a response body of the -// operation carried by ctx, matching every response schema of the operation. -// Bodies of other operations are returned as is. -func ResponseBody(ctx context.Context, body []byte) []byte { - return operations[Operation(ctx)].response.redact(body) +// operation carried by ctx, serialized as contentType, matching every +// response schema of the operation. Bodies of other operations are returned +// as is. +func ResponseBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].response.redact(contentType, body) } // A graph describes where sensitive values sit in a body. Node 0 is a @@ -70,36 +80,149 @@ type node struct { } // redact returns body unchanged when it has nothing to mask. A body that -// cannot be parsed as JSON is masked as a whole. -func (g *graph) redact(body []byte) []byte { +// cannot be parsed is masked as a whole. +func (g *graph) redact(contentType string, body []byte) []byte { if g == nil || len(body) == 0 { return body } - for _, id := range g.expand(g.roots) { - if g.nodes[id].sensitive { - return []byte(Mask) - } + if g.sensitive(g.roots) { + return []byte(Mask) + } + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil { + mediaType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) } + var masked []byte + ok := false + switch { + case mediaType == "application/x-www-form-urlencoded": + masked, _, ok = g.redactForm(body, g.roots) + case strings.HasPrefix(mediaType, "multipart/"): + masked, _, ok = g.redactMultipart(body, params["boundary"], g.roots) + case mediaType == "" || mediaType == "application/json" || mediaType == "text/json" || strings.HasSuffix(mediaType, "+json"): + masked, _, ok = g.redactJSON(body, g.roots) + } + if !ok { + return []byte(Mask) + } + return masked +} + +func (g *graph) redactJSON(body []byte, roots []int) ([]byte, bool, bool) { dec := json.NewDecoder(bytes.NewReader(body)) dec.UseNumber() var value any if err := dec.Decode(&value); err != nil { - return []byte(Mask) + return nil, false, false } if _, err := dec.Token(); err != io.EOF { - return []byte(Mask) + return nil, false, false } - masked, changed := g.mask(value, g.roots, 0) + masked, changed := g.mask(value, roots, 0) if !changed { - return body + return body, false, true } var out bytes.Buffer enc := json.NewEncoder(&out) enc.SetEscapeHTML(false) if err := enc.Encode(masked); err != nil { - return []byte(Mask) + return nil, false, false + } + return bytes.TrimSuffix(out.Bytes(), []byte("\n")), true, true +} + +// redactForm masks the fields of a form body. A field holding a sensitive +// value or array of values is masked, and a JSON-encoded field is masked +// within. +func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { + pairs := strings.Split(string(body), "&") + changed := false + for i, pair := range pairs { + rawKey, rawValue, _ := strings.Cut(pair, "=") + key, err := url.QueryUnescape(rawKey) + if err != nil { + return nil, false, false + } + value, err := url.QueryUnescape(rawValue) + if err != nil { + return nil, false, false + } + masked, fieldChanged := g.redactValue([]byte(value), g.child(roots, strings.TrimSuffix(key, "[]"))) + if !fieldChanged { + continue + } + changed = true + if string(masked) == Mask { + pairs[i] = rawKey + "=" + Mask + } else { + pairs[i] = rawKey + "=" + url.QueryEscape(string(masked)) + } + } + if !changed { + return body, false, true + } + return []byte(strings.Join(pairs, "&")), true, true +} + +// redactMultipart masks the parts of a multipart body by part name, keeping +// the boundary and the part headers. +func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]byte, bool, bool) { + if boundary == "" { + return nil, false, false + } + reader := multipart.NewReader(bytes.NewReader(body), boundary) + var out bytes.Buffer + writer := multipart.NewWriter(&out) + if err := writer.SetBoundary(boundary); err != nil { + return nil, false, false + } + changed := false + for { + part, err := reader.NextRawPart() + if err == io.EOF { + break + } + if err != nil { + return nil, false, false + } + data, err := io.ReadAll(part) + if err != nil { + return nil, false, false + } + masked, partChanged := g.redactValue(data, g.child(roots, strings.TrimSuffix(part.FormName(), "[]"))) + changed = changed || partChanged + partWriter, err := writer.CreatePart(part.Header) + if err != nil { + return nil, false, false + } + if _, err := partWriter.Write(masked); err != nil { + return nil, false, false + } + } + if !changed { + return body, false, true } - return bytes.TrimSuffix(out.Bytes(), []byte("\n")) + if err := writer.Close(); err != nil { + return nil, false, false + } + return out.Bytes(), true, true +} + +// redactValue masks a form field or multipart part. A value holding a +// sensitive value, or one that cannot be parsed as JSON while holding one, is +// masked as a whole. +func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { + if len(g.expand(roots)) == 0 { + return value, false + } + if g.sensitive(roots) || g.sensitive(g.items(roots)) { + return []byte(Mask), true + } + masked, changed, ok := g.redactJSON(value, roots) + if !ok { + return []byte(Mask), true + } + return masked, changed } func (g *graph) mask(value any, roots []int, depth int) (any, bool) { @@ -107,10 +230,8 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { if len(roots) == 0 { return value, false } - for _, id := range roots { - if g.nodes[id].sensitive { - return Mask, true - } + if g.sensitive(roots) { + return Mask, true } if depth >= maxDepth { return Mask, true @@ -119,29 +240,13 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { switch v := value.(type) { case map[string]any: for key, child := range v { - var childRoots []int - for _, id := range roots { - n := g.nodes[id] - childID, declared := n.fields[key] - if !declared { - childID = n.values - } - if childID != 0 { - childRoots = append(childRoots, childID) - } - } - if masked, ok := g.mask(child, childRoots, depth+1); ok { + if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { v[key] = masked changed = true } } case []any: - var items []int - for _, id := range roots { - if item := g.nodes[id].item; item != 0 { - items = append(items, item) - } - } + items := g.items(roots) for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { v[i] = masked @@ -152,6 +257,44 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { return value, changed } +// sensitive reports whether any of roots, or their union variants, is +// sensitive as a whole. +func (g *graph) sensitive(roots []int) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].sensitive { + return true + } + } + return false +} + +// child returns the schemas of the property name of roots. +func (g *graph) child(roots []int, name string) []int { + var children []int + for _, id := range g.expand(roots) { + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) + } + } + return children +} + +// items returns the array item schemas of roots. +func (g *graph) items(roots []int) []int { + var items []int + for _, id := range g.expand(roots) { + if item := g.nodes[id].item; item != 0 { + items = append(items, item) + } + } + return items +} + // expand adds the union variants of roots, since the variant a value matches // is not known. func (g *graph) expand(roots []int) []int { diff --git a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl index 19f0a1d79..9ac72fd29 100644 --- a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl +++ b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl @@ -1,16 +1,21 @@ package tests import ( + "bytes" "context" "errors" "io" + "mime" + "mime/multipart" "net/http" + "net/url" "strings" "testing" sdk "{{getRootModulePath}}" "{{getRootModulePath}}/{{getRedactLocation}}" "{{getRootModulePath}}/pkg/models/apierrors" + "{{getRootModulePath}}/pkg/models/operations" "{{getRootModulePath}}/pkg/models/shared" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -31,12 +36,27 @@ const sensitiveBodyPayload = `{ }` type redactCaptureClient struct { - status int - operation string + status int + operation string + contentType string + body []byte + masked []byte } +// Do records the request body as sent and as masked for the operation +// carried by the request context. func (c *redactCaptureClient) Do(req *http.Request) (*http.Response, error) { c.operation = redact.Operation(req.Context()) + c.contentType = req.Header.Get("Content-Type") + c.body = nil + if req.Body != nil { + body, err := io.ReadAll(req.Body) + if err != nil { + return nil, err + } + c.body = body + } + c.masked = redact.RequestBody(req.Context(), c.contentType, c.body) return &http.Response{ StatusCode: c.status, Header: http.Header{"Content-Type": []string{"application/json"}}, @@ -45,6 +65,16 @@ func (c *redactCaptureClient) Do(req *http.Request) (*http.Response, error) { }, nil } +func assertMasked(t *testing.T, output string, secrets, visible []string) { + t.Helper() + for _, secret := range secrets { + assert.NotContains(t, output, secret) + } + for _, value := range visible { + assert.Contains(t, output, value) + } +} + func assertSensitiveBodyMasked(t *testing.T, output string) { t.Helper() for _, secret := range []string{"secret-password", "secret-token", "secret-nested", "secret-item", "secret-tokens-0", "secret-tokens-1", "secret-label", "secret-credential", "secret-root", "secret-child", "secret-whole"} { @@ -58,20 +88,21 @@ func assertSensitiveBodyMasked(t *testing.T, output string) { func TestRedactSensitiveBodies(t *testing.T) { ctx := redact.WithOperation(context.Background(), "sensitiveBodyRedaction") - assertSensitiveBodyMasked(t, string(redact.RequestBody(ctx, []byte(sensitiveBodyPayload)))) - assertSensitiveBodyMasked(t, string(redact.ResponseBody(ctx, []byte(sensitiveBodyPayload)))) + assertSensitiveBodyMasked(t, string(redact.RequestBody(ctx, "application/json", []byte(sensitiveBodyPayload)))) + assertSensitiveBodyMasked(t, string(redact.ResponseBody(ctx, "application/json; charset=utf-8", []byte(sensitiveBodyPayload)))) plain := []byte(`{"name": "visible-name"}`) - assert.Equal(t, plain, redact.ResponseBody(ctx, plain), "bodies without sensitive values are returned unchanged") - assert.Equal(t, redact.Mask, string(redact.ResponseBody(ctx, []byte("secret-text"))), "unparseable bodies are masked as a whole") - assert.Equal(t, []byte(sensitiveBodyPayload), redact.ResponseBody(context.Background(), []byte(sensitiveBodyPayload)), "requests without an operation are not masked") + assert.Equal(t, plain, redact.ResponseBody(ctx, "application/json", plain), "bodies without sensitive values are returned unchanged") + assert.Equal(t, redact.Mask, string(redact.ResponseBody(ctx, "application/json", []byte("secret-text"))), "unparseable bodies are masked as a whole") + assert.Equal(t, redact.Mask, string(redact.ResponseBody(ctx, "text/plain", []byte(`{"name": "visible-name"}`))), "bodies of other media types are masked as a whole") + assert.Equal(t, []byte(sensitiveBodyPayload), redact.ResponseBody(context.Background(), "application/json", []byte(sensitiveBodyPayload)), "requests without an operation are not masked") - errorBody := string(redact.ResponseBody(ctx, []byte(`{"message": "visible-message", "secret": "secret-error"}`))) + errorBody := string(redact.ResponseBody(ctx, "application/json", []byte(`{"message": "visible-message", "secret": "secret-error"}`))) assert.Contains(t, errorBody, "visible-message") assert.NotContains(t, errorBody, "secret-error", "error response schemas are masked") streamCtx := redact.WithOperation(context.Background(), "sensitiveStreamRedaction") - assert.Equal(t, redact.Mask, string(redact.ResponseBody(streamCtx, []byte(`{"label": "visible", "secret": "secret-stream"}`))), "streams holding sensitive values are masked as a whole") + assert.Equal(t, redact.Mask, string(redact.ResponseBody(streamCtx, "application/jsonl", []byte(`{"label": "visible", "secret": "secret-stream"}`))), "streams holding sensitive values are masked as a whole") } func TestRedactOperationOnRequestContext(t *testing.T) { @@ -91,3 +122,90 @@ func TestRedactOperationOnRequestContext(t *testing.T) { assert.Contains(t, apiErr.Body, "secret-password", "the error body is not masked") assertSensitiveBodyMasked(t, apiErr.Error()) } + +func TestRedactFormRequestBody(t *testing.T) { + client := &redactCaptureClient{status: http.StatusOK} + s := sdk.New(sdk.WithServerURL("http://localhost"), sdk.WithClient(client)) + + _, err := s.SensitiveBodies.FormRedaction(context.Background(), shared.SensitiveBodyForm{ + Name: "visible-name", + Token: sdk.String("secret-token"), + Tokens: []string{"secret-tokens-0", "secret-tokens-1"}, + Nested: &shared.SensitiveBodyFormNested{VisibleProperty: sdk.String("visible-nested"), SecretProperty: sdk.String("secret-nested")}, + Metadata: &shared.SensitiveBodyNested{Label: sdk.String("visible-metadata"), Secret: sdk.String("secret-metadata")}, + }) + require.NoError(t, err) + assert.Equal(t, "sensitiveFormRedaction", client.operation) + assert.Contains(t, string(client.body), "secret-token", "the body sent is not masked") + + masked := string(client.masked) + assertMasked(t, masked, + []string{"secret-token", "secret-tokens-0", "secret-tokens-1", "secret-nested", "secret-metadata"}, + []string{"name=visible-name", "visible-nested", "visible-metadata"}) + values, err := url.ParseQuery(masked) + require.NoError(t, err, "the masked body is a form") + assert.Equal(t, []string{redact.Mask, redact.Mask}, values["tokens"]) + assert.JSONEq(t, `{"label": "visible-metadata", "secret": "`+redact.Mask+`"}`, values.Get("metadata")) + + plainCtx := redact.WithOperation(context.Background(), "sensitiveFormRedaction") + plain := []byte("name=visible-name&metadata=%7B%22label%22%3A%22visible%22%7D") + assert.Equal(t, plain, redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", plain), "forms without sensitive values are returned unchanged") + assert.Equal(t, "name=visible-name&metadata="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("name=visible-name&metadata=secret-text"))), "unparseable JSON-encoded fields are masked") +} + +func TestRedactMultipartRequestBody(t *testing.T) { + client := &redactCaptureClient{status: http.StatusOK} + s := sdk.New(sdk.WithServerURL("http://localhost"), sdk.WithClient(client)) + + _, err := s.SensitiveBodies.MultipartRedaction(context.Background(), shared.SensitiveBodyMultipart{ + Name: "visible-name", + Token: sdk.String("secret-token"), + Tokens: []string{"secret-tokens-0", "secret-tokens-1"}, + Metadata: &shared.SensitiveBodyNested{Label: sdk.String("visible-metadata"), Secret: sdk.String("secret-metadata")}, + File: &shared.File{FileName: "visible.txt", Content: []byte("visible-file-content")}, + }) + require.NoError(t, err) + assert.Equal(t, "sensitiveMultipartRedaction", client.operation) + assert.Contains(t, string(client.body), "secret-token", "the body sent is not masked") + + assertMasked(t, string(client.masked), + []string{"secret-token", "secret-tokens-0", "secret-tokens-1", "secret-metadata"}, + []string{"visible-name", "visible-metadata", "visible.txt", "visible-file-content"}) + + _, params, err := mime.ParseMediaType(client.contentType) + require.NoError(t, err) + reader := multipart.NewReader(bytes.NewReader(client.masked), params["boundary"]) + parts := map[string][]string{} + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + require.NoError(t, err, "the masked body is multipart with the same boundary") + data, err := io.ReadAll(part) + require.NoError(t, err) + parts[part.FormName()] = append(parts[part.FormName()], string(data)) + } + assert.Equal(t, []string{"visible-name"}, parts["name"]) + assert.Equal(t, []string{redact.Mask}, parts["token"]) + assert.Equal(t, []string{redact.Mask, redact.Mask}, parts["tokens"]) + assert.Equal(t, []string{"visible-file-content"}, parts["file"]) + + ctx := redact.WithOperation(context.Background(), "sensitiveMultipartRedaction") + assert.Equal(t, redact.Mask, string(redact.RequestBody(ctx, "multipart/form-data", client.body)), "multipart bodies without a boundary are masked as a whole") +} + +func TestRedactSplitRequestBodies(t *testing.T) { + client := &redactCaptureClient{status: http.StatusOK} + s := sdk.New(sdk.WithServerURL("http://localhost"), sdk.WithClient(client)) + + _, err := s.SensitiveBodies.SplitRedactionJSON(context.Background(), operations.SensitiveSplitRedactionJSONRequestBody{Name: "visible-name", JSONSecret: "secret-json"}) + require.NoError(t, err) + assert.Equal(t, "sensitiveSplitRedaction_json", client.operation) + assertMasked(t, string(client.masked), []string{"secret-json"}, []string{"visible-name"}) + + _, err = s.SensitiveBodies.SplitRedactionForm(context.Background(), operations.SensitiveSplitRedactionFormRequestBody{Name: "visible-name", FormSecret: "secret-form"}) + require.NoError(t, err) + assert.Equal(t, "sensitiveSplitRedaction_form", client.operation) + assertMasked(t, string(client.masked), []string{"secret-form"}, []string{"name=visible-name"}) +} diff --git a/templates/templates/terraform/auxiliary/internal/provider/utils.go.stmpl b/templates/templates/terraform/auxiliary/internal/provider/utils.go.stmpl index f9814597d..09bd99bec 100644 --- a/templates/templates/terraform/auxiliary/internal/provider/utils.go.stmpl +++ b/templates/templates/terraform/auxiliary/internal/provider/utils.go.stmpl @@ -42,10 +42,12 @@ func debugResponse(response *http.Response) string { if response.Body != nil { if body, err := io.ReadAll(response.Body); err == nil { response.Body = io.NopCloser(bytes.NewReader(body)) - dumpRes = append(dumpRes, redact.ResponseBody(ctx, body)...) + dumpRes = append(dumpRes, redact.ResponseBody(ctx, response.Header.Get("Content-Type"), body)...) } } - dumpReq = redactDumpBody(dumpReq, func(body []byte) []byte { return redact.RequestBody(ctx, body) }) + dumpReq = redactDumpBody(dumpReq, func(body []byte) []byte { + return redact.RequestBody(ctx, response.Request.Header.Get("Content-Type"), body) + }) return redactSensitiveValues(ctx, fmt.Sprintf("**Request**:\n%s\n**Response**:\n%s", string(dumpReq), string(dumpRes))) } @@ -224,7 +226,7 @@ func decomposeRequestForLogging(req *http.Request) (map[string]interface{}, erro } // Read the rest of the body content - fields[FieldHttpRequestBody] = string(redact.RequestBody(req.Context(), []byte(bodyFromRestOfRequestReader(reqReader)))) + fields[FieldHttpRequestBody] = string(redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), []byte(bodyFromRestOfRequestReader(reqReader)))) redactSensitiveFields(req, fields) return fields, nil } @@ -300,7 +302,7 @@ func decomposeResponseForLogging(res *http.Response) (map[string]interface{}, er res.Body = io.NopCloser(bytes.NewBuffer(resBody)) if res.Request != nil { - resBody = redact.ResponseBody(res.Request.Context(), resBody) + resBody = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), resBody) } fields[FieldHttpResponseBody] = string(resBody) redactSensitiveFields(res.Request, fields) diff --git a/tests/specs/fragments/primary/sensitive-body-redaction.yaml b/tests/specs/fragments/primary/sensitive-body-redaction.yaml index 646b2d1dd..6af9f765c 100644 --- a/tests/specs/fragments/primary/sensitive-body-redaction.yaml +++ b/tests/specs/fragments/primary/sensitive-body-redaction.yaml @@ -57,6 +57,86 @@ paths: application/jsonl: schema: $ref: "#/components/schemas/SensitiveBodyNested" + /anything/sensitiveFormRedaction: + post: + operationId: sensitiveFormRedaction + x-speakeasy-group: sensitiveBodies + x-speakeasy-name-override: formRedaction + description: >- + Exercises locating values marked sensitive in a form-urlencoded request + body by field name: a sensitive field, an array of sensitive values + sent as repeated fields, an object-valued field holding a sensitive + property, and a JSON-encoded field holding a sensitive property. Plain + fields stay readable. + requestBody: + required: true + content: + application/x-www-form-urlencoded: + schema: + $ref: "#/components/schemas/SensitiveBodyForm" + encoding: + metadata: + contentType: application/json + responses: + "200": + description: OK + /anything/sensitiveMultipartRedaction: + post: + operationId: sensitiveMultipartRedaction + x-speakeasy-group: sensitiveBodies + x-speakeasy-name-override: multipartRedaction + description: >- + Exercises locating values marked sensitive in a multipart request body + by part name: a sensitive part, an array of sensitive values sent as + repeated parts, and a JSON-encoded part holding a sensitive property. + Plain parts and file parts stay readable. + requestBody: + required: true + content: + multipart/form-data: + schema: + $ref: "#/components/schemas/SensitiveBodyMultipart" + encoding: + metadata: + contentType: application/json + responses: + "200": + description: OK + /anything/sensitiveSplitRedaction: + post: + operationId: sensitiveSplitRedaction + x-speakeasy-group: sensitiveBodies + x-speakeasy-name-override: splitRedaction + description: >- + Exercises a request body with distinct schemas per content type, each + holding a different sensitive property, so that the body sent in each + content type is masked with its own schema. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [name, jsonSecret] + properties: + name: + type: string + jsonSecret: + type: string + x-speakeasy-param-sensitive: true + application/x-www-form-urlencoded: + schema: + type: object + required: [name, formSecret] + properties: + name: + type: string + formSecret: + type: string + x-speakeasy-param-sensitive: true + responses: + "200": + description: OK components: schemas: SensitiveBodyPayload: @@ -164,3 +244,50 @@ components: properties: secret: type: string + SensitiveBodyForm: + type: object + required: + - name + properties: + name: + type: string + token: + type: string + x-speakeasy-param-sensitive: true + tokens: + type: array + items: + type: string + x-speakeasy-param-sensitive: true + nested: + $ref: "#/components/schemas/SensitiveBodyFormNested" + metadata: + $ref: "#/components/schemas/SensitiveBodyNested" + SensitiveBodyFormNested: + type: object + properties: + visibleProperty: + type: string + secretProperty: + type: string + x-speakeasy-param-sensitive: true + SensitiveBodyMultipart: + type: object + required: + - name + properties: + name: + type: string + token: + type: string + x-speakeasy-param-sensitive: true + tokens: + type: array + items: + type: string + x-speakeasy-param-sensitive: true + metadata: + $ref: "#/components/schemas/SensitiveBodyNested" + file: + type: string + format: binary diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index 61f3de679..a778c5dad 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -564,7 +564,7 @@ trackedFiles: internal/sdk/optionalnullable/optionalnullable_test.go: last_write_checksum: sha1:d3991667754ed67db803aeab4a6239f6f9a1be37 internal/sdk/redact/redact.go: - last_write_checksum: sha1:8090b7e04d1365758dde8e2807b14ecb697b5257 + last_write_checksum: sha1:0ef73c0bcaba31b2cf207cca3fcb38c52066fd02 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/sdk-cli/internal/sdk/redact/redact.go b/zSDKs/sdk-cli/internal/sdk/redact/redact.go index 98b5dfe1b..f46c8700d 100644 --- a/zSDKs/sdk-cli/internal/sdk/redact/redact.go +++ b/zSDKs/sdk-cli/internal/sdk/redact/redact.go @@ -9,8 +9,12 @@ // back from the request, or from a response's Request, to select the schema // graph used for masking: // -// body = redact.RequestBody(req.Context(), body) -// body = redact.ResponseBody(res.Request.Context(), body) +// body = redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), body) +// body = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), body) +// +// JSON, application/x-www-form-urlencoded and multipart bodies are masked +// value by value. Other bodies of an operation with sensitive values are +// masked as a whole. package redact import ( @@ -18,6 +22,10 @@ import ( "context" "encoding/json" "io" + "mime" + "mime/multipart" + "net/url" + "strings" ) // Mask replaces each sensitive value. @@ -42,16 +50,18 @@ func Operation(ctx context.Context) string { } // RequestBody masks the sensitive values of the request body of the -// operation carried by ctx. Bodies of other operations are returned as is. -func RequestBody(ctx context.Context, body []byte) []byte { - return operations[Operation(ctx)].request.redact(body) +// operation carried by ctx, serialized as contentType. Bodies of other +// operations are returned as is. +func RequestBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].request.redact(contentType, body) } // ResponseBody masks the sensitive values of a response body of the -// operation carried by ctx, matching every response schema of the operation. -// Bodies of other operations are returned as is. -func ResponseBody(ctx context.Context, body []byte) []byte { - return operations[Operation(ctx)].response.redact(body) +// operation carried by ctx, serialized as contentType, matching every +// response schema of the operation. Bodies of other operations are returned +// as is. +func ResponseBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].response.redact(contentType, body) } // A graph describes where sensitive values sit in a body. Node 0 is a @@ -72,36 +82,149 @@ type node struct { } // redact returns body unchanged when it has nothing to mask. A body that -// cannot be parsed as JSON is masked as a whole. -func (g *graph) redact(body []byte) []byte { +// cannot be parsed is masked as a whole. +func (g *graph) redact(contentType string, body []byte) []byte { if g == nil || len(body) == 0 { return body } - for _, id := range g.expand(g.roots) { - if g.nodes[id].sensitive { - return []byte(Mask) - } + if g.sensitive(g.roots) { + return []byte(Mask) + } + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil { + mediaType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) } + var masked []byte + ok := false + switch { + case mediaType == "application/x-www-form-urlencoded": + masked, _, ok = g.redactForm(body, g.roots) + case strings.HasPrefix(mediaType, "multipart/"): + masked, _, ok = g.redactMultipart(body, params["boundary"], g.roots) + case mediaType == "" || mediaType == "application/json" || mediaType == "text/json" || strings.HasSuffix(mediaType, "+json"): + masked, _, ok = g.redactJSON(body, g.roots) + } + if !ok { + return []byte(Mask) + } + return masked +} + +func (g *graph) redactJSON(body []byte, roots []int) ([]byte, bool, bool) { dec := json.NewDecoder(bytes.NewReader(body)) dec.UseNumber() var value any if err := dec.Decode(&value); err != nil { - return []byte(Mask) + return nil, false, false } if _, err := dec.Token(); err != io.EOF { - return []byte(Mask) + return nil, false, false } - masked, changed := g.mask(value, g.roots, 0) + masked, changed := g.mask(value, roots, 0) if !changed { - return body + return body, false, true } var out bytes.Buffer enc := json.NewEncoder(&out) enc.SetEscapeHTML(false) if err := enc.Encode(masked); err != nil { - return []byte(Mask) + return nil, false, false + } + return bytes.TrimSuffix(out.Bytes(), []byte("\n")), true, true +} + +// redactForm masks the fields of a form body. A field holding a sensitive +// value or array of values is masked, and a JSON-encoded field is masked +// within. +func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { + pairs := strings.Split(string(body), "&") + changed := false + for i, pair := range pairs { + rawKey, rawValue, _ := strings.Cut(pair, "=") + key, err := url.QueryUnescape(rawKey) + if err != nil { + return nil, false, false + } + value, err := url.QueryUnescape(rawValue) + if err != nil { + return nil, false, false + } + masked, fieldChanged := g.redactValue([]byte(value), g.child(roots, strings.TrimSuffix(key, "[]"))) + if !fieldChanged { + continue + } + changed = true + if string(masked) == Mask { + pairs[i] = rawKey + "=" + Mask + } else { + pairs[i] = rawKey + "=" + url.QueryEscape(string(masked)) + } + } + if !changed { + return body, false, true + } + return []byte(strings.Join(pairs, "&")), true, true +} + +// redactMultipart masks the parts of a multipart body by part name, keeping +// the boundary and the part headers. +func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]byte, bool, bool) { + if boundary == "" { + return nil, false, false + } + reader := multipart.NewReader(bytes.NewReader(body), boundary) + var out bytes.Buffer + writer := multipart.NewWriter(&out) + if err := writer.SetBoundary(boundary); err != nil { + return nil, false, false + } + changed := false + for { + part, err := reader.NextRawPart() + if err == io.EOF { + break + } + if err != nil { + return nil, false, false + } + data, err := io.ReadAll(part) + if err != nil { + return nil, false, false + } + masked, partChanged := g.redactValue(data, g.child(roots, strings.TrimSuffix(part.FormName(), "[]"))) + changed = changed || partChanged + partWriter, err := writer.CreatePart(part.Header) + if err != nil { + return nil, false, false + } + if _, err := partWriter.Write(masked); err != nil { + return nil, false, false + } + } + if !changed { + return body, false, true } - return bytes.TrimSuffix(out.Bytes(), []byte("\n")) + if err := writer.Close(); err != nil { + return nil, false, false + } + return out.Bytes(), true, true +} + +// redactValue masks a form field or multipart part. A value holding a +// sensitive value, or one that cannot be parsed as JSON while holding one, is +// masked as a whole. +func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { + if len(g.expand(roots)) == 0 { + return value, false + } + if g.sensitive(roots) || g.sensitive(g.items(roots)) { + return []byte(Mask), true + } + masked, changed, ok := g.redactJSON(value, roots) + if !ok { + return []byte(Mask), true + } + return masked, changed } func (g *graph) mask(value any, roots []int, depth int) (any, bool) { @@ -109,10 +232,8 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { if len(roots) == 0 { return value, false } - for _, id := range roots { - if g.nodes[id].sensitive { - return Mask, true - } + if g.sensitive(roots) { + return Mask, true } if depth >= maxDepth { return Mask, true @@ -121,29 +242,13 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { switch v := value.(type) { case map[string]any: for key, child := range v { - var childRoots []int - for _, id := range roots { - n := g.nodes[id] - childID, declared := n.fields[key] - if !declared { - childID = n.values - } - if childID != 0 { - childRoots = append(childRoots, childID) - } - } - if masked, ok := g.mask(child, childRoots, depth+1); ok { + if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { v[key] = masked changed = true } } case []any: - var items []int - for _, id := range roots { - if item := g.nodes[id].item; item != 0 { - items = append(items, item) - } - } + items := g.items(roots) for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { v[i] = masked @@ -154,6 +259,44 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { return value, changed } +// sensitive reports whether any of roots, or their union variants, is +// sensitive as a whole. +func (g *graph) sensitive(roots []int) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].sensitive { + return true + } + } + return false +} + +// child returns the schemas of the property name of roots. +func (g *graph) child(roots []int, name string) []int { + var children []int + for _, id := range g.expand(roots) { + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) + } + } + return children +} + +// items returns the array item schemas of roots. +func (g *graph) items(roots []int) []int { + var items []int + for _, id := range g.expand(roots) { + if item := g.nodes[id].item; item != 0 { + items = append(items, item) + } + } + return items +} + // expand adds the union variants of roots, since the variant a value matches // is not known. func (g *graph) expand(roots []int) []int { diff --git a/zSDKs/sdk-go/.speakeasy/gen.lock b/zSDKs/sdk-go/.speakeasy/gen.lock index b6a56f5e2..e2afa9f6f 100644 --- a/zSDKs/sdk-go/.speakeasy/gen.lock +++ b/zSDKs/sdk-go/.speakeasy/gen.lock @@ -783,7 +783,7 @@ trackedFiles: recursiveformfield.go: last_write_checksum: sha1:2e06532b8c5445500bd2ae8ae8586d24e911a3e2 redact/redact.go: - last_write_checksum: sha1:8090b7e04d1365758dde8e2807b14ecb697b5257 + last_write_checksum: sha1:0ef73c0bcaba31b2cf207cca3fcb38c52066fd02 renderassetop.go: last_write_checksum: sha1:a788e389967e88581337759b330af30c4e562bf8 requesttimeouterror.go: diff --git a/zSDKs/sdk-go/redact/redact.go b/zSDKs/sdk-go/redact/redact.go index 98b5dfe1b..f46c8700d 100644 --- a/zSDKs/sdk-go/redact/redact.go +++ b/zSDKs/sdk-go/redact/redact.go @@ -9,8 +9,12 @@ // back from the request, or from a response's Request, to select the schema // graph used for masking: // -// body = redact.RequestBody(req.Context(), body) -// body = redact.ResponseBody(res.Request.Context(), body) +// body = redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), body) +// body = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), body) +// +// JSON, application/x-www-form-urlencoded and multipart bodies are masked +// value by value. Other bodies of an operation with sensitive values are +// masked as a whole. package redact import ( @@ -18,6 +22,10 @@ import ( "context" "encoding/json" "io" + "mime" + "mime/multipart" + "net/url" + "strings" ) // Mask replaces each sensitive value. @@ -42,16 +50,18 @@ func Operation(ctx context.Context) string { } // RequestBody masks the sensitive values of the request body of the -// operation carried by ctx. Bodies of other operations are returned as is. -func RequestBody(ctx context.Context, body []byte) []byte { - return operations[Operation(ctx)].request.redact(body) +// operation carried by ctx, serialized as contentType. Bodies of other +// operations are returned as is. +func RequestBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].request.redact(contentType, body) } // ResponseBody masks the sensitive values of a response body of the -// operation carried by ctx, matching every response schema of the operation. -// Bodies of other operations are returned as is. -func ResponseBody(ctx context.Context, body []byte) []byte { - return operations[Operation(ctx)].response.redact(body) +// operation carried by ctx, serialized as contentType, matching every +// response schema of the operation. Bodies of other operations are returned +// as is. +func ResponseBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].response.redact(contentType, body) } // A graph describes where sensitive values sit in a body. Node 0 is a @@ -72,36 +82,149 @@ type node struct { } // redact returns body unchanged when it has nothing to mask. A body that -// cannot be parsed as JSON is masked as a whole. -func (g *graph) redact(body []byte) []byte { +// cannot be parsed is masked as a whole. +func (g *graph) redact(contentType string, body []byte) []byte { if g == nil || len(body) == 0 { return body } - for _, id := range g.expand(g.roots) { - if g.nodes[id].sensitive { - return []byte(Mask) - } + if g.sensitive(g.roots) { + return []byte(Mask) + } + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil { + mediaType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) } + var masked []byte + ok := false + switch { + case mediaType == "application/x-www-form-urlencoded": + masked, _, ok = g.redactForm(body, g.roots) + case strings.HasPrefix(mediaType, "multipart/"): + masked, _, ok = g.redactMultipart(body, params["boundary"], g.roots) + case mediaType == "" || mediaType == "application/json" || mediaType == "text/json" || strings.HasSuffix(mediaType, "+json"): + masked, _, ok = g.redactJSON(body, g.roots) + } + if !ok { + return []byte(Mask) + } + return masked +} + +func (g *graph) redactJSON(body []byte, roots []int) ([]byte, bool, bool) { dec := json.NewDecoder(bytes.NewReader(body)) dec.UseNumber() var value any if err := dec.Decode(&value); err != nil { - return []byte(Mask) + return nil, false, false } if _, err := dec.Token(); err != io.EOF { - return []byte(Mask) + return nil, false, false } - masked, changed := g.mask(value, g.roots, 0) + masked, changed := g.mask(value, roots, 0) if !changed { - return body + return body, false, true } var out bytes.Buffer enc := json.NewEncoder(&out) enc.SetEscapeHTML(false) if err := enc.Encode(masked); err != nil { - return []byte(Mask) + return nil, false, false + } + return bytes.TrimSuffix(out.Bytes(), []byte("\n")), true, true +} + +// redactForm masks the fields of a form body. A field holding a sensitive +// value or array of values is masked, and a JSON-encoded field is masked +// within. +func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { + pairs := strings.Split(string(body), "&") + changed := false + for i, pair := range pairs { + rawKey, rawValue, _ := strings.Cut(pair, "=") + key, err := url.QueryUnescape(rawKey) + if err != nil { + return nil, false, false + } + value, err := url.QueryUnescape(rawValue) + if err != nil { + return nil, false, false + } + masked, fieldChanged := g.redactValue([]byte(value), g.child(roots, strings.TrimSuffix(key, "[]"))) + if !fieldChanged { + continue + } + changed = true + if string(masked) == Mask { + pairs[i] = rawKey + "=" + Mask + } else { + pairs[i] = rawKey + "=" + url.QueryEscape(string(masked)) + } + } + if !changed { + return body, false, true + } + return []byte(strings.Join(pairs, "&")), true, true +} + +// redactMultipart masks the parts of a multipart body by part name, keeping +// the boundary and the part headers. +func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]byte, bool, bool) { + if boundary == "" { + return nil, false, false + } + reader := multipart.NewReader(bytes.NewReader(body), boundary) + var out bytes.Buffer + writer := multipart.NewWriter(&out) + if err := writer.SetBoundary(boundary); err != nil { + return nil, false, false + } + changed := false + for { + part, err := reader.NextRawPart() + if err == io.EOF { + break + } + if err != nil { + return nil, false, false + } + data, err := io.ReadAll(part) + if err != nil { + return nil, false, false + } + masked, partChanged := g.redactValue(data, g.child(roots, strings.TrimSuffix(part.FormName(), "[]"))) + changed = changed || partChanged + partWriter, err := writer.CreatePart(part.Header) + if err != nil { + return nil, false, false + } + if _, err := partWriter.Write(masked); err != nil { + return nil, false, false + } + } + if !changed { + return body, false, true } - return bytes.TrimSuffix(out.Bytes(), []byte("\n")) + if err := writer.Close(); err != nil { + return nil, false, false + } + return out.Bytes(), true, true +} + +// redactValue masks a form field or multipart part. A value holding a +// sensitive value, or one that cannot be parsed as JSON while holding one, is +// masked as a whole. +func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { + if len(g.expand(roots)) == 0 { + return value, false + } + if g.sensitive(roots) || g.sensitive(g.items(roots)) { + return []byte(Mask), true + } + masked, changed, ok := g.redactJSON(value, roots) + if !ok { + return []byte(Mask), true + } + return masked, changed } func (g *graph) mask(value any, roots []int, depth int) (any, bool) { @@ -109,10 +232,8 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { if len(roots) == 0 { return value, false } - for _, id := range roots { - if g.nodes[id].sensitive { - return Mask, true - } + if g.sensitive(roots) { + return Mask, true } if depth >= maxDepth { return Mask, true @@ -121,29 +242,13 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { switch v := value.(type) { case map[string]any: for key, child := range v { - var childRoots []int - for _, id := range roots { - n := g.nodes[id] - childID, declared := n.fields[key] - if !declared { - childID = n.values - } - if childID != 0 { - childRoots = append(childRoots, childID) - } - } - if masked, ok := g.mask(child, childRoots, depth+1); ok { + if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { v[key] = masked changed = true } } case []any: - var items []int - for _, id := range roots { - if item := g.nodes[id].item; item != 0 { - items = append(items, item) - } - } + items := g.items(roots) for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { v[i] = masked @@ -154,6 +259,44 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { return value, changed } +// sensitive reports whether any of roots, or their union variants, is +// sensitive as a whole. +func (g *graph) sensitive(roots []int) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].sensitive { + return true + } + } + return false +} + +// child returns the schemas of the property name of roots. +func (g *graph) child(roots []int, name string) []int { + var children []int + for _, id := range g.expand(roots) { + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) + } + } + return children +} + +// items returns the array item schemas of roots. +func (g *graph) items(roots []int) []int { + var items []int + for _, id := range g.expand(roots) { + if item := g.nodes[id].item; item != 0 { + items = append(items, item) + } + } + return items +} + // expand adds the union variants of roots, since the variant a value matches // is not known. func (g *graph) expand(roots []int) []int { diff --git a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock index 0327ab076..f0f7d57f1 100644 --- a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock +++ b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock @@ -2078,7 +2078,7 @@ trackedFiles: internal/provider/unsoundreadonlysingleop_resource_sdk.go: last_write_checksum: sha1:9ebe7ce0a238c2f58ec4fab7e587fc8d91f1b5bd internal/provider/utils.go: - last_write_checksum: sha1:e21456c199e9497aeb41d297f4cae588185a5bc8 + last_write_checksum: sha1:fe2b0224198eb372507274fbe4a5f9ca93087c5e internal/provider/utils_test.go: last_write_checksum: sha1:40da24e32eac29a8aac14e772e5fce44044f2ae0 internal/provider/xadditionalpropertiesname_data_source.go: @@ -2428,7 +2428,7 @@ trackedFiles: internal/sdk/models/errors/responsevalidationerror.go: last_write_checksum: sha1:aa7f69462e3a38d05371225adeb8c87bb433e6c4 internal/sdk/models/errors/sdkerror.go: - last_write_checksum: sha1:08765f9bb829f73021d99c4951c5d340cf1a7b07 + last_write_checksum: sha1:4e7181799f7433d1b24dd35116a9fe39510c0c9b internal/sdk/models/operations/closexentityoperationclosemultipleop1.go: last_write_checksum: sha1:7664dbc7a028e9f840a833fd5a24ade28c6bded3 internal/sdk/models/operations/closexentityoperationclosemultipleop2.go: @@ -3574,7 +3574,7 @@ trackedFiles: internal/sdk/polling/option.go: last_write_checksum: sha1:69d4557bf43a1a97b2d3735c442d2d25be53e56f internal/sdk/redact/redact.go: - last_write_checksum: sha1:ac466b2aeac1fbe0a7436e14cf2ab877794388a8 + last_write_checksum: sha1:e6d46123adf82b18afe8bf186c5acb56294b5197 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/terraform-provider-testing/internal/provider/utils.go b/zSDKs/terraform-provider-testing/internal/provider/utils.go index a31cf523e..b1d7c03c2 100644 --- a/zSDKs/terraform-provider-testing/internal/provider/utils.go +++ b/zSDKs/terraform-provider-testing/internal/provider/utils.go @@ -55,10 +55,12 @@ func debugResponse(response *http.Response) string { if response.Body != nil { if body, err := io.ReadAll(response.Body); err == nil { response.Body = io.NopCloser(bytes.NewReader(body)) - dumpRes = append(dumpRes, redact.ResponseBody(ctx, body)...) + dumpRes = append(dumpRes, redact.ResponseBody(ctx, response.Header.Get("Content-Type"), body)...) } } - dumpReq = redactDumpBody(dumpReq, func(body []byte) []byte { return redact.RequestBody(ctx, body) }) + dumpReq = redactDumpBody(dumpReq, func(body []byte) []byte { + return redact.RequestBody(ctx, response.Request.Header.Get("Content-Type"), body) + }) return redactSensitiveValues(ctx, fmt.Sprintf("**Request**:\n%s\n**Response**:\n%s", string(dumpReq), string(dumpRes))) } @@ -237,7 +239,7 @@ func decomposeRequestForLogging(req *http.Request) (map[string]interface{}, erro } // Read the rest of the body content - fields[FieldHttpRequestBody] = string(redact.RequestBody(req.Context(), []byte(bodyFromRestOfRequestReader(reqReader)))) + fields[FieldHttpRequestBody] = string(redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), []byte(bodyFromRestOfRequestReader(reqReader)))) redactSensitiveFields(req, fields) return fields, nil } @@ -324,7 +326,7 @@ func decomposeResponseForLogging(res *http.Response) (map[string]interface{}, er res.Body = io.NopCloser(bytes.NewBuffer(resBody)) if res.Request != nil { - resBody = redact.ResponseBody(res.Request.Context(), resBody) + resBody = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), resBody) } fields[FieldHttpResponseBody] = string(resBody) redactSensitiveFields(res.Request, fields) diff --git a/zSDKs/terraform-provider-testing/internal/sdk/models/errors/sdkerror.go b/zSDKs/terraform-provider-testing/internal/sdk/models/errors/sdkerror.go index 6ae2da4cb..0601b95a5 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/models/errors/sdkerror.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/models/errors/sdkerror.go @@ -32,7 +32,7 @@ func (e *SDKError) Error() string { if len(e.Body) > 0 { redacted := e.Body if e.RawResponse != nil && e.RawResponse.Request != nil { - redacted = string(redact.ResponseBody(e.RawResponse.Request.Context(), []byte(e.Body))) + redacted = string(redact.ResponseBody(e.RawResponse.Request.Context(), e.RawResponse.Header.Get("Content-Type"), []byte(e.Body))) } body = fmt.Sprintf("\n%s", redacted) } diff --git a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go index 5bc4ee54c..dea278899 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go @@ -9,8 +9,12 @@ // back from the request, or from a response's Request, to select the schema // graph used for masking: // -// body = redact.RequestBody(req.Context(), body) -// body = redact.ResponseBody(res.Request.Context(), body) +// body = redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), body) +// body = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), body) +// +// JSON, application/x-www-form-urlencoded and multipart bodies are masked +// value by value. Other bodies of an operation with sensitive values are +// masked as a whole. package redact import ( @@ -18,6 +22,10 @@ import ( "context" "encoding/json" "io" + "mime" + "mime/multipart" + "net/url" + "strings" ) // Mask replaces each sensitive value. @@ -42,16 +50,18 @@ func Operation(ctx context.Context) string { } // RequestBody masks the sensitive values of the request body of the -// operation carried by ctx. Bodies of other operations are returned as is. -func RequestBody(ctx context.Context, body []byte) []byte { - return operations[Operation(ctx)].request.redact(body) +// operation carried by ctx, serialized as contentType. Bodies of other +// operations are returned as is. +func RequestBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].request.redact(contentType, body) } // ResponseBody masks the sensitive values of a response body of the -// operation carried by ctx, matching every response schema of the operation. -// Bodies of other operations are returned as is. -func ResponseBody(ctx context.Context, body []byte) []byte { - return operations[Operation(ctx)].response.redact(body) +// operation carried by ctx, serialized as contentType, matching every +// response schema of the operation. Bodies of other operations are returned +// as is. +func ResponseBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].response.redact(contentType, body) } // A graph describes where sensitive values sit in a body. Node 0 is a @@ -72,36 +82,149 @@ type node struct { } // redact returns body unchanged when it has nothing to mask. A body that -// cannot be parsed as JSON is masked as a whole. -func (g *graph) redact(body []byte) []byte { +// cannot be parsed is masked as a whole. +func (g *graph) redact(contentType string, body []byte) []byte { if g == nil || len(body) == 0 { return body } - for _, id := range g.expand(g.roots) { - if g.nodes[id].sensitive { - return []byte(Mask) - } + if g.sensitive(g.roots) { + return []byte(Mask) + } + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil { + mediaType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) } + var masked []byte + ok := false + switch { + case mediaType == "application/x-www-form-urlencoded": + masked, _, ok = g.redactForm(body, g.roots) + case strings.HasPrefix(mediaType, "multipart/"): + masked, _, ok = g.redactMultipart(body, params["boundary"], g.roots) + case mediaType == "" || mediaType == "application/json" || mediaType == "text/json" || strings.HasSuffix(mediaType, "+json"): + masked, _, ok = g.redactJSON(body, g.roots) + } + if !ok { + return []byte(Mask) + } + return masked +} + +func (g *graph) redactJSON(body []byte, roots []int) ([]byte, bool, bool) { dec := json.NewDecoder(bytes.NewReader(body)) dec.UseNumber() var value any if err := dec.Decode(&value); err != nil { - return []byte(Mask) + return nil, false, false } if _, err := dec.Token(); err != io.EOF { - return []byte(Mask) + return nil, false, false } - masked, changed := g.mask(value, g.roots, 0) + masked, changed := g.mask(value, roots, 0) if !changed { - return body + return body, false, true } var out bytes.Buffer enc := json.NewEncoder(&out) enc.SetEscapeHTML(false) if err := enc.Encode(masked); err != nil { - return []byte(Mask) + return nil, false, false + } + return bytes.TrimSuffix(out.Bytes(), []byte("\n")), true, true +} + +// redactForm masks the fields of a form body. A field holding a sensitive +// value or array of values is masked, and a JSON-encoded field is masked +// within. +func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { + pairs := strings.Split(string(body), "&") + changed := false + for i, pair := range pairs { + rawKey, rawValue, _ := strings.Cut(pair, "=") + key, err := url.QueryUnescape(rawKey) + if err != nil { + return nil, false, false + } + value, err := url.QueryUnescape(rawValue) + if err != nil { + return nil, false, false + } + masked, fieldChanged := g.redactValue([]byte(value), g.child(roots, strings.TrimSuffix(key, "[]"))) + if !fieldChanged { + continue + } + changed = true + if string(masked) == Mask { + pairs[i] = rawKey + "=" + Mask + } else { + pairs[i] = rawKey + "=" + url.QueryEscape(string(masked)) + } + } + if !changed { + return body, false, true + } + return []byte(strings.Join(pairs, "&")), true, true +} + +// redactMultipart masks the parts of a multipart body by part name, keeping +// the boundary and the part headers. +func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]byte, bool, bool) { + if boundary == "" { + return nil, false, false + } + reader := multipart.NewReader(bytes.NewReader(body), boundary) + var out bytes.Buffer + writer := multipart.NewWriter(&out) + if err := writer.SetBoundary(boundary); err != nil { + return nil, false, false + } + changed := false + for { + part, err := reader.NextRawPart() + if err == io.EOF { + break + } + if err != nil { + return nil, false, false + } + data, err := io.ReadAll(part) + if err != nil { + return nil, false, false + } + masked, partChanged := g.redactValue(data, g.child(roots, strings.TrimSuffix(part.FormName(), "[]"))) + changed = changed || partChanged + partWriter, err := writer.CreatePart(part.Header) + if err != nil { + return nil, false, false + } + if _, err := partWriter.Write(masked); err != nil { + return nil, false, false + } + } + if !changed { + return body, false, true } - return bytes.TrimSuffix(out.Bytes(), []byte("\n")) + if err := writer.Close(); err != nil { + return nil, false, false + } + return out.Bytes(), true, true +} + +// redactValue masks a form field or multipart part. A value holding a +// sensitive value, or one that cannot be parsed as JSON while holding one, is +// masked as a whole. +func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { + if len(g.expand(roots)) == 0 { + return value, false + } + if g.sensitive(roots) || g.sensitive(g.items(roots)) { + return []byte(Mask), true + } + masked, changed, ok := g.redactJSON(value, roots) + if !ok { + return []byte(Mask), true + } + return masked, changed } func (g *graph) mask(value any, roots []int, depth int) (any, bool) { @@ -109,10 +232,8 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { if len(roots) == 0 { return value, false } - for _, id := range roots { - if g.nodes[id].sensitive { - return Mask, true - } + if g.sensitive(roots) { + return Mask, true } if depth >= maxDepth { return Mask, true @@ -121,29 +242,13 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { switch v := value.(type) { case map[string]any: for key, child := range v { - var childRoots []int - for _, id := range roots { - n := g.nodes[id] - childID, declared := n.fields[key] - if !declared { - childID = n.values - } - if childID != 0 { - childRoots = append(childRoots, childID) - } - } - if masked, ok := g.mask(child, childRoots, depth+1); ok { + if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { v[key] = masked changed = true } } case []any: - var items []int - for _, id := range roots { - if item := g.nodes[id].item; item != 0 { - items = append(items, item) - } - } + items := g.items(roots) for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { v[i] = masked @@ -154,6 +259,44 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { return value, changed } +// sensitive reports whether any of roots, or their union variants, is +// sensitive as a whole. +func (g *graph) sensitive(roots []int) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].sensitive { + return true + } + } + return false +} + +// child returns the schemas of the property name of roots. +func (g *graph) child(roots []int, name string) []int { + var children []int + for _, id := range g.expand(roots) { + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) + } + } + return children +} + +// items returns the array item schemas of roots. +func (g *graph) items(roots []int) []int { + var items []int + for _, id := range g.expand(roots) { + if item := g.nodes[id].item; item != 0 { + items = append(items, item) + } + } + return items +} + // expand adds the union variants of roots, since the variant a value matches // is not known. func (g *graph) expand(roots []int) []int { From 55c788d635ec2530b33e856ea269af5bebce5abc Mon Sep 17 00:00:00 2001 From: Thomas Rooney Date: Thu, 8 Oct 2026 18:45:35 +0100 Subject: [PATCH 04/23] fix(go): mask schema-mismatched values and only multipart/form-data parts A JSON value whose shape cannot hold the sensitive values of its schema is masked as a whole. Only multipart/form-data bodies are masked part by part, since other multipart subtypes have no part names, and masked parts drop their Content-Transfer-Encoding. --- internal/ast/sensitive_body.go | 16 +++++--- .../{{getRedactLocation}}/redact.go.stmpl | 40 +++++++++++++++++-- .../primary/redact_additional_test.go.stmpl | 11 +++++ zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/sdk/redact/redact.go | 40 +++++++++++++++++-- zSDKs/sdk-go/.speakeasy/gen.lock | 2 +- zSDKs/sdk-go/redact/redact.go | 40 +++++++++++++++++-- .../.speakeasy/gen.lock | 2 +- .../internal/sdk/redact/redact.go | 40 +++++++++++++++++-- 9 files changed, 168 insertions(+), 25 deletions(-) diff --git a/internal/ast/sensitive_body.go b/internal/ast/sensitive_body.go index 81a09f979..40ead34be 100644 --- a/internal/ast/sensitive_body.go +++ b/internal/ast/sensitive_body.go @@ -20,7 +20,8 @@ type SensitiveBodyNode struct { // Sensitive masks the whole value. Streams are masked as a whole when any // of their items holds a sensitive value. Sensitive bool - // Fields are object properties by JSON name, sorted by name. A property + // Fields are object properties by wire name (JSON property, form field + // or multipart part name), sorted by name. A property // that is not listed uses Values. When Values is set, declared properties // are listed even when they hold nothing sensitive (with Node 0), so that // they never fall back to Values. @@ -171,11 +172,7 @@ func (b *sensitiveBodyBuilder) addRoot(t *TypeDef) { node.values = walk(child.ItemType) } default: - name := field.OriginalName - if name == "" { - name = field.Name - } - node.fields = append(node.fields, SensitiveBodyField{Name: name}) + node.fields = append(node.fields, SensitiveBodyField{Name: sensitiveBodyFieldName(field)}) if field.Const == nil { node.fields[len(node.fields)-1].Node = walk(child) } @@ -186,6 +183,13 @@ func (b *sensitiveBodyBuilder) addRoot(t *TypeDef) { b.roots = append(b.roots, walk(t)) } +func sensitiveBodyFieldName(field *FieldDef) string { + if field.OriginalName != "" { + return field.OriginalName + } + return field.Name +} + // containsSensitiveBodyType reports whether t or any type below it is marked // sensitive, so that schemas without sensitive values are not walked further. func containsSensitiveBodyType(t *TypeDef) bool { diff --git a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl index 0ab74539b..d8b9678fd 100644 --- a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl +++ b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl @@ -10,8 +10,8 @@ // body = redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), body) // body = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), body) // -// JSON, application/x-www-form-urlencoded and multipart bodies are masked -// value by value. Other bodies of an operation with sensitive values are +// JSON, application/x-www-form-urlencoded and multipart/form-data bodies are +// masked value by value. Other bodies of an operation with sensitive values are // masked as a whole. package redact @@ -22,6 +22,7 @@ import ( "io" "mime" "mime/multipart" + "net/textproto" "net/url" "strings" ) @@ -97,7 +98,7 @@ func (g *graph) redact(contentType string, body []byte) []byte { switch { case mediaType == "application/x-www-form-urlencoded": masked, _, ok = g.redactForm(body, g.roots) - case strings.HasPrefix(mediaType, "multipart/"): + case mediaType == "multipart/form-data": masked, _, ok = g.redactMultipart(body, params["boundary"], g.roots) case mediaType == "" || mediaType == "application/json" || mediaType == "text/json" || strings.HasSuffix(mediaType, "+json"): masked, _, ok = g.redactJSON(body, g.roots) @@ -191,7 +192,16 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by } masked, partChanged := g.redactValue(data, g.child(roots, strings.TrimSuffix(part.FormName(), "[]"))) changed = changed || partChanged - partWriter, err := writer.CreatePart(part.Header) + header := part.Header + if partChanged { + // The mask is written as is, not in the part's transfer encoding. + header = make(textproto.MIMEHeader, len(part.Header)) + for key, values := range part.Header { + header[key] = values + } + header.Del("Content-Transfer-Encoding") + } + partWriter, err := writer.CreatePart(header) if err != nil { return nil, false, false } @@ -236,9 +246,15 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { if depth >= maxDepth { return Mask, true } + // A value that does not have the shape of its schema cannot be followed + // to its sensitive values, so it is masked as a whole. changed := false switch v := value.(type) { + case nil: case map[string]any: + if !g.hasProperties(roots) { + return Mask, true + } for key, child := range v { if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { v[key] = masked @@ -247,16 +263,32 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { } case []any: items := g.items(roots) + if len(items) == 0 { + return Mask, true + } for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { v[i] = masked changed = true } } + default: + return Mask, true } return value, changed } +// hasProperties reports whether any of roots describes object properties or +// map values. +func (g *graph) hasProperties(roots []int) bool { + for _, id := range g.expand(roots) { + if len(g.nodes[id].fields) > 0 || g.nodes[id].values != 0 { + return true + } + } + return false +} + // sensitive reports whether any of roots, or their union variants, is // sensitive as a whole. func (g *graph) sensitive(roots []int) bool { diff --git a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl index 9ac72fd29..b700a0d41 100644 --- a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl +++ b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl @@ -97,6 +97,10 @@ func TestRedactSensitiveBodies(t *testing.T) { assert.Equal(t, redact.Mask, string(redact.ResponseBody(ctx, "text/plain", []byte(`{"name": "visible-name"}`))), "bodies of other media types are masked as a whole") assert.Equal(t, []byte(sensitiveBodyPayload), redact.ResponseBody(context.Background(), "application/json", []byte(sensitiveBodyPayload)), "requests without an operation are not masked") + assert.Equal(t, `"`+redact.Mask+`"`, string(redact.ResponseBody(ctx, "application/json", []byte(`"secret-scalar"`))), "values not shaped like their schema are masked") + assert.Equal(t, `{"name":"visible-name","nested":"`+redact.Mask+`"}`, string(redact.ResponseBody(ctx, "application/json", []byte(`{"name": "visible-name", "nested": ["secret-shape"]}`))), "values not shaped like their schema are masked") + assert.Equal(t, `{"name":null}`, string(redact.ResponseBody(ctx, "application/json", []byte(`{"name":null}`))), "null values are kept") + errorBody := string(redact.ResponseBody(ctx, "application/json", []byte(`{"message": "visible-message", "secret": "secret-error"}`))) assert.Contains(t, errorBody, "visible-message") assert.NotContains(t, errorBody, "secret-error", "error response schemas are masked") @@ -193,6 +197,13 @@ func TestRedactMultipartRequestBody(t *testing.T) { ctx := redact.WithOperation(context.Background(), "sensitiveMultipartRedaction") assert.Equal(t, redact.Mask, string(redact.RequestBody(ctx, "multipart/form-data", client.body)), "multipart bodies without a boundary are masked as a whole") + assert.Equal(t, redact.Mask, string(redact.RequestBody(ctx, "multipart/mixed; boundary="+params["boundary"], client.body)), "other multipart bodies are masked as a whole") + + encoded := "--b\r\nContent-Disposition: form-data; name=\"token\"\r\nContent-Transfer-Encoding: base64\r\n\r\nc2VjcmV0LWVuY29kZWQ=\r\n--b--\r\n" + masked := string(redact.RequestBody(ctx, "multipart/form-data; boundary=b", []byte(encoded))) + assert.NotContains(t, masked, "c2VjcmV0LWVuY29kZWQ=") + assert.NotContains(t, masked, "Content-Transfer-Encoding", "masked parts are not transfer encoded") + assert.Contains(t, masked, redact.Mask) } func TestRedactSplitRequestBodies(t *testing.T) { diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index a778c5dad..60b1671ff 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -564,7 +564,7 @@ trackedFiles: internal/sdk/optionalnullable/optionalnullable_test.go: last_write_checksum: sha1:d3991667754ed67db803aeab4a6239f6f9a1be37 internal/sdk/redact/redact.go: - last_write_checksum: sha1:0ef73c0bcaba31b2cf207cca3fcb38c52066fd02 + last_write_checksum: sha1:f71a44d2c541eb4eedf31a1bb85101d80e304cc1 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/sdk-cli/internal/sdk/redact/redact.go b/zSDKs/sdk-cli/internal/sdk/redact/redact.go index f46c8700d..b998e1601 100644 --- a/zSDKs/sdk-cli/internal/sdk/redact/redact.go +++ b/zSDKs/sdk-cli/internal/sdk/redact/redact.go @@ -12,8 +12,8 @@ // body = redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), body) // body = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), body) // -// JSON, application/x-www-form-urlencoded and multipart bodies are masked -// value by value. Other bodies of an operation with sensitive values are +// JSON, application/x-www-form-urlencoded and multipart/form-data bodies are +// masked value by value. Other bodies of an operation with sensitive values are // masked as a whole. package redact @@ -24,6 +24,7 @@ import ( "io" "mime" "mime/multipart" + "net/textproto" "net/url" "strings" ) @@ -99,7 +100,7 @@ func (g *graph) redact(contentType string, body []byte) []byte { switch { case mediaType == "application/x-www-form-urlencoded": masked, _, ok = g.redactForm(body, g.roots) - case strings.HasPrefix(mediaType, "multipart/"): + case mediaType == "multipart/form-data": masked, _, ok = g.redactMultipart(body, params["boundary"], g.roots) case mediaType == "" || mediaType == "application/json" || mediaType == "text/json" || strings.HasSuffix(mediaType, "+json"): masked, _, ok = g.redactJSON(body, g.roots) @@ -193,7 +194,16 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by } masked, partChanged := g.redactValue(data, g.child(roots, strings.TrimSuffix(part.FormName(), "[]"))) changed = changed || partChanged - partWriter, err := writer.CreatePart(part.Header) + header := part.Header + if partChanged { + // The mask is written as is, not in the part's transfer encoding. + header = make(textproto.MIMEHeader, len(part.Header)) + for key, values := range part.Header { + header[key] = values + } + header.Del("Content-Transfer-Encoding") + } + partWriter, err := writer.CreatePart(header) if err != nil { return nil, false, false } @@ -238,9 +248,15 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { if depth >= maxDepth { return Mask, true } + // A value that does not have the shape of its schema cannot be followed + // to its sensitive values, so it is masked as a whole. changed := false switch v := value.(type) { + case nil: case map[string]any: + if !g.hasProperties(roots) { + return Mask, true + } for key, child := range v { if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { v[key] = masked @@ -249,16 +265,32 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { } case []any: items := g.items(roots) + if len(items) == 0 { + return Mask, true + } for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { v[i] = masked changed = true } } + default: + return Mask, true } return value, changed } +// hasProperties reports whether any of roots describes object properties or +// map values. +func (g *graph) hasProperties(roots []int) bool { + for _, id := range g.expand(roots) { + if len(g.nodes[id].fields) > 0 || g.nodes[id].values != 0 { + return true + } + } + return false +} + // sensitive reports whether any of roots, or their union variants, is // sensitive as a whole. func (g *graph) sensitive(roots []int) bool { diff --git a/zSDKs/sdk-go/.speakeasy/gen.lock b/zSDKs/sdk-go/.speakeasy/gen.lock index e2afa9f6f..254a20078 100644 --- a/zSDKs/sdk-go/.speakeasy/gen.lock +++ b/zSDKs/sdk-go/.speakeasy/gen.lock @@ -783,7 +783,7 @@ trackedFiles: recursiveformfield.go: last_write_checksum: sha1:2e06532b8c5445500bd2ae8ae8586d24e911a3e2 redact/redact.go: - last_write_checksum: sha1:0ef73c0bcaba31b2cf207cca3fcb38c52066fd02 + last_write_checksum: sha1:f71a44d2c541eb4eedf31a1bb85101d80e304cc1 renderassetop.go: last_write_checksum: sha1:a788e389967e88581337759b330af30c4e562bf8 requesttimeouterror.go: diff --git a/zSDKs/sdk-go/redact/redact.go b/zSDKs/sdk-go/redact/redact.go index f46c8700d..b998e1601 100644 --- a/zSDKs/sdk-go/redact/redact.go +++ b/zSDKs/sdk-go/redact/redact.go @@ -12,8 +12,8 @@ // body = redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), body) // body = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), body) // -// JSON, application/x-www-form-urlencoded and multipart bodies are masked -// value by value. Other bodies of an operation with sensitive values are +// JSON, application/x-www-form-urlencoded and multipart/form-data bodies are +// masked value by value. Other bodies of an operation with sensitive values are // masked as a whole. package redact @@ -24,6 +24,7 @@ import ( "io" "mime" "mime/multipart" + "net/textproto" "net/url" "strings" ) @@ -99,7 +100,7 @@ func (g *graph) redact(contentType string, body []byte) []byte { switch { case mediaType == "application/x-www-form-urlencoded": masked, _, ok = g.redactForm(body, g.roots) - case strings.HasPrefix(mediaType, "multipart/"): + case mediaType == "multipart/form-data": masked, _, ok = g.redactMultipart(body, params["boundary"], g.roots) case mediaType == "" || mediaType == "application/json" || mediaType == "text/json" || strings.HasSuffix(mediaType, "+json"): masked, _, ok = g.redactJSON(body, g.roots) @@ -193,7 +194,16 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by } masked, partChanged := g.redactValue(data, g.child(roots, strings.TrimSuffix(part.FormName(), "[]"))) changed = changed || partChanged - partWriter, err := writer.CreatePart(part.Header) + header := part.Header + if partChanged { + // The mask is written as is, not in the part's transfer encoding. + header = make(textproto.MIMEHeader, len(part.Header)) + for key, values := range part.Header { + header[key] = values + } + header.Del("Content-Transfer-Encoding") + } + partWriter, err := writer.CreatePart(header) if err != nil { return nil, false, false } @@ -238,9 +248,15 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { if depth >= maxDepth { return Mask, true } + // A value that does not have the shape of its schema cannot be followed + // to its sensitive values, so it is masked as a whole. changed := false switch v := value.(type) { + case nil: case map[string]any: + if !g.hasProperties(roots) { + return Mask, true + } for key, child := range v { if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { v[key] = masked @@ -249,16 +265,32 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { } case []any: items := g.items(roots) + if len(items) == 0 { + return Mask, true + } for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { v[i] = masked changed = true } } + default: + return Mask, true } return value, changed } +// hasProperties reports whether any of roots describes object properties or +// map values. +func (g *graph) hasProperties(roots []int) bool { + for _, id := range g.expand(roots) { + if len(g.nodes[id].fields) > 0 || g.nodes[id].values != 0 { + return true + } + } + return false +} + // sensitive reports whether any of roots, or their union variants, is // sensitive as a whole. func (g *graph) sensitive(roots []int) bool { diff --git a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock index f0f7d57f1..b348054b8 100644 --- a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock +++ b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock @@ -3574,7 +3574,7 @@ trackedFiles: internal/sdk/polling/option.go: last_write_checksum: sha1:69d4557bf43a1a97b2d3735c442d2d25be53e56f internal/sdk/redact/redact.go: - last_write_checksum: sha1:e6d46123adf82b18afe8bf186c5acb56294b5197 + last_write_checksum: sha1:2dfa00e72ee22ba791195dc20ff5ada11afaba31 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go index dea278899..0ce70f66a 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go @@ -12,8 +12,8 @@ // body = redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), body) // body = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), body) // -// JSON, application/x-www-form-urlencoded and multipart bodies are masked -// value by value. Other bodies of an operation with sensitive values are +// JSON, application/x-www-form-urlencoded and multipart/form-data bodies are +// masked value by value. Other bodies of an operation with sensitive values are // masked as a whole. package redact @@ -24,6 +24,7 @@ import ( "io" "mime" "mime/multipart" + "net/textproto" "net/url" "strings" ) @@ -99,7 +100,7 @@ func (g *graph) redact(contentType string, body []byte) []byte { switch { case mediaType == "application/x-www-form-urlencoded": masked, _, ok = g.redactForm(body, g.roots) - case strings.HasPrefix(mediaType, "multipart/"): + case mediaType == "multipart/form-data": masked, _, ok = g.redactMultipart(body, params["boundary"], g.roots) case mediaType == "" || mediaType == "application/json" || mediaType == "text/json" || strings.HasSuffix(mediaType, "+json"): masked, _, ok = g.redactJSON(body, g.roots) @@ -193,7 +194,16 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by } masked, partChanged := g.redactValue(data, g.child(roots, strings.TrimSuffix(part.FormName(), "[]"))) changed = changed || partChanged - partWriter, err := writer.CreatePart(part.Header) + header := part.Header + if partChanged { + // The mask is written as is, not in the part's transfer encoding. + header = make(textproto.MIMEHeader, len(part.Header)) + for key, values := range part.Header { + header[key] = values + } + header.Del("Content-Transfer-Encoding") + } + partWriter, err := writer.CreatePart(header) if err != nil { return nil, false, false } @@ -238,9 +248,15 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { if depth >= maxDepth { return Mask, true } + // A value that does not have the shape of its schema cannot be followed + // to its sensitive values, so it is masked as a whole. changed := false switch v := value.(type) { + case nil: case map[string]any: + if !g.hasProperties(roots) { + return Mask, true + } for key, child := range v { if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { v[key] = masked @@ -249,16 +265,32 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { } case []any: items := g.items(roots) + if len(items) == 0 { + return Mask, true + } for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { v[i] = masked changed = true } } + default: + return Mask, true } return value, changed } +// hasProperties reports whether any of roots describes object properties or +// map values. +func (g *graph) hasProperties(roots []int) bool { + for _, id := range g.expand(roots) { + if len(g.nodes[id].fields) > 0 || g.nodes[id].values != 0 { + return true + } + } + return false +} + // sensitive reports whether any of roots, or their union variants, is // sensitive as a whole. func (g *graph) sensitive(roots []int) bool { From 0144d41643d91062d1a906ea9d5d225e6f5cba6f Mon Sep 17 00:00:00 2001 From: 2ynn Date: Thu, 8 Oct 2026 14:16:46 -0400 Subject: [PATCH 05/23] fix(go): keep values matching plain union variants readable The sensitive body graph pruned union members that hold nothing sensitive, so a value matching such a member looked like it did not match its schema and was masked as a whole. Union nodes now record that a plain member exists, and the redactor leaves values of an unexpected shape readable when one does. --- internal/ast/sensitive_body.go | 5 +++ internal/ast/sensitive_body_test.go | 23 ++++++++++++++ .../templates/common/common/typeDefs.d.ts | 2 ++ .../{{getRedactLocation}}/redact.go.stmpl | 31 +++++++++++++++++-- templates/templates/go/includes/redact.ts | 1 + .../primary/redact_additional_test.go.stmpl | 14 +++++++++ .../primary/sensitive-body-redaction.yaml | 25 +++++++++++++++ zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/sdk/redact/redact.go | 31 +++++++++++++++++-- zSDKs/sdk-go/.speakeasy/gen.lock | 2 +- zSDKs/sdk-go/redact/redact.go | 31 +++++++++++++++++-- .../.speakeasy/gen.lock | 2 +- .../internal/sdk/redact/redact.go | 31 +++++++++++++++++-- 13 files changed, 185 insertions(+), 15 deletions(-) diff --git a/internal/ast/sensitive_body.go b/internal/ast/sensitive_body.go index 40ead34be..5094e0aa0 100644 --- a/internal/ast/sensitive_body.go +++ b/internal/ast/sensitive_body.go @@ -32,6 +32,9 @@ type SensitiveBodyNode struct { Values int // Variants are union members; the member a value matches is not known. Variants []int + // Plain reports a union member holding nothing sensitive, so a value + // that matches none of Variants may still match the schema. + Plain bool } // SensitiveBodyField is an object property of a SensitiveBodyNode. @@ -265,6 +268,8 @@ func (b *sensitiveBodyBuilder) build() *SensitiveBodyGraph { for _, variant := range node.variants { if remap[variant] != 0 { out.Variants = append(out.Variants, remap[variant]) + } else { + out.Plain = true } } } diff --git a/internal/ast/sensitive_body_test.go b/internal/ast/sensitive_body_test.go index 24ca9258c..8c66e499a 100644 --- a/internal/ast/sensitive_body_test.go +++ b/internal/ast/sensitive_body_test.go @@ -188,3 +188,26 @@ func TestSensitiveRequestBodyGraph(t *testing.T) { assert.Nil(t, (&Operation{}).SensitiveRequestBodyGraph()) } + +func TestSensitiveBodyGraphPlainUnionVariants(t *testing.T) { + t.Parallel() + + union := func(name string, variants ...*TypeDef) *TypeDef { + return &TypeDef{Type: DataTypeUnion, Name: name, AssociatedTypes: variants} + } + body := class("Body", + field("mixed", union("Mixed", &TypeDef{Type: DataTypeArray, ItemType: sensitiveString()}, class("Label", field("label", plainString())))), + field("sensitive", union("Sensitive", class("APIKey", field("key", sensitiveString())), sensitiveString())), + ) + g := responseOperation(body).SensitiveResponseBodyGraph() + require.NotNil(t, g) + + mixed := sensitiveChild(g, g.Roots, "mixed") + require.Len(t, mixed, 1) + assert.True(t, g.Nodes[mixed[0]].Plain, "a union with a member holding nothing sensitive is plain") + assert.Len(t, g.Nodes[mixed[0]].Variants, 1) + + sensitive := sensitiveChild(g, g.Roots, "sensitive") + require.Len(t, sensitive, 1) + assert.False(t, g.Nodes[sensitive[0]].Plain, "a union whose members all hold sensitive values is not plain") +} diff --git a/templates/templates/common/common/typeDefs.d.ts b/templates/templates/common/common/typeDefs.d.ts index 46b82efb0..0ea3ce5ce 100644 --- a/templates/templates/common/common/typeDefs.d.ts +++ b/templates/templates/common/common/typeDefs.d.ts @@ -2133,6 +2133,8 @@ declare global { Values: number; /** Union members. */ Variants?: number[]; + /** A union member holds nothing sensitive. */ + Plain: boolean; }; type OpenAPILocation = { diff --git a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl index d8b9678fd..846fa4920 100644 --- a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl +++ b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl @@ -78,6 +78,8 @@ type node struct { item int values int variants []int + // plain reports a union member holding nothing sensitive. + plain bool } // redact returns body unchanged when it has nothing to mask. A body that @@ -230,6 +232,9 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { } masked, changed, ok := g.redactJSON(value, roots) if !ok { + if g.plain(roots) { + return value, false + } return []byte(Mask), true } return masked, changed @@ -253,7 +258,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case nil: case map[string]any: if !g.hasProperties(roots) { - return Mask, true + return g.mismatch(value, roots) } for key, child := range v { if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { @@ -264,7 +269,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case []any: items := g.items(roots) if len(items) == 0 { - return Mask, true + return g.mismatch(value, roots) } for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { @@ -273,11 +278,31 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { } } default: - return Mask, true + return g.mismatch(value, roots) } return value, changed } +// mismatch masks a value that does not have the shape of roots, unless a +// union member holding nothing sensitive may match it. +func (g *graph) mismatch(value any, roots []int) (any, bool) { + if g.plain(roots) { + return value, false + } + return Mask, true +} + +// plain reports whether any of roots, or their union variants, is a union +// with a member holding nothing sensitive. +func (g *graph) plain(roots []int) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].plain { + return true + } + } + return false +} + // hasProperties reports whether any of roots describes object properties or // map values. func (g *graph) hasProperties(roots []int) bool { diff --git a/templates/templates/go/includes/redact.ts b/templates/templates/go/includes/redact.ts index 43ad84509..22f187f08 100644 --- a/templates/templates/go/includes/redact.ts +++ b/templates/templates/go/includes/redact.ts @@ -18,6 +18,7 @@ function renderRedactGraph(graph: SensitiveBodyGraph | null): string { if (node.Values) parts.push(`values: ${node.Values}`); if (node.Variants?.length) parts.push(`variants: []int{ ${node.Variants.join(", ")} }`); + if (node.Plain) parts.push("plain: true"); return `{ ${parts.join(", ")} }`; }); return `&graph{ nodes: []node{ ${nodes.join( diff --git a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl index b700a0d41..cc4ce7894 100644 --- a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl +++ b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl @@ -109,6 +109,20 @@ func TestRedactSensitiveBodies(t *testing.T) { assert.Equal(t, redact.Mask, string(redact.ResponseBody(streamCtx, "application/jsonl", []byte(`{"label": "visible", "secret": "secret-stream"}`))), "streams holding sensitive values are masked as a whole") } +func TestRedactUnionVariantsWithoutSensitiveValues(t *testing.T) { + ctx := redact.WithOperation(context.Background(), "sensitiveBodyRedaction") + redactJSON := func(body string) string { + return string(redact.RequestBody(ctx, "application/json", []byte(body))) + } + + assertMasked(t, redactJSON(`{"name": "visible-name", "tokensOrLabel": ["secret-token-item"]}`), []string{"secret-token-item"}, []string{"visible-name"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "tokensOrLabel": {"label": "visible-label"}}`), nil, []string{"visible-label"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrLabels": {"label": "visible-nested", "secret": "secret-nested"}}`), []string{"secret-nested"}, []string{"visible-nested"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrLabels": ["visible-item"]}`), nil, []string{"visible-item"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrText": {"label": "visible-nested", "secret": "secret-nested"}}`), []string{"secret-nested"}, []string{"visible-nested"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrText": "visible-text"}`), nil, []string{"visible-text"}) +} + func TestRedactOperationOnRequestContext(t *testing.T) { client := &redactCaptureClient{status: http.StatusOK} s := sdk.New(sdk.WithServerURL("http://localhost"), sdk.WithClient(client)) diff --git a/tests/specs/fragments/primary/sensitive-body-redaction.yaml b/tests/specs/fragments/primary/sensitive-body-redaction.yaml index 6af9f765c..33c1ee516 100644 --- a/tests/specs/fragments/primary/sensitive-body-redaction.yaml +++ b/tests/specs/fragments/primary/sensitive-body-redaction.yaml @@ -179,6 +179,31 @@ components: type: string unrelated: $ref: "#/components/schemas/SensitiveBodyUnrelated" + tokensOrLabel: + description: Union of an array of sensitive values and an object without sensitive values; an object value stays readable. + oneOf: + - type: array + items: + type: string + x-speakeasy-param-sensitive: true + - $ref: "#/components/schemas/SensitiveBodyPlainLabel" + secretOrLabels: + description: Union of an object with a sensitive value and an array without sensitive values; an array value stays readable. + oneOf: + - $ref: "#/components/schemas/SensitiveBodyNested" + - type: array + items: + type: string + secretOrText: + description: Union of an object with a sensitive value and a string without sensitive values; a string value stays readable. + oneOf: + - $ref: "#/components/schemas/SensitiveBodyNested" + - type: string + SensitiveBodyPlainLabel: + type: object + properties: + label: + type: string SensitiveBodyError: type: object properties: diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index 50b92e81a..06642a1c8 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -564,7 +564,7 @@ trackedFiles: internal/sdk/optionalnullable/optionalnullable_test.go: last_write_checksum: sha1:d3991667754ed67db803aeab4a6239f6f9a1be37 internal/sdk/redact/redact.go: - last_write_checksum: sha1:f71a44d2c541eb4eedf31a1bb85101d80e304cc1 + last_write_checksum: sha1:2cbc41e5258ecb10e70144ff613ac1eca8b9a3e8 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/sdk-cli/internal/sdk/redact/redact.go b/zSDKs/sdk-cli/internal/sdk/redact/redact.go index b998e1601..fbb3048b8 100644 --- a/zSDKs/sdk-cli/internal/sdk/redact/redact.go +++ b/zSDKs/sdk-cli/internal/sdk/redact/redact.go @@ -80,6 +80,8 @@ type node struct { item int values int variants []int + // plain reports a union member holding nothing sensitive. + plain bool } // redact returns body unchanged when it has nothing to mask. A body that @@ -232,6 +234,9 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { } masked, changed, ok := g.redactJSON(value, roots) if !ok { + if g.plain(roots) { + return value, false + } return []byte(Mask), true } return masked, changed @@ -255,7 +260,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case nil: case map[string]any: if !g.hasProperties(roots) { - return Mask, true + return g.mismatch(value, roots) } for key, child := range v { if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { @@ -266,7 +271,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case []any: items := g.items(roots) if len(items) == 0 { - return Mask, true + return g.mismatch(value, roots) } for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { @@ -275,11 +280,31 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { } } default: - return Mask, true + return g.mismatch(value, roots) } return value, changed } +// mismatch masks a value that does not have the shape of roots, unless a +// union member holding nothing sensitive may match it. +func (g *graph) mismatch(value any, roots []int) (any, bool) { + if g.plain(roots) { + return value, false + } + return Mask, true +} + +// plain reports whether any of roots, or their union variants, is a union +// with a member holding nothing sensitive. +func (g *graph) plain(roots []int) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].plain { + return true + } + } + return false +} + // hasProperties reports whether any of roots describes object properties or // map values. func (g *graph) hasProperties(roots []int) bool { diff --git a/zSDKs/sdk-go/.speakeasy/gen.lock b/zSDKs/sdk-go/.speakeasy/gen.lock index 254a20078..dc4caec55 100644 --- a/zSDKs/sdk-go/.speakeasy/gen.lock +++ b/zSDKs/sdk-go/.speakeasy/gen.lock @@ -783,7 +783,7 @@ trackedFiles: recursiveformfield.go: last_write_checksum: sha1:2e06532b8c5445500bd2ae8ae8586d24e911a3e2 redact/redact.go: - last_write_checksum: sha1:f71a44d2c541eb4eedf31a1bb85101d80e304cc1 + last_write_checksum: sha1:2cbc41e5258ecb10e70144ff613ac1eca8b9a3e8 renderassetop.go: last_write_checksum: sha1:a788e389967e88581337759b330af30c4e562bf8 requesttimeouterror.go: diff --git a/zSDKs/sdk-go/redact/redact.go b/zSDKs/sdk-go/redact/redact.go index b998e1601..fbb3048b8 100644 --- a/zSDKs/sdk-go/redact/redact.go +++ b/zSDKs/sdk-go/redact/redact.go @@ -80,6 +80,8 @@ type node struct { item int values int variants []int + // plain reports a union member holding nothing sensitive. + plain bool } // redact returns body unchanged when it has nothing to mask. A body that @@ -232,6 +234,9 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { } masked, changed, ok := g.redactJSON(value, roots) if !ok { + if g.plain(roots) { + return value, false + } return []byte(Mask), true } return masked, changed @@ -255,7 +260,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case nil: case map[string]any: if !g.hasProperties(roots) { - return Mask, true + return g.mismatch(value, roots) } for key, child := range v { if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { @@ -266,7 +271,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case []any: items := g.items(roots) if len(items) == 0 { - return Mask, true + return g.mismatch(value, roots) } for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { @@ -275,11 +280,31 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { } } default: - return Mask, true + return g.mismatch(value, roots) } return value, changed } +// mismatch masks a value that does not have the shape of roots, unless a +// union member holding nothing sensitive may match it. +func (g *graph) mismatch(value any, roots []int) (any, bool) { + if g.plain(roots) { + return value, false + } + return Mask, true +} + +// plain reports whether any of roots, or their union variants, is a union +// with a member holding nothing sensitive. +func (g *graph) plain(roots []int) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].plain { + return true + } + } + return false +} + // hasProperties reports whether any of roots describes object properties or // map values. func (g *graph) hasProperties(roots []int) bool { diff --git a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock index b348054b8..eace73c35 100644 --- a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock +++ b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock @@ -3574,7 +3574,7 @@ trackedFiles: internal/sdk/polling/option.go: last_write_checksum: sha1:69d4557bf43a1a97b2d3735c442d2d25be53e56f internal/sdk/redact/redact.go: - last_write_checksum: sha1:2dfa00e72ee22ba791195dc20ff5ada11afaba31 + last_write_checksum: sha1:a8790491c577ebbcfc6803028e07032d47e7b111 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go index 0ce70f66a..4345b4f66 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go @@ -80,6 +80,8 @@ type node struct { item int values int variants []int + // plain reports a union member holding nothing sensitive. + plain bool } // redact returns body unchanged when it has nothing to mask. A body that @@ -232,6 +234,9 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { } masked, changed, ok := g.redactJSON(value, roots) if !ok { + if g.plain(roots) { + return value, false + } return []byte(Mask), true } return masked, changed @@ -255,7 +260,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case nil: case map[string]any: if !g.hasProperties(roots) { - return Mask, true + return g.mismatch(value, roots) } for key, child := range v { if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { @@ -266,7 +271,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case []any: items := g.items(roots) if len(items) == 0 { - return Mask, true + return g.mismatch(value, roots) } for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { @@ -275,11 +280,31 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { } } default: - return Mask, true + return g.mismatch(value, roots) } return value, changed } +// mismatch masks a value that does not have the shape of roots, unless a +// union member holding nothing sensitive may match it. +func (g *graph) mismatch(value any, roots []int) (any, bool) { + if g.plain(roots) { + return value, false + } + return Mask, true +} + +// plain reports whether any of roots, or their union variants, is a union +// with a member holding nothing sensitive. +func (g *graph) plain(roots []int) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].plain { + return true + } + } + return false +} + // hasProperties reports whether any of roots describes object properties or // map values. func (g *graph) hasProperties(roots []int) bool { From 0247cd52d0cdcea68826bdb3acbd3b6a7fa8c429 Mon Sep 17 00:00:00 2001 From: 2ynn Date: Thu, 8 Oct 2026 14:28:12 -0400 Subject: [PATCH 06/23] fix(go): match plain union members by shape when redacting Union nodes now record the shapes (object, array, scalar) of members that hold nothing sensitive, instead of a single flag. A value that matches no sensitive member stays readable only when a plain member of the same shape exists. Form fields and multipart parts are parsed as JSON before the repeated array item rule applies, so a valid plain object beside a sensitive array stays readable. Text that is not JSON stays readable only when it is not a sensitive array item and a plain scalar member exists; malformed JSON is otherwise masked as before. --- internal/ast/sensitive_body.go | 39 +++++++++++++-- internal/ast/sensitive_body_test.go | 14 +++++- .../templates/common/common/typeDefs.d.ts | 6 ++- .../{{getRedactLocation}}/redact.go.stmpl | 50 ++++++++++++------- templates/templates/go/includes/redact.ts | 4 +- .../primary/redact_additional_test.go.stmpl | 23 +++++++++ zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/sdk/redact/redact.go | 50 ++++++++++++------- zSDKs/sdk-go/.speakeasy/gen.lock | 2 +- zSDKs/sdk-go/redact/redact.go | 50 ++++++++++++------- .../.speakeasy/gen.lock | 2 +- .../internal/sdk/redact/redact.go | 50 ++++++++++++------- 12 files changed, 203 insertions(+), 89 deletions(-) diff --git a/internal/ast/sensitive_body.go b/internal/ast/sensitive_body.go index 5094e0aa0..0ae546cf6 100644 --- a/internal/ast/sensitive_body.go +++ b/internal/ast/sensitive_body.go @@ -32,9 +32,12 @@ type SensitiveBodyNode struct { Values int // Variants are union members; the member a value matches is not known. Variants []int - // Plain reports a union member holding nothing sensitive, so a value - // that matches none of Variants may still match the schema. - Plain bool + // PlainObject, PlainArray and PlainScalar report union members of that + // shape holding nothing sensitive, so a value of that shape that matches + // none of Variants may still match the schema. + PlainObject bool + PlainArray bool + PlainScalar bool } // SensitiveBodyField is an object property of a SensitiveBodyNode. @@ -113,6 +116,9 @@ type sensitiveBodyBuildNode struct { item int values int variants []int + // variantTypes are the schemas of variants, so that pruned variants + // still describe the shapes the union accepts. + variantTypes []*TypeDef } func newSensitiveBodyBuilder() *sensitiveBodyBuilder { @@ -168,6 +174,7 @@ func (b *sensitiveBodyBuilder) addRoot(t *TypeDef) { node.item = walk(child) case DataTypeUnion: node.variants = append(node.variants, walk(child)) + node.variantTypes = append(node.variantTypes, child) } case t.Type != DataTypeClass && t.Type != DataTypeError: case field.IsAdditionalProperties: @@ -265,13 +272,35 @@ func (b *sensitiveBodyBuilder) build() *SensitiveBodyGraph { } sort.SliceStable(out.Fields, func(a, b int) bool { return out.Fields[a].Name < out.Fields[b].Name }) out.Item = remap[node.item] - for _, variant := range node.variants { + for v, variant := range node.variants { if remap[variant] != 0 { out.Variants = append(out.Variants, remap[variant]) } else { - out.Plain = true + out.addPlainShapes(node.variantTypes[v]) } } } return graph } + +// addPlainShapes records the shapes of values that t, a union member +// holding nothing sensitive, accepts. +func (n *SensitiveBodyNode) addPlainShapes(t *TypeDef) { + if t == nil { + return + } + switch t.Type { + case DataTypeClass, DataTypeError, DataTypeMap: + n.PlainObject = true + case DataTypeArray, DataTypeSet: + n.PlainArray = true + case DataTypeUnion: + for _, member := range t.AssociatedTypes { + n.addPlainShapes(member) + } + case DataTypeAny: + n.PlainObject, n.PlainArray, n.PlainScalar = true, true, true + default: + n.PlainScalar = true + } +} diff --git a/internal/ast/sensitive_body_test.go b/internal/ast/sensitive_body_test.go index 8c66e499a..e1f3dd7d7 100644 --- a/internal/ast/sensitive_body_test.go +++ b/internal/ast/sensitive_body_test.go @@ -197,6 +197,7 @@ func TestSensitiveBodyGraphPlainUnionVariants(t *testing.T) { } body := class("Body", field("mixed", union("Mixed", &TypeDef{Type: DataTypeArray, ItemType: sensitiveString()}, class("Label", field("label", plainString())))), + field("text", union("Text", class("APIKey", field("key", sensitiveString())), union("Scalars", plainString(), &TypeDef{Type: DataTypeInteger}))), field("sensitive", union("Sensitive", class("APIKey", field("key", sensitiveString())), sensitiveString())), ) g := responseOperation(body).SensitiveResponseBodyGraph() @@ -204,10 +205,19 @@ func TestSensitiveBodyGraphPlainUnionVariants(t *testing.T) { mixed := sensitiveChild(g, g.Roots, "mixed") require.Len(t, mixed, 1) - assert.True(t, g.Nodes[mixed[0]].Plain, "a union with a member holding nothing sensitive is plain") + assert.True(t, g.Nodes[mixed[0]].PlainObject, "a plain object member is recorded") + assert.False(t, g.Nodes[mixed[0]].PlainArray) + assert.False(t, g.Nodes[mixed[0]].PlainScalar) assert.Len(t, g.Nodes[mixed[0]].Variants, 1) + text := sensitiveChild(g, g.Roots, "text") + require.Len(t, text, 1) + assert.True(t, g.Nodes[text[0]].PlainScalar, "members of a plain nested union are recorded") + assert.False(t, g.Nodes[text[0]].PlainObject) + assert.False(t, g.Nodes[text[0]].PlainArray) + sensitive := sensitiveChild(g, g.Roots, "sensitive") require.Len(t, sensitive, 1) - assert.False(t, g.Nodes[sensitive[0]].Plain, "a union whose members all hold sensitive values is not plain") + node := g.Nodes[sensitive[0]] + assert.False(t, node.PlainObject || node.PlainArray || node.PlainScalar, "a union whose members all hold sensitive values has no plain member") } diff --git a/templates/templates/common/common/typeDefs.d.ts b/templates/templates/common/common/typeDefs.d.ts index 0ea3ce5ce..34580e931 100644 --- a/templates/templates/common/common/typeDefs.d.ts +++ b/templates/templates/common/common/typeDefs.d.ts @@ -2133,8 +2133,10 @@ declare global { Values: number; /** Union members. */ Variants?: number[]; - /** A union member holds nothing sensitive. */ - Plain: boolean; + /** Union members of each shape that hold nothing sensitive. */ + PlainObject: boolean; + PlainArray: boolean; + PlainScalar: boolean; }; type OpenAPILocation = { diff --git a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl index 846fa4920..cd3e7f5b7 100644 --- a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl +++ b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl @@ -78,10 +78,21 @@ type node struct { item int values int variants []int - // plain reports a union member holding nothing sensitive. - plain bool + // plainObject, plainArray and plainScalar report union members of that + // shape holding nothing sensitive. + plainObject bool + plainArray bool + plainScalar bool } +type shape int + +const ( + objectShape shape = iota + arrayShape + scalarShape +) + // redact returns body unchanged when it has nothing to mask. A body that // cannot be parsed is masked as a whole. func (g *graph) redact(contentType string, body []byte) []byte { @@ -227,17 +238,17 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if len(g.expand(roots)) == 0 { return value, false } - if g.sensitive(roots) || g.sensitive(g.items(roots)) { + if g.sensitive(roots) { return []byte(Mask), true } - masked, changed, ok := g.redactJSON(value, roots) - if !ok { - if g.plain(roots) { - return value, false - } - return []byte(Mask), true + if masked, changed, ok := g.redactJSON(value, roots); ok { + return masked, changed } - return masked, changed + // Text that is not JSON is a repeated array item or a plain string. + if !g.sensitive(g.items(roots)) && g.plain(roots, scalarShape) { + return value, false + } + return []byte(Mask), true } func (g *graph) mask(value any, roots []int, depth int) (any, bool) { @@ -258,7 +269,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case nil: case map[string]any: if !g.hasProperties(roots) { - return g.mismatch(value, roots) + return g.mismatch(value, roots, objectShape) } for key, child := range v { if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { @@ -269,7 +280,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case []any: items := g.items(roots) if len(items) == 0 { - return g.mismatch(value, roots) + return g.mismatch(value, roots, arrayShape) } for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { @@ -278,25 +289,26 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { } } default: - return g.mismatch(value, roots) + return g.mismatch(value, roots, scalarShape) } return value, changed } // mismatch masks a value that does not have the shape of roots, unless a -// union member holding nothing sensitive may match it. -func (g *graph) mismatch(value any, roots []int) (any, bool) { - if g.plain(roots) { +// union member of its shape holding nothing sensitive may match it. +func (g *graph) mismatch(value any, roots []int, s shape) (any, bool) { + if g.plain(roots, s) { return value, false } return Mask, true } // plain reports whether any of roots, or their union variants, is a union -// with a member holding nothing sensitive. -func (g *graph) plain(roots []int) bool { +// with a member of shape s holding nothing sensitive. +func (g *graph) plain(roots []int, s shape) bool { for _, id := range g.expand(roots) { - if g.nodes[id].plain { + n := g.nodes[id] + if s == objectShape && n.plainObject || s == arrayShape && n.plainArray || s == scalarShape && n.plainScalar { return true } } diff --git a/templates/templates/go/includes/redact.ts b/templates/templates/go/includes/redact.ts index 22f187f08..d2235aed3 100644 --- a/templates/templates/go/includes/redact.ts +++ b/templates/templates/go/includes/redact.ts @@ -18,7 +18,9 @@ function renderRedactGraph(graph: SensitiveBodyGraph | null): string { if (node.Values) parts.push(`values: ${node.Values}`); if (node.Variants?.length) parts.push(`variants: []int{ ${node.Variants.join(", ")} }`); - if (node.Plain) parts.push("plain: true"); + if (node.PlainObject) parts.push("plainObject: true"); + if (node.PlainArray) parts.push("plainArray: true"); + if (node.PlainScalar) parts.push("plainScalar: true"); return `{ ${parts.join(", ")} }`; }); return `&graph{ nodes: []node{ ${nodes.join( diff --git a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl index cc4ce7894..4748540df 100644 --- a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl +++ b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl @@ -121,6 +121,29 @@ func TestRedactUnionVariantsWithoutSensitiveValues(t *testing.T) { assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrLabels": ["visible-item"]}`), nil, []string{"visible-item"}) assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrText": {"label": "visible-nested", "secret": "secret-nested"}}`), []string{"secret-nested"}, []string{"visible-nested"}) assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrText": "visible-text"}`), nil, []string{"visible-text"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrText": ["secret-shape"]}`), []string{"secret-shape"}, []string{"visible-name"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrLabels": "secret-shape"}`), []string{"secret-shape"}, []string{"visible-name"}) + + redactForm := func(field, value string) string { + masked, err := url.ParseQuery(string(redact.RequestBody(ctx, "application/x-www-form-urlencoded", []byte("name=visible-name&"+field+"="+url.QueryEscape(value))))) + require.NoError(t, err) + return masked.Get(field) + } + redactPart := func(field, value string) string { + body := "--b\r\nContent-Disposition: form-data; name=\"" + field + "\"\r\n\r\n" + value + "\r\n--b--\r\n" + part, err := multipart.NewReader(bytes.NewReader(redact.RequestBody(ctx, "multipart/form-data; boundary=b", []byte(body))), "b").NextPart() + require.NoError(t, err) + data, err := io.ReadAll(part) + require.NoError(t, err) + return string(data) + } + for name, redactField := range map[string]func(field, value string) string{"form": redactForm, "multipart": redactPart} { + assert.JSONEq(t, `{"label": "visible-label"}`, redactField("tokensOrLabel", `{"label": "visible-label"}`), "%s: a plain object member stays readable", name) + assert.JSONEq(t, `["`+redact.Mask+`"]`, redactField("tokensOrLabel", `["secret-token-item"]`), "%s: sensitive array items are masked", name) + assert.Equal(t, redact.Mask, redactField("tokensOrLabel", "secret-token-item"), "%s: a repeated sensitive array item is masked", name) + assert.Equal(t, "visible-text", redactField("secretOrText", "visible-text"), "%s: a plain string member stays readable", name) + assert.Equal(t, redact.Mask, redactField("secretOrLabels", `{"secret": "secret-malformed",`), "%s: malformed JSON without a plain string member is masked", name) + } } func TestRedactOperationOnRequestContext(t *testing.T) { diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index 06642a1c8..b66e9f7c8 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -564,7 +564,7 @@ trackedFiles: internal/sdk/optionalnullable/optionalnullable_test.go: last_write_checksum: sha1:d3991667754ed67db803aeab4a6239f6f9a1be37 internal/sdk/redact/redact.go: - last_write_checksum: sha1:2cbc41e5258ecb10e70144ff613ac1eca8b9a3e8 + last_write_checksum: sha1:dbb9bc42c72d450fe72d76adb8da84653dc37d72 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/sdk-cli/internal/sdk/redact/redact.go b/zSDKs/sdk-cli/internal/sdk/redact/redact.go index fbb3048b8..a08de8824 100644 --- a/zSDKs/sdk-cli/internal/sdk/redact/redact.go +++ b/zSDKs/sdk-cli/internal/sdk/redact/redact.go @@ -80,10 +80,21 @@ type node struct { item int values int variants []int - // plain reports a union member holding nothing sensitive. - plain bool + // plainObject, plainArray and plainScalar report union members of that + // shape holding nothing sensitive. + plainObject bool + plainArray bool + plainScalar bool } +type shape int + +const ( + objectShape shape = iota + arrayShape + scalarShape +) + // redact returns body unchanged when it has nothing to mask. A body that // cannot be parsed is masked as a whole. func (g *graph) redact(contentType string, body []byte) []byte { @@ -229,17 +240,17 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if len(g.expand(roots)) == 0 { return value, false } - if g.sensitive(roots) || g.sensitive(g.items(roots)) { + if g.sensitive(roots) { return []byte(Mask), true } - masked, changed, ok := g.redactJSON(value, roots) - if !ok { - if g.plain(roots) { - return value, false - } - return []byte(Mask), true + if masked, changed, ok := g.redactJSON(value, roots); ok { + return masked, changed } - return masked, changed + // Text that is not JSON is a repeated array item or a plain string. + if !g.sensitive(g.items(roots)) && g.plain(roots, scalarShape) { + return value, false + } + return []byte(Mask), true } func (g *graph) mask(value any, roots []int, depth int) (any, bool) { @@ -260,7 +271,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case nil: case map[string]any: if !g.hasProperties(roots) { - return g.mismatch(value, roots) + return g.mismatch(value, roots, objectShape) } for key, child := range v { if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { @@ -271,7 +282,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case []any: items := g.items(roots) if len(items) == 0 { - return g.mismatch(value, roots) + return g.mismatch(value, roots, arrayShape) } for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { @@ -280,25 +291,26 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { } } default: - return g.mismatch(value, roots) + return g.mismatch(value, roots, scalarShape) } return value, changed } // mismatch masks a value that does not have the shape of roots, unless a -// union member holding nothing sensitive may match it. -func (g *graph) mismatch(value any, roots []int) (any, bool) { - if g.plain(roots) { +// union member of its shape holding nothing sensitive may match it. +func (g *graph) mismatch(value any, roots []int, s shape) (any, bool) { + if g.plain(roots, s) { return value, false } return Mask, true } // plain reports whether any of roots, or their union variants, is a union -// with a member holding nothing sensitive. -func (g *graph) plain(roots []int) bool { +// with a member of shape s holding nothing sensitive. +func (g *graph) plain(roots []int, s shape) bool { for _, id := range g.expand(roots) { - if g.nodes[id].plain { + n := g.nodes[id] + if s == objectShape && n.plainObject || s == arrayShape && n.plainArray || s == scalarShape && n.plainScalar { return true } } diff --git a/zSDKs/sdk-go/.speakeasy/gen.lock b/zSDKs/sdk-go/.speakeasy/gen.lock index dc4caec55..15ecbd1b4 100644 --- a/zSDKs/sdk-go/.speakeasy/gen.lock +++ b/zSDKs/sdk-go/.speakeasy/gen.lock @@ -783,7 +783,7 @@ trackedFiles: recursiveformfield.go: last_write_checksum: sha1:2e06532b8c5445500bd2ae8ae8586d24e911a3e2 redact/redact.go: - last_write_checksum: sha1:2cbc41e5258ecb10e70144ff613ac1eca8b9a3e8 + last_write_checksum: sha1:dbb9bc42c72d450fe72d76adb8da84653dc37d72 renderassetop.go: last_write_checksum: sha1:a788e389967e88581337759b330af30c4e562bf8 requesttimeouterror.go: diff --git a/zSDKs/sdk-go/redact/redact.go b/zSDKs/sdk-go/redact/redact.go index fbb3048b8..a08de8824 100644 --- a/zSDKs/sdk-go/redact/redact.go +++ b/zSDKs/sdk-go/redact/redact.go @@ -80,10 +80,21 @@ type node struct { item int values int variants []int - // plain reports a union member holding nothing sensitive. - plain bool + // plainObject, plainArray and plainScalar report union members of that + // shape holding nothing sensitive. + plainObject bool + plainArray bool + plainScalar bool } +type shape int + +const ( + objectShape shape = iota + arrayShape + scalarShape +) + // redact returns body unchanged when it has nothing to mask. A body that // cannot be parsed is masked as a whole. func (g *graph) redact(contentType string, body []byte) []byte { @@ -229,17 +240,17 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if len(g.expand(roots)) == 0 { return value, false } - if g.sensitive(roots) || g.sensitive(g.items(roots)) { + if g.sensitive(roots) { return []byte(Mask), true } - masked, changed, ok := g.redactJSON(value, roots) - if !ok { - if g.plain(roots) { - return value, false - } - return []byte(Mask), true + if masked, changed, ok := g.redactJSON(value, roots); ok { + return masked, changed } - return masked, changed + // Text that is not JSON is a repeated array item or a plain string. + if !g.sensitive(g.items(roots)) && g.plain(roots, scalarShape) { + return value, false + } + return []byte(Mask), true } func (g *graph) mask(value any, roots []int, depth int) (any, bool) { @@ -260,7 +271,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case nil: case map[string]any: if !g.hasProperties(roots) { - return g.mismatch(value, roots) + return g.mismatch(value, roots, objectShape) } for key, child := range v { if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { @@ -271,7 +282,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case []any: items := g.items(roots) if len(items) == 0 { - return g.mismatch(value, roots) + return g.mismatch(value, roots, arrayShape) } for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { @@ -280,25 +291,26 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { } } default: - return g.mismatch(value, roots) + return g.mismatch(value, roots, scalarShape) } return value, changed } // mismatch masks a value that does not have the shape of roots, unless a -// union member holding nothing sensitive may match it. -func (g *graph) mismatch(value any, roots []int) (any, bool) { - if g.plain(roots) { +// union member of its shape holding nothing sensitive may match it. +func (g *graph) mismatch(value any, roots []int, s shape) (any, bool) { + if g.plain(roots, s) { return value, false } return Mask, true } // plain reports whether any of roots, or their union variants, is a union -// with a member holding nothing sensitive. -func (g *graph) plain(roots []int) bool { +// with a member of shape s holding nothing sensitive. +func (g *graph) plain(roots []int, s shape) bool { for _, id := range g.expand(roots) { - if g.nodes[id].plain { + n := g.nodes[id] + if s == objectShape && n.plainObject || s == arrayShape && n.plainArray || s == scalarShape && n.plainScalar { return true } } diff --git a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock index eace73c35..e05e4dd70 100644 --- a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock +++ b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock @@ -3574,7 +3574,7 @@ trackedFiles: internal/sdk/polling/option.go: last_write_checksum: sha1:69d4557bf43a1a97b2d3735c442d2d25be53e56f internal/sdk/redact/redact.go: - last_write_checksum: sha1:a8790491c577ebbcfc6803028e07032d47e7b111 + last_write_checksum: sha1:2151c354ec3cfdadceeca95d8e73e99b778ccf9e internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go index 4345b4f66..e2326e429 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go @@ -80,10 +80,21 @@ type node struct { item int values int variants []int - // plain reports a union member holding nothing sensitive. - plain bool + // plainObject, plainArray and plainScalar report union members of that + // shape holding nothing sensitive. + plainObject bool + plainArray bool + plainScalar bool } +type shape int + +const ( + objectShape shape = iota + arrayShape + scalarShape +) + // redact returns body unchanged when it has nothing to mask. A body that // cannot be parsed is masked as a whole. func (g *graph) redact(contentType string, body []byte) []byte { @@ -229,17 +240,17 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if len(g.expand(roots)) == 0 { return value, false } - if g.sensitive(roots) || g.sensitive(g.items(roots)) { + if g.sensitive(roots) { return []byte(Mask), true } - masked, changed, ok := g.redactJSON(value, roots) - if !ok { - if g.plain(roots) { - return value, false - } - return []byte(Mask), true + if masked, changed, ok := g.redactJSON(value, roots); ok { + return masked, changed } - return masked, changed + // Text that is not JSON is a repeated array item or a plain string. + if !g.sensitive(g.items(roots)) && g.plain(roots, scalarShape) { + return value, false + } + return []byte(Mask), true } func (g *graph) mask(value any, roots []int, depth int) (any, bool) { @@ -260,7 +271,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case nil: case map[string]any: if !g.hasProperties(roots) { - return g.mismatch(value, roots) + return g.mismatch(value, roots, objectShape) } for key, child := range v { if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { @@ -271,7 +282,7 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { case []any: items := g.items(roots) if len(items) == 0 { - return g.mismatch(value, roots) + return g.mismatch(value, roots, arrayShape) } for i, child := range v { if masked, ok := g.mask(child, items, depth+1); ok { @@ -280,25 +291,26 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { } } default: - return g.mismatch(value, roots) + return g.mismatch(value, roots, scalarShape) } return value, changed } // mismatch masks a value that does not have the shape of roots, unless a -// union member holding nothing sensitive may match it. -func (g *graph) mismatch(value any, roots []int) (any, bool) { - if g.plain(roots) { +// union member of its shape holding nothing sensitive may match it. +func (g *graph) mismatch(value any, roots []int, s shape) (any, bool) { + if g.plain(roots, s) { return value, false } return Mask, true } // plain reports whether any of roots, or their union variants, is a union -// with a member holding nothing sensitive. -func (g *graph) plain(roots []int) bool { +// with a member of shape s holding nothing sensitive. +func (g *graph) plain(roots []int, s shape) bool { for _, id := range g.expand(roots) { - if g.nodes[id].plain { + n := g.nodes[id] + if s == objectShape && n.plainObject || s == arrayShape && n.plainArray || s == scalarShape && n.plainScalar { return true } } From 69a0749aa8094b0d6081baf5a8b2285c57523da1 Mon Sep 17 00:00:00 2001 From: 2ynn Date: Thu, 8 Oct 2026 14:43:48 -0400 Subject: [PATCH 07/23] fix(go): accept only text a plain string member can hold when redacting Plain union members are now recorded by JSON type (object, array, string, number, boolean), with enums following their underlying type. A form field or multipart part that is not valid JSON stays readable only beside a plain string member, of the field or of its items when sent as repeated fields; a plain integer or boolean member no longer lets malformed JSON through. Text starting like a JSON object or array is never accepted as a plain string. JSON scalars must match a plain member of their own type. Nested plain unions are followed once each, so recursive unions terminate. Masked multipart parts also drop Content-Length and Content-Encoding, which no longer describe the masked payload. --- internal/ast/sensitive_body.go | 47 ++++++++----- internal/ast/sensitive_body_test.go | 23 +++++-- .../templates/common/common/typeDefs.d.ts | 6 +- .../{{getRedactLocation}}/redact.go.stmpl | 66 +++++++++++++++---- templates/templates/go/includes/redact.ts | 4 +- .../primary/redact_additional_test.go.stmpl | 16 ++++- .../primary/sensitive-body-redaction.yaml | 17 +++++ zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/sdk/redact/redact.go | 66 +++++++++++++++---- zSDKs/sdk-go/.speakeasy/gen.lock | 2 +- zSDKs/sdk-go/redact/redact.go | 66 +++++++++++++++---- .../.speakeasy/gen.lock | 2 +- .../internal/sdk/redact/redact.go | 66 +++++++++++++++---- 13 files changed, 299 insertions(+), 84 deletions(-) diff --git a/internal/ast/sensitive_body.go b/internal/ast/sensitive_body.go index 0ae546cf6..40bc8286c 100644 --- a/internal/ast/sensitive_body.go +++ b/internal/ast/sensitive_body.go @@ -32,12 +32,15 @@ type SensitiveBodyNode struct { Values int // Variants are union members; the member a value matches is not known. Variants []int - // PlainObject, PlainArray and PlainScalar report union members of that - // shape holding nothing sensitive, so a value of that shape that matches - // none of Variants may still match the schema. - PlainObject bool - PlainArray bool - PlainScalar bool + // PlainObject, PlainArray, PlainString, PlainNumber and PlainBoolean + // report union members accepting JSON values of that type and holding + // nothing sensitive, so a value of that type that matches none of + // Variants may still match the schema. + PlainObject bool + PlainArray bool + PlainString bool + PlainNumber bool + PlainBoolean bool } // SensitiveBodyField is an object property of a SensitiveBodyNode. @@ -276,19 +279,21 @@ func (b *sensitiveBodyBuilder) build() *SensitiveBodyGraph { if remap[variant] != 0 { out.Variants = append(out.Variants, remap[variant]) } else { - out.addPlainShapes(node.variantTypes[v]) + out.addPlainShapes(node.variantTypes[v], map[*TypeDef]bool{}) } } } return graph } -// addPlainShapes records the shapes of values that t, a union member -// holding nothing sensitive, accepts. -func (n *SensitiveBodyNode) addPlainShapes(t *TypeDef) { - if t == nil { +// addPlainShapes records the JSON types of values that t, a union member +// holding nothing sensitive, accepts. Nested unions are followed once each, +// so that recursive unions terminate. +func (n *SensitiveBodyNode) addPlainShapes(t *TypeDef, visited map[*TypeDef]bool) { + if t == nil || visited[t] { return } + visited[t] = true switch t.Type { case DataTypeClass, DataTypeError, DataTypeMap: n.PlainObject = true @@ -296,11 +301,23 @@ func (n *SensitiveBodyNode) addPlainShapes(t *TypeDef) { n.PlainArray = true case DataTypeUnion: for _, member := range t.AssociatedTypes { - n.addPlainShapes(member) + n.addPlainShapes(member, visited) + } + case DataTypeEnum: + if t.Enum != nil && t.Enum.Type != nil && t.Enum.Type.Type != DataTypeEnum { + n.addPlainShapes(t.Enum.Type, visited) + } else { + n.PlainString = true } case DataTypeAny: - n.PlainObject, n.PlainArray, n.PlainScalar = true, true, true - default: - n.PlainScalar = true + n.PlainObject, n.PlainArray, n.PlainString, n.PlainNumber, n.PlainBoolean = true, true, true, true, true + case DataTypeBoolean: + n.PlainBoolean = true + case DataTypeInteger, DataTypeNumber: + n.PlainNumber = true + case DataTypeBigInt, DataTypeDecimal: + n.PlainNumber, n.PlainString = true, true + case DataTypeString, DataTypeDate, DataTypeDateTime, DataTypeUUID, DataTypeDuration, DataTypeBytes: + n.PlainString = true } } diff --git a/internal/ast/sensitive_body_test.go b/internal/ast/sensitive_body_test.go index e1f3dd7d7..7ef6e5bf9 100644 --- a/internal/ast/sensitive_body_test.go +++ b/internal/ast/sensitive_body_test.go @@ -195,9 +195,13 @@ func TestSensitiveBodyGraphPlainUnionVariants(t *testing.T) { union := func(name string, variants ...*TypeDef) *TypeDef { return &TypeDef{Type: DataTypeUnion, Name: name, AssociatedTypes: variants} } + recursivePlain := union("RecursivePlain", plainString()) + recursivePlain.AssociatedTypes = append(recursivePlain.AssociatedTypes, recursivePlain) body := class("Body", field("mixed", union("Mixed", &TypeDef{Type: DataTypeArray, ItemType: sensitiveString()}, class("Label", field("label", plainString())))), field("text", union("Text", class("APIKey", field("key", sensitiveString())), union("Scalars", plainString(), &TypeDef{Type: DataTypeInteger}))), + field("flag", union("Flag", class("APIKey", field("key", sensitiveString())), &TypeDef{Type: DataTypeBoolean})), + field("loop", union("Loop", class("APIKey", field("key", sensitiveString())), recursivePlain)), field("sensitive", union("Sensitive", class("APIKey", field("key", sensitiveString())), sensitiveString())), ) g := responseOperation(body).SensitiveResponseBodyGraph() @@ -206,18 +210,25 @@ func TestSensitiveBodyGraphPlainUnionVariants(t *testing.T) { mixed := sensitiveChild(g, g.Roots, "mixed") require.Len(t, mixed, 1) assert.True(t, g.Nodes[mixed[0]].PlainObject, "a plain object member is recorded") - assert.False(t, g.Nodes[mixed[0]].PlainArray) - assert.False(t, g.Nodes[mixed[0]].PlainScalar) + assert.False(t, g.Nodes[mixed[0]].PlainArray || g.Nodes[mixed[0]].PlainString || g.Nodes[mixed[0]].PlainNumber || g.Nodes[mixed[0]].PlainBoolean) assert.Len(t, g.Nodes[mixed[0]].Variants, 1) text := sensitiveChild(g, g.Roots, "text") require.Len(t, text, 1) - assert.True(t, g.Nodes[text[0]].PlainScalar, "members of a plain nested union are recorded") - assert.False(t, g.Nodes[text[0]].PlainObject) - assert.False(t, g.Nodes[text[0]].PlainArray) + assert.True(t, g.Nodes[text[0]].PlainString && g.Nodes[text[0]].PlainNumber, "members of a plain nested union are recorded") + assert.False(t, g.Nodes[text[0]].PlainObject || g.Nodes[text[0]].PlainArray || g.Nodes[text[0]].PlainBoolean) + + flag := sensitiveChild(g, g.Roots, "flag") + require.Len(t, flag, 1) + assert.True(t, g.Nodes[flag[0]].PlainBoolean) + assert.False(t, g.Nodes[flag[0]].PlainString || g.Nodes[flag[0]].PlainNumber, "a boolean member does not accept text or numbers") + + loop := sensitiveChild(g, g.Roots, "loop") + require.Len(t, loop, 1) + assert.True(t, g.Nodes[loop[0]].PlainString, "a recursive plain union is followed once") sensitive := sensitiveChild(g, g.Roots, "sensitive") require.Len(t, sensitive, 1) node := g.Nodes[sensitive[0]] - assert.False(t, node.PlainObject || node.PlainArray || node.PlainScalar, "a union whose members all hold sensitive values has no plain member") + assert.False(t, node.PlainObject || node.PlainArray || node.PlainString || node.PlainNumber || node.PlainBoolean, "a union whose members all hold sensitive values has no plain member") } diff --git a/templates/templates/common/common/typeDefs.d.ts b/templates/templates/common/common/typeDefs.d.ts index 34580e931..c865854c3 100644 --- a/templates/templates/common/common/typeDefs.d.ts +++ b/templates/templates/common/common/typeDefs.d.ts @@ -2133,10 +2133,12 @@ declare global { Values: number; /** Union members. */ Variants?: number[]; - /** Union members of each shape that hold nothing sensitive. */ + /** Union members of each JSON type that hold nothing sensitive. */ PlainObject: boolean; PlainArray: boolean; - PlainScalar: boolean; + PlainString: boolean; + PlainNumber: boolean; + PlainBoolean: boolean; }; type OpenAPILocation = { diff --git a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl index cd3e7f5b7..b1c8eec0b 100644 --- a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl +++ b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl @@ -78,11 +78,14 @@ type node struct { item int values int variants []int - // plainObject, plainArray and plainScalar report union members of that - // shape holding nothing sensitive. - plainObject bool - plainArray bool - plainScalar bool + // plainObject, plainArray, plainString, plainNumber and plainBoolean + // report union members accepting JSON values of that type and holding + // nothing sensitive. + plainObject bool + plainArray bool + plainString bool + plainNumber bool + plainBoolean bool } type shape int @@ -90,7 +93,9 @@ type shape int const ( objectShape shape = iota arrayShape - scalarShape + stringShape + numberShape + booleanShape ) // redact returns body unchanged when it has nothing to mask. A body that @@ -207,12 +212,15 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by changed = changed || partChanged header := part.Header if partChanged { - // The mask is written as is, not in the part's transfer encoding. + // The mask is written as is, not in the part's transfer encoding, + // and with its own length. header = make(textproto.MIMEHeader, len(part.Header)) for key, values := range part.Header { header[key] = values } header.Del("Content-Transfer-Encoding") + header.Del("Content-Encoding") + header.Del("Content-Length") } partWriter, err := writer.CreatePart(header) if err != nil { @@ -241,16 +249,25 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if g.sensitive(roots) { return []byte(Mask), true } + // Unencoded text matches a plain string member of the field or of its + // items when sent as repeated fields, unless it is a sensitive repeated + // item or starts like a JSON object or array, which may carry sensitive + // values. + items := g.items(roots) + if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !g.sensitive(items) && !isJSONStructure(value) { + return value, false + } if masked, changed, ok := g.redactJSON(value, roots); ok { return masked, changed } - // Text that is not JSON is a repeated array item or a plain string. - if !g.sensitive(g.items(roots)) && g.plain(roots, scalarShape) { - return value, false - } return []byte(Mask), true } +func isJSONStructure(value []byte) bool { + trimmed := bytes.TrimLeft(value, " \t\r\n") + return len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '[') +} + func (g *graph) mask(value any, roots []int, depth int) (any, bool) { roots = g.expand(roots) if len(roots) == 0 { @@ -288,8 +305,14 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { changed = true } } + case string: + return g.mismatch(value, roots, stringShape) + case json.Number, float64: + return g.mismatch(value, roots, numberShape) + case bool: + return g.mismatch(value, roots, booleanShape) default: - return g.mismatch(value, roots, scalarShape) + return Mask, true } return value, changed } @@ -307,14 +330,29 @@ func (g *graph) mismatch(value any, roots []int, s shape) (any, bool) { // with a member of shape s holding nothing sensitive. func (g *graph) plain(roots []int, s shape) bool { for _, id := range g.expand(roots) { - n := g.nodes[id] - if s == objectShape && n.plainObject || s == arrayShape && n.plainArray || s == scalarShape && n.plainScalar { + if g.nodes[id].plainShape(s) { return true } } return false } +func (n node) plainShape(s shape) bool { + switch s { + case objectShape: + return n.plainObject + case arrayShape: + return n.plainArray + case stringShape: + return n.plainString + case numberShape: + return n.plainNumber + case booleanShape: + return n.plainBoolean + } + return false +} + // hasProperties reports whether any of roots describes object properties or // map values. func (g *graph) hasProperties(roots []int) bool { diff --git a/templates/templates/go/includes/redact.ts b/templates/templates/go/includes/redact.ts index d2235aed3..9fcad2124 100644 --- a/templates/templates/go/includes/redact.ts +++ b/templates/templates/go/includes/redact.ts @@ -20,7 +20,9 @@ function renderRedactGraph(graph: SensitiveBodyGraph | null): string { parts.push(`variants: []int{ ${node.Variants.join(", ")} }`); if (node.PlainObject) parts.push("plainObject: true"); if (node.PlainArray) parts.push("plainArray: true"); - if (node.PlainScalar) parts.push("plainScalar: true"); + if (node.PlainString) parts.push("plainString: true"); + if (node.PlainNumber) parts.push("plainNumber: true"); + if (node.PlainBoolean) parts.push("plainBoolean: true"); return `{ ${parts.join(", ")} }`; }); return `&graph{ nodes: []node{ ${nodes.join( diff --git a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl index 4748540df..2a3aa1460 100644 --- a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl +++ b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl @@ -123,6 +123,9 @@ func TestRedactUnionVariantsWithoutSensitiveValues(t *testing.T) { assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrText": "visible-text"}`), nil, []string{"visible-text"}) assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrText": ["secret-shape"]}`), []string{"secret-shape"}, []string{"visible-name"}) assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrLabels": "secret-shape"}`), []string{"secret-shape"}, []string{"visible-name"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrCount": 4242, "secretOrFlag": true}`), nil, []string{"4242", "true"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrCount": "secret-count", "secretOrFlag": "secret-flag"}`), []string{"secret-count", "secret-flag"}, []string{"visible-name"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrText": 4242}`), []string{"4242"}, []string{"visible-name"}) redactForm := func(field, value string) string { masked, err := url.ParseQuery(string(redact.RequestBody(ctx, "application/x-www-form-urlencoded", []byte("name=visible-name&"+field+"="+url.QueryEscape(value))))) @@ -143,6 +146,16 @@ func TestRedactUnionVariantsWithoutSensitiveValues(t *testing.T) { assert.Equal(t, redact.Mask, redactField("tokensOrLabel", "secret-token-item"), "%s: a repeated sensitive array item is masked", name) assert.Equal(t, "visible-text", redactField("secretOrText", "visible-text"), "%s: a plain string member stays readable", name) assert.Equal(t, redact.Mask, redactField("secretOrLabels", `{"secret": "secret-malformed",`), "%s: malformed JSON without a plain string member is masked", name) + assert.Equal(t, "4242", redactField("secretOrText", "4242"), "%s: numeric text of a plain string member stays readable", name) + assert.Equal(t, redact.Mask, redactField("secretOrText", `{"secret": "secret-malformed",`), "%s: malformed JSON is masked beside a plain string member", name) + for _, field := range []string{"secretOrCount", "secretOrFlag"} { + assert.Equal(t, redact.Mask, redactField(field, `{"secret": "secret-malformed",`), "%s: malformed JSON in %s is masked", name, field) + assert.Equal(t, redact.Mask, redactField(field, "secret-text"), "%s: text in %s is masked", name, field) + } + assert.Equal(t, "4242", redactField("secretOrCount", "4242"), "%s: a plain integer member stays readable", name) + assert.Equal(t, "true", redactField("secretOrFlag", "true"), "%s: a plain boolean member stays readable", name) + assert.Equal(t, "visible-item-name", redactField("secretsOrNames", "visible-item-name"), "%s: a repeated item matching a plain string member stays readable", name) + assert.NotContains(t, redactField("secretsOrNames", `{"label": "visible-label", "secret": "secret-item"}`), "secret-item", "%s: a repeated object item is masked", name) } } @@ -236,10 +249,11 @@ func TestRedactMultipartRequestBody(t *testing.T) { assert.Equal(t, redact.Mask, string(redact.RequestBody(ctx, "multipart/form-data", client.body)), "multipart bodies without a boundary are masked as a whole") assert.Equal(t, redact.Mask, string(redact.RequestBody(ctx, "multipart/mixed; boundary="+params["boundary"], client.body)), "other multipart bodies are masked as a whole") - encoded := "--b\r\nContent-Disposition: form-data; name=\"token\"\r\nContent-Transfer-Encoding: base64\r\n\r\nc2VjcmV0LWVuY29kZWQ=\r\n--b--\r\n" + encoded := "--b\r\nContent-Disposition: form-data; name=\"token\"\r\nContent-Transfer-Encoding: base64\r\nContent-Length: 20\r\n\r\nc2VjcmV0LWVuY29kZWQ=\r\n--b--\r\n" masked := string(redact.RequestBody(ctx, "multipart/form-data; boundary=b", []byte(encoded))) assert.NotContains(t, masked, "c2VjcmV0LWVuY29kZWQ=") assert.NotContains(t, masked, "Content-Transfer-Encoding", "masked parts are not transfer encoded") + assert.NotContains(t, masked, "Content-Length", "masked parts do not keep the original length") assert.Contains(t, masked, redact.Mask) } diff --git a/tests/specs/fragments/primary/sensitive-body-redaction.yaml b/tests/specs/fragments/primary/sensitive-body-redaction.yaml index 33c1ee516..3ced4d095 100644 --- a/tests/specs/fragments/primary/sensitive-body-redaction.yaml +++ b/tests/specs/fragments/primary/sensitive-body-redaction.yaml @@ -199,6 +199,23 @@ components: oneOf: - $ref: "#/components/schemas/SensitiveBodyNested" - type: string + secretOrCount: + description: Union of an object with a sensitive value and an integer; an integer value stays readable, text does not. + oneOf: + - $ref: "#/components/schemas/SensitiveBodyNested" + - type: integer + secretOrFlag: + description: Union of an object with a sensitive value and a boolean; a boolean value stays readable, text does not. + oneOf: + - $ref: "#/components/schemas/SensitiveBodyNested" + - type: boolean + secretsOrNames: + description: Array of a union of an object with a sensitive value and a string; string items sent as repeated fields stay readable. + type: array + items: + oneOf: + - $ref: "#/components/schemas/SensitiveBodyNested" + - type: string SensitiveBodyPlainLabel: type: object properties: diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index b66e9f7c8..420cfd074 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -564,7 +564,7 @@ trackedFiles: internal/sdk/optionalnullable/optionalnullable_test.go: last_write_checksum: sha1:d3991667754ed67db803aeab4a6239f6f9a1be37 internal/sdk/redact/redact.go: - last_write_checksum: sha1:dbb9bc42c72d450fe72d76adb8da84653dc37d72 + last_write_checksum: sha1:e5c8b1373dc7b811b249e29ed231a47636b83b73 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/sdk-cli/internal/sdk/redact/redact.go b/zSDKs/sdk-cli/internal/sdk/redact/redact.go index a08de8824..c3a261d74 100644 --- a/zSDKs/sdk-cli/internal/sdk/redact/redact.go +++ b/zSDKs/sdk-cli/internal/sdk/redact/redact.go @@ -80,11 +80,14 @@ type node struct { item int values int variants []int - // plainObject, plainArray and plainScalar report union members of that - // shape holding nothing sensitive. - plainObject bool - plainArray bool - plainScalar bool + // plainObject, plainArray, plainString, plainNumber and plainBoolean + // report union members accepting JSON values of that type and holding + // nothing sensitive. + plainObject bool + plainArray bool + plainString bool + plainNumber bool + plainBoolean bool } type shape int @@ -92,7 +95,9 @@ type shape int const ( objectShape shape = iota arrayShape - scalarShape + stringShape + numberShape + booleanShape ) // redact returns body unchanged when it has nothing to mask. A body that @@ -209,12 +214,15 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by changed = changed || partChanged header := part.Header if partChanged { - // The mask is written as is, not in the part's transfer encoding. + // The mask is written as is, not in the part's transfer encoding, + // and with its own length. header = make(textproto.MIMEHeader, len(part.Header)) for key, values := range part.Header { header[key] = values } header.Del("Content-Transfer-Encoding") + header.Del("Content-Encoding") + header.Del("Content-Length") } partWriter, err := writer.CreatePart(header) if err != nil { @@ -243,16 +251,25 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if g.sensitive(roots) { return []byte(Mask), true } + // Unencoded text matches a plain string member of the field or of its + // items when sent as repeated fields, unless it is a sensitive repeated + // item or starts like a JSON object or array, which may carry sensitive + // values. + items := g.items(roots) + if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !g.sensitive(items) && !isJSONStructure(value) { + return value, false + } if masked, changed, ok := g.redactJSON(value, roots); ok { return masked, changed } - // Text that is not JSON is a repeated array item or a plain string. - if !g.sensitive(g.items(roots)) && g.plain(roots, scalarShape) { - return value, false - } return []byte(Mask), true } +func isJSONStructure(value []byte) bool { + trimmed := bytes.TrimLeft(value, " \t\r\n") + return len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '[') +} + func (g *graph) mask(value any, roots []int, depth int) (any, bool) { roots = g.expand(roots) if len(roots) == 0 { @@ -290,8 +307,14 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { changed = true } } + case string: + return g.mismatch(value, roots, stringShape) + case json.Number, float64: + return g.mismatch(value, roots, numberShape) + case bool: + return g.mismatch(value, roots, booleanShape) default: - return g.mismatch(value, roots, scalarShape) + return Mask, true } return value, changed } @@ -309,14 +332,29 @@ func (g *graph) mismatch(value any, roots []int, s shape) (any, bool) { // with a member of shape s holding nothing sensitive. func (g *graph) plain(roots []int, s shape) bool { for _, id := range g.expand(roots) { - n := g.nodes[id] - if s == objectShape && n.plainObject || s == arrayShape && n.plainArray || s == scalarShape && n.plainScalar { + if g.nodes[id].plainShape(s) { return true } } return false } +func (n node) plainShape(s shape) bool { + switch s { + case objectShape: + return n.plainObject + case arrayShape: + return n.plainArray + case stringShape: + return n.plainString + case numberShape: + return n.plainNumber + case booleanShape: + return n.plainBoolean + } + return false +} + // hasProperties reports whether any of roots describes object properties or // map values. func (g *graph) hasProperties(roots []int) bool { diff --git a/zSDKs/sdk-go/.speakeasy/gen.lock b/zSDKs/sdk-go/.speakeasy/gen.lock index 15ecbd1b4..fe10812db 100644 --- a/zSDKs/sdk-go/.speakeasy/gen.lock +++ b/zSDKs/sdk-go/.speakeasy/gen.lock @@ -783,7 +783,7 @@ trackedFiles: recursiveformfield.go: last_write_checksum: sha1:2e06532b8c5445500bd2ae8ae8586d24e911a3e2 redact/redact.go: - last_write_checksum: sha1:dbb9bc42c72d450fe72d76adb8da84653dc37d72 + last_write_checksum: sha1:e5c8b1373dc7b811b249e29ed231a47636b83b73 renderassetop.go: last_write_checksum: sha1:a788e389967e88581337759b330af30c4e562bf8 requesttimeouterror.go: diff --git a/zSDKs/sdk-go/redact/redact.go b/zSDKs/sdk-go/redact/redact.go index a08de8824..c3a261d74 100644 --- a/zSDKs/sdk-go/redact/redact.go +++ b/zSDKs/sdk-go/redact/redact.go @@ -80,11 +80,14 @@ type node struct { item int values int variants []int - // plainObject, plainArray and plainScalar report union members of that - // shape holding nothing sensitive. - plainObject bool - plainArray bool - plainScalar bool + // plainObject, plainArray, plainString, plainNumber and plainBoolean + // report union members accepting JSON values of that type and holding + // nothing sensitive. + plainObject bool + plainArray bool + plainString bool + plainNumber bool + plainBoolean bool } type shape int @@ -92,7 +95,9 @@ type shape int const ( objectShape shape = iota arrayShape - scalarShape + stringShape + numberShape + booleanShape ) // redact returns body unchanged when it has nothing to mask. A body that @@ -209,12 +214,15 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by changed = changed || partChanged header := part.Header if partChanged { - // The mask is written as is, not in the part's transfer encoding. + // The mask is written as is, not in the part's transfer encoding, + // and with its own length. header = make(textproto.MIMEHeader, len(part.Header)) for key, values := range part.Header { header[key] = values } header.Del("Content-Transfer-Encoding") + header.Del("Content-Encoding") + header.Del("Content-Length") } partWriter, err := writer.CreatePart(header) if err != nil { @@ -243,16 +251,25 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if g.sensitive(roots) { return []byte(Mask), true } + // Unencoded text matches a plain string member of the field or of its + // items when sent as repeated fields, unless it is a sensitive repeated + // item or starts like a JSON object or array, which may carry sensitive + // values. + items := g.items(roots) + if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !g.sensitive(items) && !isJSONStructure(value) { + return value, false + } if masked, changed, ok := g.redactJSON(value, roots); ok { return masked, changed } - // Text that is not JSON is a repeated array item or a plain string. - if !g.sensitive(g.items(roots)) && g.plain(roots, scalarShape) { - return value, false - } return []byte(Mask), true } +func isJSONStructure(value []byte) bool { + trimmed := bytes.TrimLeft(value, " \t\r\n") + return len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '[') +} + func (g *graph) mask(value any, roots []int, depth int) (any, bool) { roots = g.expand(roots) if len(roots) == 0 { @@ -290,8 +307,14 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { changed = true } } + case string: + return g.mismatch(value, roots, stringShape) + case json.Number, float64: + return g.mismatch(value, roots, numberShape) + case bool: + return g.mismatch(value, roots, booleanShape) default: - return g.mismatch(value, roots, scalarShape) + return Mask, true } return value, changed } @@ -309,14 +332,29 @@ func (g *graph) mismatch(value any, roots []int, s shape) (any, bool) { // with a member of shape s holding nothing sensitive. func (g *graph) plain(roots []int, s shape) bool { for _, id := range g.expand(roots) { - n := g.nodes[id] - if s == objectShape && n.plainObject || s == arrayShape && n.plainArray || s == scalarShape && n.plainScalar { + if g.nodes[id].plainShape(s) { return true } } return false } +func (n node) plainShape(s shape) bool { + switch s { + case objectShape: + return n.plainObject + case arrayShape: + return n.plainArray + case stringShape: + return n.plainString + case numberShape: + return n.plainNumber + case booleanShape: + return n.plainBoolean + } + return false +} + // hasProperties reports whether any of roots describes object properties or // map values. func (g *graph) hasProperties(roots []int) bool { diff --git a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock index e05e4dd70..499a83231 100644 --- a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock +++ b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock @@ -3574,7 +3574,7 @@ trackedFiles: internal/sdk/polling/option.go: last_write_checksum: sha1:69d4557bf43a1a97b2d3735c442d2d25be53e56f internal/sdk/redact/redact.go: - last_write_checksum: sha1:2151c354ec3cfdadceeca95d8e73e99b778ccf9e + last_write_checksum: sha1:9fa7a24ffc99bee61484e5731ac47b8ad01d9cc3 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go index e2326e429..6750f5c86 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go @@ -80,11 +80,14 @@ type node struct { item int values int variants []int - // plainObject, plainArray and plainScalar report union members of that - // shape holding nothing sensitive. - plainObject bool - plainArray bool - plainScalar bool + // plainObject, plainArray, plainString, plainNumber and plainBoolean + // report union members accepting JSON values of that type and holding + // nothing sensitive. + plainObject bool + plainArray bool + plainString bool + plainNumber bool + plainBoolean bool } type shape int @@ -92,7 +95,9 @@ type shape int const ( objectShape shape = iota arrayShape - scalarShape + stringShape + numberShape + booleanShape ) // redact returns body unchanged when it has nothing to mask. A body that @@ -209,12 +214,15 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by changed = changed || partChanged header := part.Header if partChanged { - // The mask is written as is, not in the part's transfer encoding. + // The mask is written as is, not in the part's transfer encoding, + // and with its own length. header = make(textproto.MIMEHeader, len(part.Header)) for key, values := range part.Header { header[key] = values } header.Del("Content-Transfer-Encoding") + header.Del("Content-Encoding") + header.Del("Content-Length") } partWriter, err := writer.CreatePart(header) if err != nil { @@ -243,16 +251,25 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if g.sensitive(roots) { return []byte(Mask), true } + // Unencoded text matches a plain string member of the field or of its + // items when sent as repeated fields, unless it is a sensitive repeated + // item or starts like a JSON object or array, which may carry sensitive + // values. + items := g.items(roots) + if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !g.sensitive(items) && !isJSONStructure(value) { + return value, false + } if masked, changed, ok := g.redactJSON(value, roots); ok { return masked, changed } - // Text that is not JSON is a repeated array item or a plain string. - if !g.sensitive(g.items(roots)) && g.plain(roots, scalarShape) { - return value, false - } return []byte(Mask), true } +func isJSONStructure(value []byte) bool { + trimmed := bytes.TrimLeft(value, " \t\r\n") + return len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '[') +} + func (g *graph) mask(value any, roots []int, depth int) (any, bool) { roots = g.expand(roots) if len(roots) == 0 { @@ -290,8 +307,14 @@ func (g *graph) mask(value any, roots []int, depth int) (any, bool) { changed = true } } + case string: + return g.mismatch(value, roots, stringShape) + case json.Number, float64: + return g.mismatch(value, roots, numberShape) + case bool: + return g.mismatch(value, roots, booleanShape) default: - return g.mismatch(value, roots, scalarShape) + return Mask, true } return value, changed } @@ -309,14 +332,29 @@ func (g *graph) mismatch(value any, roots []int, s shape) (any, bool) { // with a member of shape s holding nothing sensitive. func (g *graph) plain(roots []int, s shape) bool { for _, id := range g.expand(roots) { - n := g.nodes[id] - if s == objectShape && n.plainObject || s == arrayShape && n.plainArray || s == scalarShape && n.plainScalar { + if g.nodes[id].plainShape(s) { return true } } return false } +func (n node) plainShape(s shape) bool { + switch s { + case objectShape: + return n.plainObject + case arrayShape: + return n.plainArray + case stringShape: + return n.plainString + case numberShape: + return n.plainNumber + case booleanShape: + return n.plainBoolean + } + return false +} + // hasProperties reports whether any of roots describes object properties or // map values. func (g *graph) hasProperties(roots []int) bool { From b7166cc37e038fec5b0d73642de8e5f3938febd0 Mon Sep 17 00:00:00 2001 From: 2ynn Date: Thu, 8 Oct 2026 14:54:27 -0400 Subject: [PATCH 08/23] fix(go): treat int32 and float32 union members as numbers when redacting --- internal/ast/sensitive_body.go | 2 +- internal/ast/sensitive_body_test.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/ast/sensitive_body.go b/internal/ast/sensitive_body.go index 40bc8286c..65e45f1b4 100644 --- a/internal/ast/sensitive_body.go +++ b/internal/ast/sensitive_body.go @@ -313,7 +313,7 @@ func (n *SensitiveBodyNode) addPlainShapes(t *TypeDef, visited map[*TypeDef]bool n.PlainObject, n.PlainArray, n.PlainString, n.PlainNumber, n.PlainBoolean = true, true, true, true, true case DataTypeBoolean: n.PlainBoolean = true - case DataTypeInteger, DataTypeNumber: + case DataTypeInteger, DataTypeInt32, DataTypeNumber, DataTypeFloat32: n.PlainNumber = true case DataTypeBigInt, DataTypeDecimal: n.PlainNumber, n.PlainString = true, true diff --git a/internal/ast/sensitive_body_test.go b/internal/ast/sensitive_body_test.go index 7ef6e5bf9..37eebe7a7 100644 --- a/internal/ast/sensitive_body_test.go +++ b/internal/ast/sensitive_body_test.go @@ -201,6 +201,7 @@ func TestSensitiveBodyGraphPlainUnionVariants(t *testing.T) { field("mixed", union("Mixed", &TypeDef{Type: DataTypeArray, ItemType: sensitiveString()}, class("Label", field("label", plainString())))), field("text", union("Text", class("APIKey", field("key", sensitiveString())), union("Scalars", plainString(), &TypeDef{Type: DataTypeInteger}))), field("flag", union("Flag", class("APIKey", field("key", sensitiveString())), &TypeDef{Type: DataTypeBoolean})), + field("count", union("Count", class("APIKey", field("key", sensitiveString())), &TypeDef{Type: DataTypeInt32}, &TypeDef{Type: DataTypeFloat32})), field("loop", union("Loop", class("APIKey", field("key", sensitiveString())), recursivePlain)), field("sensitive", union("Sensitive", class("APIKey", field("key", sensitiveString())), sensitiveString())), ) @@ -223,6 +224,11 @@ func TestSensitiveBodyGraphPlainUnionVariants(t *testing.T) { assert.True(t, g.Nodes[flag[0]].PlainBoolean) assert.False(t, g.Nodes[flag[0]].PlainString || g.Nodes[flag[0]].PlainNumber, "a boolean member does not accept text or numbers") + count := sensitiveChild(g, g.Roots, "count") + require.Len(t, count, 1) + assert.True(t, g.Nodes[count[0]].PlainNumber, "int32 and float32 members are numbers") + assert.False(t, g.Nodes[count[0]].PlainString) + loop := sensitiveChild(g, g.Roots, "loop") require.Len(t, loop, 1) assert.True(t, g.Nodes[loop[0]].PlainString, "a recursive plain union is followed once") From 0bf043978e72b590d44104f25385885b3f07379d Mon Sep 17 00:00:00 2001 From: 2ynn Date: Thu, 8 Oct 2026 15:09:56 -0400 Subject: [PATCH 09/23] fix(go): match form fields and parts by exact name before array suffix Form fields and multipart parts were looked up with any trailing [] removed, so a property whose wire name ends in brackets missed its sensitivity marker and was logged as sent. The exact name is now looked up first; the suffix is only removed when no schema declares the bracketed name, as for arrays sent as repeated fields. --- .../{{getRedactLocation}}/redact.go.stmpl | 24 +++++++++++++++++-- .../primary/redact_additional_test.go.stmpl | 9 +++++-- .../primary/sensitive-body-redaction.yaml | 10 ++++++++ zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/sdk/redact/redact.go | 24 +++++++++++++++++-- zSDKs/sdk-go/.speakeasy/gen.lock | 2 +- zSDKs/sdk-go/redact/redact.go | 24 +++++++++++++++++-- .../.speakeasy/gen.lock | 2 +- .../internal/sdk/redact/redact.go | 24 +++++++++++++++++-- 9 files changed, 108 insertions(+), 13 deletions(-) diff --git a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl index b1c8eec0b..c03e2bab7 100644 --- a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl +++ b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl @@ -166,7 +166,7 @@ func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { if err != nil { return nil, false, false } - masked, fieldChanged := g.redactValue([]byte(value), g.child(roots, strings.TrimSuffix(key, "[]"))) + masked, fieldChanged := g.redactValue([]byte(value), g.field(roots, key)) if !fieldChanged { continue } @@ -208,7 +208,7 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by if err != nil { return nil, false, false } - masked, partChanged := g.redactValue(data, g.child(roots, strings.TrimSuffix(part.FormName(), "[]"))) + masked, partChanged := g.redactValue(data, g.field(roots, part.FormName())) changed = changed || partChanged header := part.Header if partChanged { @@ -391,6 +391,26 @@ func (g *graph) child(roots []int, name string) []int { return children } +// field returns the schemas of a form field or multipart part of roots. A +// name ending in [] that no schema declares as such names an array sent as +// repeated fields. +func (g *graph) field(roots []int, name string) []int { + if strings.HasSuffix(name, "[]") && !g.declares(roots, name) { + name = strings.TrimSuffix(name, "[]") + } + return g.child(roots, name) +} + +// declares reports whether any of roots declares the property name. +func (g *graph) declares(roots []int, name string) bool { + for _, id := range g.expand(roots) { + if _, ok := g.nodes[id].fields[name]; ok { + return true + } + } + return false +} + // items returns the array item schemas of roots. func (g *graph) items(roots []int) []int { var items []int diff --git a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl index 2a3aa1460..a614e9e14 100644 --- a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl +++ b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl @@ -187,6 +187,7 @@ func TestRedactFormRequestBody(t *testing.T) { Tokens: []string{"secret-tokens-0", "secret-tokens-1"}, Nested: &shared.SensitiveBodyFormNested{VisibleProperty: sdk.String("visible-nested"), SecretProperty: sdk.String("secret-nested")}, Metadata: &shared.SensitiveBodyNested{Label: sdk.String("visible-metadata"), Secret: sdk.String("secret-metadata")}, + BracketedPayload: sdk.String("secret-brackets"), }) require.NoError(t, err) assert.Equal(t, "sensitiveFormRedaction", client.operation) @@ -194,17 +195,19 @@ func TestRedactFormRequestBody(t *testing.T) { masked := string(client.masked) assertMasked(t, masked, - []string{"secret-token", "secret-tokens-0", "secret-tokens-1", "secret-nested", "secret-metadata"}, + []string{"secret-token", "secret-tokens-0", "secret-tokens-1", "secret-nested", "secret-metadata", "secret-brackets"}, []string{"name=visible-name", "visible-nested", "visible-metadata"}) values, err := url.ParseQuery(masked) require.NoError(t, err, "the masked body is a form") assert.Equal(t, []string{redact.Mask, redact.Mask}, values["tokens"]) assert.JSONEq(t, `{"label": "visible-metadata", "secret": "`+redact.Mask+`"}`, values.Get("metadata")) + assert.Equal(t, []string{redact.Mask}, values["payload[]"], "a property whose name ends in brackets is matched by its exact name") plainCtx := redact.WithOperation(context.Background(), "sensitiveFormRedaction") plain := []byte("name=visible-name&metadata=%7B%22label%22%3A%22visible%22%7D") assert.Equal(t, plain, redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", plain), "forms without sensitive values are returned unchanged") assert.Equal(t, "name=visible-name&metadata="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("name=visible-name&metadata=secret-text"))), "unparseable JSON-encoded fields are masked") + assert.Equal(t, "name=visible-name&tokens[]="+redact.Mask+"&tokens[]="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("name=visible-name&tokens[]=secret-tokens-0&tokens[]=secret-tokens-1"))), "arrays sent with a [] suffix are matched by their property name") } func TestRedactMultipartRequestBody(t *testing.T) { @@ -217,13 +220,14 @@ func TestRedactMultipartRequestBody(t *testing.T) { Tokens: []string{"secret-tokens-0", "secret-tokens-1"}, Metadata: &shared.SensitiveBodyNested{Label: sdk.String("visible-metadata"), Secret: sdk.String("secret-metadata")}, File: &shared.File{FileName: "visible.txt", Content: []byte("visible-file-content")}, + BracketedPayload: sdk.String("secret-brackets"), }) require.NoError(t, err) assert.Equal(t, "sensitiveMultipartRedaction", client.operation) assert.Contains(t, string(client.body), "secret-token", "the body sent is not masked") assertMasked(t, string(client.masked), - []string{"secret-token", "secret-tokens-0", "secret-tokens-1", "secret-metadata"}, + []string{"secret-token", "secret-tokens-0", "secret-tokens-1", "secret-metadata", "secret-brackets"}, []string{"visible-name", "visible-metadata", "visible.txt", "visible-file-content"}) _, params, err := mime.ParseMediaType(client.contentType) @@ -243,6 +247,7 @@ func TestRedactMultipartRequestBody(t *testing.T) { assert.Equal(t, []string{"visible-name"}, parts["name"]) assert.Equal(t, []string{redact.Mask}, parts["token"]) assert.Equal(t, []string{redact.Mask, redact.Mask}, parts["tokens"]) + assert.Equal(t, []string{redact.Mask}, parts["payload[]"], "a property whose name ends in brackets is matched by its exact name") assert.Equal(t, []string{"visible-file-content"}, parts["file"]) ctx := redact.WithOperation(context.Background(), "sensitiveMultipartRedaction") diff --git a/tests/specs/fragments/primary/sensitive-body-redaction.yaml b/tests/specs/fragments/primary/sensitive-body-redaction.yaml index 3ced4d095..b6370fd9a 100644 --- a/tests/specs/fragments/primary/sensitive-body-redaction.yaml +++ b/tests/specs/fragments/primary/sensitive-body-redaction.yaml @@ -291,6 +291,11 @@ components: required: - name properties: + payload[]: + description: Property whose wire name ends in brackets; it is not an array. + type: string + x-speakeasy-name-override: bracketedPayload + x-speakeasy-param-sensitive: true name: type: string token: @@ -318,6 +323,11 @@ components: required: - name properties: + payload[]: + description: Property whose wire name ends in brackets; it is not an array. + type: string + x-speakeasy-name-override: bracketedPayload + x-speakeasy-param-sensitive: true name: type: string token: diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index f6a2f3f9f..bbc30415d 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -566,7 +566,7 @@ trackedFiles: internal/sdk/optionalnullable/optionalnullable_test.go: last_write_checksum: sha1:d3991667754ed67db803aeab4a6239f6f9a1be37 internal/sdk/redact/redact.go: - last_write_checksum: sha1:e5c8b1373dc7b811b249e29ed231a47636b83b73 + last_write_checksum: sha1:e2645b39d8c3734c46f59e5df11bcedb74c39338 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/sdk-cli/internal/sdk/redact/redact.go b/zSDKs/sdk-cli/internal/sdk/redact/redact.go index c3a261d74..0b2cf5598 100644 --- a/zSDKs/sdk-cli/internal/sdk/redact/redact.go +++ b/zSDKs/sdk-cli/internal/sdk/redact/redact.go @@ -168,7 +168,7 @@ func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { if err != nil { return nil, false, false } - masked, fieldChanged := g.redactValue([]byte(value), g.child(roots, strings.TrimSuffix(key, "[]"))) + masked, fieldChanged := g.redactValue([]byte(value), g.field(roots, key)) if !fieldChanged { continue } @@ -210,7 +210,7 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by if err != nil { return nil, false, false } - masked, partChanged := g.redactValue(data, g.child(roots, strings.TrimSuffix(part.FormName(), "[]"))) + masked, partChanged := g.redactValue(data, g.field(roots, part.FormName())) changed = changed || partChanged header := part.Header if partChanged { @@ -393,6 +393,26 @@ func (g *graph) child(roots []int, name string) []int { return children } +// field returns the schemas of a form field or multipart part of roots. A +// name ending in [] that no schema declares as such names an array sent as +// repeated fields. +func (g *graph) field(roots []int, name string) []int { + if strings.HasSuffix(name, "[]") && !g.declares(roots, name) { + name = strings.TrimSuffix(name, "[]") + } + return g.child(roots, name) +} + +// declares reports whether any of roots declares the property name. +func (g *graph) declares(roots []int, name string) bool { + for _, id := range g.expand(roots) { + if _, ok := g.nodes[id].fields[name]; ok { + return true + } + } + return false +} + // items returns the array item schemas of roots. func (g *graph) items(roots []int) []int { var items []int diff --git a/zSDKs/sdk-go/.speakeasy/gen.lock b/zSDKs/sdk-go/.speakeasy/gen.lock index fe10812db..b05985c3c 100644 --- a/zSDKs/sdk-go/.speakeasy/gen.lock +++ b/zSDKs/sdk-go/.speakeasy/gen.lock @@ -783,7 +783,7 @@ trackedFiles: recursiveformfield.go: last_write_checksum: sha1:2e06532b8c5445500bd2ae8ae8586d24e911a3e2 redact/redact.go: - last_write_checksum: sha1:e5c8b1373dc7b811b249e29ed231a47636b83b73 + last_write_checksum: sha1:e2645b39d8c3734c46f59e5df11bcedb74c39338 renderassetop.go: last_write_checksum: sha1:a788e389967e88581337759b330af30c4e562bf8 requesttimeouterror.go: diff --git a/zSDKs/sdk-go/redact/redact.go b/zSDKs/sdk-go/redact/redact.go index c3a261d74..0b2cf5598 100644 --- a/zSDKs/sdk-go/redact/redact.go +++ b/zSDKs/sdk-go/redact/redact.go @@ -168,7 +168,7 @@ func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { if err != nil { return nil, false, false } - masked, fieldChanged := g.redactValue([]byte(value), g.child(roots, strings.TrimSuffix(key, "[]"))) + masked, fieldChanged := g.redactValue([]byte(value), g.field(roots, key)) if !fieldChanged { continue } @@ -210,7 +210,7 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by if err != nil { return nil, false, false } - masked, partChanged := g.redactValue(data, g.child(roots, strings.TrimSuffix(part.FormName(), "[]"))) + masked, partChanged := g.redactValue(data, g.field(roots, part.FormName())) changed = changed || partChanged header := part.Header if partChanged { @@ -393,6 +393,26 @@ func (g *graph) child(roots []int, name string) []int { return children } +// field returns the schemas of a form field or multipart part of roots. A +// name ending in [] that no schema declares as such names an array sent as +// repeated fields. +func (g *graph) field(roots []int, name string) []int { + if strings.HasSuffix(name, "[]") && !g.declares(roots, name) { + name = strings.TrimSuffix(name, "[]") + } + return g.child(roots, name) +} + +// declares reports whether any of roots declares the property name. +func (g *graph) declares(roots []int, name string) bool { + for _, id := range g.expand(roots) { + if _, ok := g.nodes[id].fields[name]; ok { + return true + } + } + return false +} + // items returns the array item schemas of roots. func (g *graph) items(roots []int) []int { var items []int diff --git a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock index 499a83231..152c72fbb 100644 --- a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock +++ b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock @@ -3574,7 +3574,7 @@ trackedFiles: internal/sdk/polling/option.go: last_write_checksum: sha1:69d4557bf43a1a97b2d3735c442d2d25be53e56f internal/sdk/redact/redact.go: - last_write_checksum: sha1:9fa7a24ffc99bee61484e5731ac47b8ad01d9cc3 + last_write_checksum: sha1:684d205fc4ea0088f2ac5a564a2d8739c1c0a9f0 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go index 6750f5c86..ddbbf469e 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go @@ -168,7 +168,7 @@ func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { if err != nil { return nil, false, false } - masked, fieldChanged := g.redactValue([]byte(value), g.child(roots, strings.TrimSuffix(key, "[]"))) + masked, fieldChanged := g.redactValue([]byte(value), g.field(roots, key)) if !fieldChanged { continue } @@ -210,7 +210,7 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by if err != nil { return nil, false, false } - masked, partChanged := g.redactValue(data, g.child(roots, strings.TrimSuffix(part.FormName(), "[]"))) + masked, partChanged := g.redactValue(data, g.field(roots, part.FormName())) changed = changed || partChanged header := part.Header if partChanged { @@ -393,6 +393,26 @@ func (g *graph) child(roots []int, name string) []int { return children } +// field returns the schemas of a form field or multipart part of roots. A +// name ending in [] that no schema declares as such names an array sent as +// repeated fields. +func (g *graph) field(roots []int, name string) []int { + if strings.HasSuffix(name, "[]") && !g.declares(roots, name) { + name = strings.TrimSuffix(name, "[]") + } + return g.child(roots, name) +} + +// declares reports whether any of roots declares the property name. +func (g *graph) declares(roots []int, name string) bool { + for _, id := range g.expand(roots) { + if _, ok := g.nodes[id].fields[name]; ok { + return true + } + } + return false +} + // items returns the array item schemas of roots. func (g *graph) items(roots []int) []int { var items []int From 7fec6201e4791f96a2c8af2012daa855eeb5572b Mon Sep 17 00:00:00 2001 From: 2ynn Date: Thu, 8 Oct 2026 15:26:48 -0400 Subject: [PATCH 10/23] fix(go): resolve bracketed form names per schema when redacting Whether a form field or multipart part name ending in [] named a literal property or an array sent as repeated fields was decided across all schemas at once, and only from properties that survived pruning of the sensitive graph. A literal name in one response schema could hide a sensitive array of another, and a plain bracketed property beside a sensitive one was masked. The graph now lists every declared property of the object nodes it keeps, and each schema and union variant resolves the exact name, then the name without the suffix, then its additional properties. --- internal/ast/sensitive_body.go | 14 ++++---- internal/ast/sensitive_body_test.go | 6 ++++ .../templates/common/common/typeDefs.d.ts | 4 +-- .../{{getRedactLocation}}/redact.go.stmpl | 30 ++++++++-------- .../primary/redact_additional_test.go.stmpl | 4 +++ .../primary/sensitive-body-redaction.yaml | 8 +++++ zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/sdk/redact/redact.go | 30 ++++++++-------- zSDKs/sdk-go/.speakeasy/gen.lock | 2 +- zSDKs/sdk-go/redact/redact.go | 30 ++++++++-------- .../.speakeasy/gen.lock | 2 +- .../internal/sdk/redact/redact.go | 36 ++++++++++--------- 12 files changed, 96 insertions(+), 72 deletions(-) diff --git a/internal/ast/sensitive_body.go b/internal/ast/sensitive_body.go index 65e45f1b4..08c5798ca 100644 --- a/internal/ast/sensitive_body.go +++ b/internal/ast/sensitive_body.go @@ -20,11 +20,11 @@ type SensitiveBodyNode struct { // Sensitive masks the whole value. Streams are masked as a whole when any // of their items holds a sensitive value. Sensitive bool - // Fields are object properties by wire name (JSON property, form field - // or multipart part name), sorted by name. A property - // that is not listed uses Values. When Values is set, declared properties - // are listed even when they hold nothing sensitive (with Node 0), so that - // they never fall back to Values. + // Fields are the declared object properties by wire name (JSON property, + // form field or multipart part name), sorted by name, including those + // holding nothing sensitive (with Node 0), so that a declared name is + // never mistaken for another one or for Values. A property that is not + // listed uses Values. Fields []SensitiveBodyField // Item is the schema of array items. Item int @@ -269,9 +269,7 @@ func (b *sensitiveBodyBuilder) build() *SensitiveBodyGraph { } out.Values = remap[node.values] for _, field := range node.fields { - if out.Values != 0 || remap[field.Node] != 0 { - out.Fields = append(out.Fields, SensitiveBodyField{Name: field.Name, Node: remap[field.Node]}) - } + out.Fields = append(out.Fields, SensitiveBodyField{Name: field.Name, Node: remap[field.Node]}) } sort.SliceStable(out.Fields, func(a, b int) bool { return out.Fields[a].Name < out.Fields[b].Name }) out.Item = remap[node.item] diff --git a/internal/ast/sensitive_body_test.go b/internal/ast/sensitive_body_test.go index 37eebe7a7..b2f44c76c 100644 --- a/internal/ast/sensitive_body_test.go +++ b/internal/ast/sensitive_body_test.go @@ -214,6 +214,12 @@ func TestSensitiveBodyGraphPlainUnionVariants(t *testing.T) { assert.False(t, g.Nodes[mixed[0]].PlainArray || g.Nodes[mixed[0]].PlainString || g.Nodes[mixed[0]].PlainNumber || g.Nodes[mixed[0]].PlainBoolean) assert.Len(t, g.Nodes[mixed[0]].Variants, 1) + var declared []string + for _, f := range g.Nodes[g.Roots[0]].Fields { + declared = append(declared, f.Name) + } + assert.Equal(t, []string{"count", "flag", "loop", "mixed", "sensitive", "text"}, declared, "declared properties are listed even when they hold nothing sensitive") + text := sensitiveChild(g, g.Roots, "text") require.Len(t, text, 1) assert.True(t, g.Nodes[text[0]].PlainString && g.Nodes[text[0]].PlainNumber, "members of a plain nested union are recorded") diff --git a/templates/templates/common/common/typeDefs.d.ts b/templates/templates/common/common/typeDefs.d.ts index c865854c3..7ee47eff1 100644 --- a/templates/templates/common/common/typeDefs.d.ts +++ b/templates/templates/common/common/typeDefs.d.ts @@ -2123,8 +2123,8 @@ declare global { /** Masks the whole value. */ Sensitive: boolean; /** - * Object properties by JSON name, sorted by name. Unlisted properties use - * Values. + * Declared object properties by wire name, sorted by name, including those + * holding nothing sensitive (Node 0). Unlisted properties use Values. */ Fields?: { Name: string; Node: number }[]; /** Schema of array items. */ diff --git a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl index c03e2bab7..955006d62 100644 --- a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl +++ b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl @@ -391,24 +391,26 @@ func (g *graph) child(roots []int, name string) []int { return children } -// field returns the schemas of a form field or multipart part of roots. A -// name ending in [] that no schema declares as such names an array sent as -// repeated fields. +// field returns the schemas of a form field or multipart part of roots. In +// each schema, a name ending in [] that is not declared as such names an +// array sent as repeated fields. func (g *graph) field(roots []int, name string) []int { - if strings.HasSuffix(name, "[]") && !g.declares(roots, name) { - name = strings.TrimSuffix(name, "[]") - } - return g.child(roots, name) -} - -// declares reports whether any of roots declares the property name. -func (g *graph) declares(roots []int, name string) bool { + trimmed, bracketed := strings.CutSuffix(name, "[]") + var children []int for _, id := range g.expand(roots) { - if _, ok := g.nodes[id].fields[name]; ok { - return true + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared && bracketed { + childID, declared = n.fields[trimmed] + } + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) } } - return false + return children } // items returns the array item schemas of roots. diff --git a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl index a614e9e14..f60b5ed43 100644 --- a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl +++ b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl @@ -208,6 +208,10 @@ func TestRedactFormRequestBody(t *testing.T) { assert.Equal(t, plain, redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", plain), "forms without sensitive values are returned unchanged") assert.Equal(t, "name=visible-name&metadata="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("name=visible-name&metadata=secret-text"))), "unparseable JSON-encoded fields are masked") assert.Equal(t, "name=visible-name&tokens[]="+redact.Mask+"&tokens[]="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("name=visible-name&tokens[]=secret-tokens-0&tokens[]=secret-tokens-1"))), "arrays sent with a [] suffix are matched by their property name") + assert.Equal(t, "token%5B%5D=visible-bracketed&token="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("token%5B%5D=visible-bracketed&token=secret-token"))), "a plain property whose name ends in brackets is not mistaken for a sensitive one") + + responseCtx := redact.WithOperation(context.Background(), "sensitiveBodyRedaction") + assert.Equal(t, "tokens[]="+redact.Mask, string(redact.ResponseBody(responseCtx, "application/x-www-form-urlencoded", []byte("tokens[]=secret-tokens-0"))), "a bracketed name declared by one response schema still matches a sensitive array of another") } func TestRedactMultipartRequestBody(t *testing.T) { diff --git a/tests/specs/fragments/primary/sensitive-body-redaction.yaml b/tests/specs/fragments/primary/sensitive-body-redaction.yaml index b6370fd9a..e7298e8a7 100644 --- a/tests/specs/fragments/primary/sensitive-body-redaction.yaml +++ b/tests/specs/fragments/primary/sensitive-body-redaction.yaml @@ -226,6 +226,10 @@ components: properties: message: type: string + tokens[]: + description: Property whose wire name ends in brackets, holding nothing sensitive, while another response schema of the operation has a sensitive tokens array. + type: string + x-speakeasy-name-override: bracketedTokens secret: type: string x-speakeasy-param-sensitive: true @@ -291,6 +295,10 @@ components: required: - name properties: + token[]: + description: Property whose wire name ends in brackets, holding nothing sensitive, beside a sensitive token property. + type: string + x-speakeasy-name-override: bracketedToken payload[]: description: Property whose wire name ends in brackets; it is not an array. type: string diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index bbc30415d..d22aa120b 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -566,7 +566,7 @@ trackedFiles: internal/sdk/optionalnullable/optionalnullable_test.go: last_write_checksum: sha1:d3991667754ed67db803aeab4a6239f6f9a1be37 internal/sdk/redact/redact.go: - last_write_checksum: sha1:e2645b39d8c3734c46f59e5df11bcedb74c39338 + last_write_checksum: sha1:c954a1e23247c7d7587d2dba0f4bd52a68127603 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/sdk-cli/internal/sdk/redact/redact.go b/zSDKs/sdk-cli/internal/sdk/redact/redact.go index 0b2cf5598..7053a4d81 100644 --- a/zSDKs/sdk-cli/internal/sdk/redact/redact.go +++ b/zSDKs/sdk-cli/internal/sdk/redact/redact.go @@ -393,24 +393,26 @@ func (g *graph) child(roots []int, name string) []int { return children } -// field returns the schemas of a form field or multipart part of roots. A -// name ending in [] that no schema declares as such names an array sent as -// repeated fields. +// field returns the schemas of a form field or multipart part of roots. In +// each schema, a name ending in [] that is not declared as such names an +// array sent as repeated fields. func (g *graph) field(roots []int, name string) []int { - if strings.HasSuffix(name, "[]") && !g.declares(roots, name) { - name = strings.TrimSuffix(name, "[]") - } - return g.child(roots, name) -} - -// declares reports whether any of roots declares the property name. -func (g *graph) declares(roots []int, name string) bool { + trimmed, bracketed := strings.CutSuffix(name, "[]") + var children []int for _, id := range g.expand(roots) { - if _, ok := g.nodes[id].fields[name]; ok { - return true + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared && bracketed { + childID, declared = n.fields[trimmed] + } + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) } } - return false + return children } // items returns the array item schemas of roots. diff --git a/zSDKs/sdk-go/.speakeasy/gen.lock b/zSDKs/sdk-go/.speakeasy/gen.lock index b05985c3c..2be991c2b 100644 --- a/zSDKs/sdk-go/.speakeasy/gen.lock +++ b/zSDKs/sdk-go/.speakeasy/gen.lock @@ -783,7 +783,7 @@ trackedFiles: recursiveformfield.go: last_write_checksum: sha1:2e06532b8c5445500bd2ae8ae8586d24e911a3e2 redact/redact.go: - last_write_checksum: sha1:e2645b39d8c3734c46f59e5df11bcedb74c39338 + last_write_checksum: sha1:c954a1e23247c7d7587d2dba0f4bd52a68127603 renderassetop.go: last_write_checksum: sha1:a788e389967e88581337759b330af30c4e562bf8 requesttimeouterror.go: diff --git a/zSDKs/sdk-go/redact/redact.go b/zSDKs/sdk-go/redact/redact.go index 0b2cf5598..7053a4d81 100644 --- a/zSDKs/sdk-go/redact/redact.go +++ b/zSDKs/sdk-go/redact/redact.go @@ -393,24 +393,26 @@ func (g *graph) child(roots []int, name string) []int { return children } -// field returns the schemas of a form field or multipart part of roots. A -// name ending in [] that no schema declares as such names an array sent as -// repeated fields. +// field returns the schemas of a form field or multipart part of roots. In +// each schema, a name ending in [] that is not declared as such names an +// array sent as repeated fields. func (g *graph) field(roots []int, name string) []int { - if strings.HasSuffix(name, "[]") && !g.declares(roots, name) { - name = strings.TrimSuffix(name, "[]") - } - return g.child(roots, name) -} - -// declares reports whether any of roots declares the property name. -func (g *graph) declares(roots []int, name string) bool { + trimmed, bracketed := strings.CutSuffix(name, "[]") + var children []int for _, id := range g.expand(roots) { - if _, ok := g.nodes[id].fields[name]; ok { - return true + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared && bracketed { + childID, declared = n.fields[trimmed] + } + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) } } - return false + return children } // items returns the array item schemas of roots. diff --git a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock index 152c72fbb..6b6eea114 100644 --- a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock +++ b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock @@ -3574,7 +3574,7 @@ trackedFiles: internal/sdk/polling/option.go: last_write_checksum: sha1:69d4557bf43a1a97b2d3735c442d2d25be53e56f internal/sdk/redact/redact.go: - last_write_checksum: sha1:684d205fc4ea0088f2ac5a564a2d8739c1c0a9f0 + last_write_checksum: sha1:ce4b345162fb9db152df0aae502178aa6d54a166 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go index ddbbf469e..77d39c90b 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go @@ -393,24 +393,26 @@ func (g *graph) child(roots []int, name string) []int { return children } -// field returns the schemas of a form field or multipart part of roots. A -// name ending in [] that no schema declares as such names an array sent as -// repeated fields. +// field returns the schemas of a form field or multipart part of roots. In +// each schema, a name ending in [] that is not declared as such names an +// array sent as repeated fields. func (g *graph) field(roots []int, name string) []int { - if strings.HasSuffix(name, "[]") && !g.declares(roots, name) { - name = strings.TrimSuffix(name, "[]") - } - return g.child(roots, name) -} - -// declares reports whether any of roots declares the property name. -func (g *graph) declares(roots []int, name string) bool { + trimmed, bracketed := strings.CutSuffix(name, "[]") + var children []int for _, id := range g.expand(roots) { - if _, ok := g.nodes[id].fields[name]; ok { - return true + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared && bracketed { + childID, declared = n.fields[trimmed] + } + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) } } - return false + return children } // items returns the array item schemas of roots. @@ -447,7 +449,7 @@ func (g *graph) expand(roots []int) []int { } var operations = map[string]struct{ request, response *graph }{ - "create-x-param-sensitive": {request: &graph{nodes: []node{{}, {fields: map[string]int{"string_request": 3, "string_request_and_response": 2}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}, response: &graph{nodes: []node{{}, {fields: map[string]int{"string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, - "get-x-param-sensitive": {response: &graph{nodes: []node{{}, {fields: map[string]int{"string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, - "update-x-param-sensitive": {request: &graph{nodes: []node{{}, {fields: map[string]int{"string_request": 3, "string_request_and_response": 2}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}, response: &graph{nodes: []node{{}, {fields: map[string]int{"string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, + "create-x-param-sensitive": {request: &graph{nodes: []node{{}, {fields: map[string]int{"string_request": 3, "string_request_and_response": 2, "string_response": 0}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}, response: &graph{nodes: []node{{}, {fields: map[string]int{"id": 0, "string_request": 0, "string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, + "get-x-param-sensitive": {response: &graph{nodes: []node{{}, {fields: map[string]int{"id": 0, "string_request": 0, "string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, + "update-x-param-sensitive": {request: &graph{nodes: []node{{}, {fields: map[string]int{"string_request": 3, "string_request_and_response": 2, "string_response": 0}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}, response: &graph{nodes: []node{{}, {fields: map[string]int{"id": 0, "string_request": 0, "string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, } From 67c01a023d39a58d01bc8be964c69c74107577ab Mon Sep 17 00:00:00 2001 From: 2ynn Date: Thu, 8 Oct 2026 15:30:55 -0400 Subject: [PATCH 11/23] fix(go): mask duplicate keys, encoded parts and mislabelled forms when redacting - JSON with a repeated object key is masked as a whole: decoding keeps only the last value, so the bytes returned unchanged could still hold an earlier sensitive one. - Multipart parts of a field with sensitive values are masked as a whole when transfer or content encoded, since their bytes are not the value. - Every repeated item of a sensitive array is masked, including `null`. - A form body whose keys start like JSON is masked as a whole. --- .../{{getRedactLocation}}/redact.go.stmpl | 75 +++++++++++++++++-- .../primary/redact_additional_test.go.stmpl | 16 ++++ zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/sdk/redact/redact.go | 75 +++++++++++++++++-- zSDKs/sdk-go/.speakeasy/gen.lock | 2 +- zSDKs/sdk-go/redact/redact.go | 75 +++++++++++++++++-- .../.speakeasy/gen.lock | 2 +- .../internal/sdk/redact/redact.go | 75 +++++++++++++++++-- 8 files changed, 287 insertions(+), 35 deletions(-) diff --git a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl index 955006d62..3d7917bde 100644 --- a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl +++ b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl @@ -137,6 +137,11 @@ func (g *graph) redactJSON(body []byte, roots []int) ([]byte, bool, bool) { if _, err := dec.Token(); err != io.EOF { return nil, false, false } + // Decoding keeps the last of duplicate keys, so the bytes returned as is + // could hold an earlier, unmasked one. + if hasDuplicateKeys(body) { + return nil, false, false + } masked, changed := g.mask(value, roots, 0) if !changed { return body, false, true @@ -159,7 +164,7 @@ func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { for i, pair := range pairs { rawKey, rawValue, _ := strings.Cut(pair, "=") key, err := url.QueryUnescape(rawKey) - if err != nil { + if err != nil || strings.HasPrefix(key, "{") || strings.HasPrefix(key, "[") { return nil, false, false } value, err := url.QueryUnescape(rawValue) @@ -208,7 +213,11 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by if err != nil { return nil, false, false } - masked, partChanged := g.redactValue(data, g.field(roots, part.FormName())) + partRoots := g.field(roots, part.FormName()) + masked, partChanged := g.redactValue(data, partRoots) + if !partChanged && len(g.expand(partRoots)) > 0 && isEncodedPart(part.Header) { + masked, partChanged = []byte(Mask), true + } changed = changed || partChanged header := part.Header if partChanged { @@ -246,15 +255,14 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if len(g.expand(roots)) == 0 { return value, false } - if g.sensitive(roots) { + items := g.items(roots) + if g.sensitive(roots) || g.sensitive(items) && !isJSONStructure(value) { return []byte(Mask), true } // Unencoded text matches a plain string member of the field or of its - // items when sent as repeated fields, unless it is a sensitive repeated - // item or starts like a JSON object or array, which may carry sensitive - // values. - items := g.items(roots) - if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !g.sensitive(items) && !isJSONStructure(value) { + // items when sent as repeated fields, unless it starts like a JSON object + // or array, which may carry sensitive values. + if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !isJSONStructure(value) { return value, false } if masked, changed, ok := g.redactJSON(value, roots); ok { @@ -263,6 +271,57 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { return []byte(Mask), true } +// isEncodedPart reports whether a multipart part is transfer or content +// encoded, so that its bytes cannot be read as its value. +func isEncodedPart(header textproto.MIMEHeader) bool { + switch strings.ToLower(strings.TrimSpace(header.Get("Content-Transfer-Encoding"))) { + case "", "7bit", "8bit", "binary": + return header.Get("Content-Encoding") != "" + } + return true +} + +// hasDuplicateKeys reports whether a JSON object in body repeats a key. +func hasDuplicateKeys(body []byte) bool { + type object struct { + keys map[string]bool + expectKey bool + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var stack []*object + for { + token, err := dec.Token() + if err != nil { + return false + } + switch t := token.(type) { + case json.Delim: + switch t { + case '{': + stack = append(stack, &object{keys: map[string]bool{}, expectKey: true}) + continue + case '[': + stack = append(stack, nil) + continue + } + stack = stack[:len(stack)-1] + case string: + if n := len(stack); n > 0 && stack[n-1] != nil && stack[n-1].expectKey { + if stack[n-1].keys[t] { + return true + } + stack[n-1].keys[t] = true + stack[n-1].expectKey = false + continue + } + } + if n := len(stack); n > 0 && stack[n-1] != nil { + stack[n-1].expectKey = true + } + } +} + func isJSONStructure(value []byte) bool { trimmed := bytes.TrimLeft(value, " \t\r\n") return len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '[') diff --git a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl index f60b5ed43..ed2a92bb2 100644 --- a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl +++ b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl @@ -105,6 +105,12 @@ func TestRedactSensitiveBodies(t *testing.T) { assert.Contains(t, errorBody, "visible-message") assert.NotContains(t, errorBody, "secret-error", "error response schemas are masked") + for _, body := range []string{`{"nested": {"secret": "secret-duplicate"}, "nested": null}`, `{"tokens": ["secret-duplicate"], "tokens": null}`} { + assert.Equal(t, redact.Mask, string(redact.ResponseBody(ctx, "application/json", []byte(body))), "bodies repeating a key are masked as a whole") + } + assert.JSONEq(t, `{"name": "visible-name", "nested": {"label": "a", "secret": "`+redact.Mask+`"}, "items": [{"label": "a"}, {"label": "a"}]}`, + string(redact.ResponseBody(ctx, "application/json", []byte(`{"name": "visible-name", "nested": {"label": "a", "secret": "secret-nested"}, "items": [{"label": "a"}, {"label": "a"}]}`))), "keys repeated across objects are not duplicates") + streamCtx := redact.WithOperation(context.Background(), "sensitiveStreamRedaction") assert.Equal(t, redact.Mask, string(redact.ResponseBody(streamCtx, "application/jsonl", []byte(`{"label": "visible", "secret": "secret-stream"}`))), "streams holding sensitive values are masked as a whole") } @@ -154,6 +160,7 @@ func TestRedactUnionVariantsWithoutSensitiveValues(t *testing.T) { } assert.Equal(t, "4242", redactField("secretOrCount", "4242"), "%s: a plain integer member stays readable", name) assert.Equal(t, "true", redactField("secretOrFlag", "true"), "%s: a plain boolean member stays readable", name) + assert.Equal(t, "visible-name", redactField("name", "visible-name"), "%s: fields without sensitive values stay readable", name) assert.Equal(t, "visible-item-name", redactField("secretsOrNames", "visible-item-name"), "%s: a repeated item matching a plain string member stays readable", name) assert.NotContains(t, redactField("secretsOrNames", `{"label": "visible-label", "secret": "secret-item"}`), "secret-item", "%s: a repeated object item is masked", name) } @@ -208,6 +215,8 @@ func TestRedactFormRequestBody(t *testing.T) { assert.Equal(t, plain, redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", plain), "forms without sensitive values are returned unchanged") assert.Equal(t, "name=visible-name&metadata="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("name=visible-name&metadata=secret-text"))), "unparseable JSON-encoded fields are masked") assert.Equal(t, "name=visible-name&tokens[]="+redact.Mask+"&tokens[]="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("name=visible-name&tokens[]=secret-tokens-0&tokens[]=secret-tokens-1"))), "arrays sent with a [] suffix are matched by their property name") + assert.Equal(t, "tokens="+redact.Mask+"&tokens="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("tokens=null&tokens=secret-tokens-0"))), "every repeated sensitive item is masked") + assert.Equal(t, redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte(`{"token": "secret-json", "name": "visible-name"}`))), "a JSON body labelled as a form is masked as a whole") assert.Equal(t, "token%5B%5D=visible-bracketed&token="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("token%5B%5D=visible-bracketed&token=secret-token"))), "a plain property whose name ends in brackets is not mistaken for a sensitive one") responseCtx := redact.WithOperation(context.Background(), "sensitiveBodyRedaction") @@ -264,6 +273,13 @@ func TestRedactMultipartRequestBody(t *testing.T) { assert.NotContains(t, masked, "Content-Transfer-Encoding", "masked parts are not transfer encoded") assert.NotContains(t, masked, "Content-Length", "masked parts do not keep the original length") assert.Contains(t, masked, redact.Mask) + + payloadCtx := redact.WithOperation(context.Background(), "sensitiveBodyRedaction") + encodedText := "--b\r\nContent-Disposition: form-data; name=\"secretOrText\"\r\nContent-Transfer-Encoding: base64\r\n\r\neyJzZWNyZXQiOiJzZWNyZXQtZW5jb2RlZCJ9\r\n--b--\r\n" + masked = string(redact.RequestBody(payloadCtx, "multipart/form-data; boundary=b", []byte(encodedText))) + assert.NotContains(t, masked, "eyJzZWNyZXQiOiJzZWNyZXQtZW5jb2RlZCJ9", "encoded parts of a field with sensitive values are masked") + plainEncoded := "--b\r\nContent-Disposition: form-data; name=\"name\"\r\nContent-Transfer-Encoding: base64\r\n\r\ndmlzaWJsZS1uYW1l\r\n--b--\r\n" + assert.Equal(t, plainEncoded, string(redact.RequestBody(payloadCtx, "multipart/form-data; boundary=b", []byte(plainEncoded))), "encoded parts without sensitive values are kept") } func TestRedactSplitRequestBodies(t *testing.T) { diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index d22aa120b..95b2cf4ac 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -566,7 +566,7 @@ trackedFiles: internal/sdk/optionalnullable/optionalnullable_test.go: last_write_checksum: sha1:d3991667754ed67db803aeab4a6239f6f9a1be37 internal/sdk/redact/redact.go: - last_write_checksum: sha1:c954a1e23247c7d7587d2dba0f4bd52a68127603 + last_write_checksum: sha1:c1d7f8de55abd1651db1e9e5ed38ca01d2c595dd internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/sdk-cli/internal/sdk/redact/redact.go b/zSDKs/sdk-cli/internal/sdk/redact/redact.go index 7053a4d81..355368704 100644 --- a/zSDKs/sdk-cli/internal/sdk/redact/redact.go +++ b/zSDKs/sdk-cli/internal/sdk/redact/redact.go @@ -139,6 +139,11 @@ func (g *graph) redactJSON(body []byte, roots []int) ([]byte, bool, bool) { if _, err := dec.Token(); err != io.EOF { return nil, false, false } + // Decoding keeps the last of duplicate keys, so the bytes returned as is + // could hold an earlier, unmasked one. + if hasDuplicateKeys(body) { + return nil, false, false + } masked, changed := g.mask(value, roots, 0) if !changed { return body, false, true @@ -161,7 +166,7 @@ func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { for i, pair := range pairs { rawKey, rawValue, _ := strings.Cut(pair, "=") key, err := url.QueryUnescape(rawKey) - if err != nil { + if err != nil || strings.HasPrefix(key, "{") || strings.HasPrefix(key, "[") { return nil, false, false } value, err := url.QueryUnescape(rawValue) @@ -210,7 +215,11 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by if err != nil { return nil, false, false } - masked, partChanged := g.redactValue(data, g.field(roots, part.FormName())) + partRoots := g.field(roots, part.FormName()) + masked, partChanged := g.redactValue(data, partRoots) + if !partChanged && len(g.expand(partRoots)) > 0 && isEncodedPart(part.Header) { + masked, partChanged = []byte(Mask), true + } changed = changed || partChanged header := part.Header if partChanged { @@ -248,15 +257,14 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if len(g.expand(roots)) == 0 { return value, false } - if g.sensitive(roots) { + items := g.items(roots) + if g.sensitive(roots) || g.sensitive(items) && !isJSONStructure(value) { return []byte(Mask), true } // Unencoded text matches a plain string member of the field or of its - // items when sent as repeated fields, unless it is a sensitive repeated - // item or starts like a JSON object or array, which may carry sensitive - // values. - items := g.items(roots) - if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !g.sensitive(items) && !isJSONStructure(value) { + // items when sent as repeated fields, unless it starts like a JSON object + // or array, which may carry sensitive values. + if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !isJSONStructure(value) { return value, false } if masked, changed, ok := g.redactJSON(value, roots); ok { @@ -265,6 +273,57 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { return []byte(Mask), true } +// isEncodedPart reports whether a multipart part is transfer or content +// encoded, so that its bytes cannot be read as its value. +func isEncodedPart(header textproto.MIMEHeader) bool { + switch strings.ToLower(strings.TrimSpace(header.Get("Content-Transfer-Encoding"))) { + case "", "7bit", "8bit", "binary": + return header.Get("Content-Encoding") != "" + } + return true +} + +// hasDuplicateKeys reports whether a JSON object in body repeats a key. +func hasDuplicateKeys(body []byte) bool { + type object struct { + keys map[string]bool + expectKey bool + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var stack []*object + for { + token, err := dec.Token() + if err != nil { + return false + } + switch t := token.(type) { + case json.Delim: + switch t { + case '{': + stack = append(stack, &object{keys: map[string]bool{}, expectKey: true}) + continue + case '[': + stack = append(stack, nil) + continue + } + stack = stack[:len(stack)-1] + case string: + if n := len(stack); n > 0 && stack[n-1] != nil && stack[n-1].expectKey { + if stack[n-1].keys[t] { + return true + } + stack[n-1].keys[t] = true + stack[n-1].expectKey = false + continue + } + } + if n := len(stack); n > 0 && stack[n-1] != nil { + stack[n-1].expectKey = true + } + } +} + func isJSONStructure(value []byte) bool { trimmed := bytes.TrimLeft(value, " \t\r\n") return len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '[') diff --git a/zSDKs/sdk-go/.speakeasy/gen.lock b/zSDKs/sdk-go/.speakeasy/gen.lock index 2be991c2b..86cebce7e 100644 --- a/zSDKs/sdk-go/.speakeasy/gen.lock +++ b/zSDKs/sdk-go/.speakeasy/gen.lock @@ -783,7 +783,7 @@ trackedFiles: recursiveformfield.go: last_write_checksum: sha1:2e06532b8c5445500bd2ae8ae8586d24e911a3e2 redact/redact.go: - last_write_checksum: sha1:c954a1e23247c7d7587d2dba0f4bd52a68127603 + last_write_checksum: sha1:c1d7f8de55abd1651db1e9e5ed38ca01d2c595dd renderassetop.go: last_write_checksum: sha1:a788e389967e88581337759b330af30c4e562bf8 requesttimeouterror.go: diff --git a/zSDKs/sdk-go/redact/redact.go b/zSDKs/sdk-go/redact/redact.go index 7053a4d81..355368704 100644 --- a/zSDKs/sdk-go/redact/redact.go +++ b/zSDKs/sdk-go/redact/redact.go @@ -139,6 +139,11 @@ func (g *graph) redactJSON(body []byte, roots []int) ([]byte, bool, bool) { if _, err := dec.Token(); err != io.EOF { return nil, false, false } + // Decoding keeps the last of duplicate keys, so the bytes returned as is + // could hold an earlier, unmasked one. + if hasDuplicateKeys(body) { + return nil, false, false + } masked, changed := g.mask(value, roots, 0) if !changed { return body, false, true @@ -161,7 +166,7 @@ func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { for i, pair := range pairs { rawKey, rawValue, _ := strings.Cut(pair, "=") key, err := url.QueryUnescape(rawKey) - if err != nil { + if err != nil || strings.HasPrefix(key, "{") || strings.HasPrefix(key, "[") { return nil, false, false } value, err := url.QueryUnescape(rawValue) @@ -210,7 +215,11 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by if err != nil { return nil, false, false } - masked, partChanged := g.redactValue(data, g.field(roots, part.FormName())) + partRoots := g.field(roots, part.FormName()) + masked, partChanged := g.redactValue(data, partRoots) + if !partChanged && len(g.expand(partRoots)) > 0 && isEncodedPart(part.Header) { + masked, partChanged = []byte(Mask), true + } changed = changed || partChanged header := part.Header if partChanged { @@ -248,15 +257,14 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if len(g.expand(roots)) == 0 { return value, false } - if g.sensitive(roots) { + items := g.items(roots) + if g.sensitive(roots) || g.sensitive(items) && !isJSONStructure(value) { return []byte(Mask), true } // Unencoded text matches a plain string member of the field or of its - // items when sent as repeated fields, unless it is a sensitive repeated - // item or starts like a JSON object or array, which may carry sensitive - // values. - items := g.items(roots) - if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !g.sensitive(items) && !isJSONStructure(value) { + // items when sent as repeated fields, unless it starts like a JSON object + // or array, which may carry sensitive values. + if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !isJSONStructure(value) { return value, false } if masked, changed, ok := g.redactJSON(value, roots); ok { @@ -265,6 +273,57 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { return []byte(Mask), true } +// isEncodedPart reports whether a multipart part is transfer or content +// encoded, so that its bytes cannot be read as its value. +func isEncodedPart(header textproto.MIMEHeader) bool { + switch strings.ToLower(strings.TrimSpace(header.Get("Content-Transfer-Encoding"))) { + case "", "7bit", "8bit", "binary": + return header.Get("Content-Encoding") != "" + } + return true +} + +// hasDuplicateKeys reports whether a JSON object in body repeats a key. +func hasDuplicateKeys(body []byte) bool { + type object struct { + keys map[string]bool + expectKey bool + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var stack []*object + for { + token, err := dec.Token() + if err != nil { + return false + } + switch t := token.(type) { + case json.Delim: + switch t { + case '{': + stack = append(stack, &object{keys: map[string]bool{}, expectKey: true}) + continue + case '[': + stack = append(stack, nil) + continue + } + stack = stack[:len(stack)-1] + case string: + if n := len(stack); n > 0 && stack[n-1] != nil && stack[n-1].expectKey { + if stack[n-1].keys[t] { + return true + } + stack[n-1].keys[t] = true + stack[n-1].expectKey = false + continue + } + } + if n := len(stack); n > 0 && stack[n-1] != nil { + stack[n-1].expectKey = true + } + } +} + func isJSONStructure(value []byte) bool { trimmed := bytes.TrimLeft(value, " \t\r\n") return len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '[') diff --git a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock index 6b6eea114..db77d9811 100644 --- a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock +++ b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock @@ -3574,7 +3574,7 @@ trackedFiles: internal/sdk/polling/option.go: last_write_checksum: sha1:69d4557bf43a1a97b2d3735c442d2d25be53e56f internal/sdk/redact/redact.go: - last_write_checksum: sha1:ce4b345162fb9db152df0aae502178aa6d54a166 + last_write_checksum: sha1:6defebe94d439ac2404f7ade5fc0b5d4ebab7a83 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go index 77d39c90b..0b4a63674 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go @@ -139,6 +139,11 @@ func (g *graph) redactJSON(body []byte, roots []int) ([]byte, bool, bool) { if _, err := dec.Token(); err != io.EOF { return nil, false, false } + // Decoding keeps the last of duplicate keys, so the bytes returned as is + // could hold an earlier, unmasked one. + if hasDuplicateKeys(body) { + return nil, false, false + } masked, changed := g.mask(value, roots, 0) if !changed { return body, false, true @@ -161,7 +166,7 @@ func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { for i, pair := range pairs { rawKey, rawValue, _ := strings.Cut(pair, "=") key, err := url.QueryUnescape(rawKey) - if err != nil { + if err != nil || strings.HasPrefix(key, "{") || strings.HasPrefix(key, "[") { return nil, false, false } value, err := url.QueryUnescape(rawValue) @@ -210,7 +215,11 @@ func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]by if err != nil { return nil, false, false } - masked, partChanged := g.redactValue(data, g.field(roots, part.FormName())) + partRoots := g.field(roots, part.FormName()) + masked, partChanged := g.redactValue(data, partRoots) + if !partChanged && len(g.expand(partRoots)) > 0 && isEncodedPart(part.Header) { + masked, partChanged = []byte(Mask), true + } changed = changed || partChanged header := part.Header if partChanged { @@ -248,15 +257,14 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if len(g.expand(roots)) == 0 { return value, false } - if g.sensitive(roots) { + items := g.items(roots) + if g.sensitive(roots) || g.sensitive(items) && !isJSONStructure(value) { return []byte(Mask), true } // Unencoded text matches a plain string member of the field or of its - // items when sent as repeated fields, unless it is a sensitive repeated - // item or starts like a JSON object or array, which may carry sensitive - // values. - items := g.items(roots) - if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !g.sensitive(items) && !isJSONStructure(value) { + // items when sent as repeated fields, unless it starts like a JSON object + // or array, which may carry sensitive values. + if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !isJSONStructure(value) { return value, false } if masked, changed, ok := g.redactJSON(value, roots); ok { @@ -265,6 +273,57 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { return []byte(Mask), true } +// isEncodedPart reports whether a multipart part is transfer or content +// encoded, so that its bytes cannot be read as its value. +func isEncodedPart(header textproto.MIMEHeader) bool { + switch strings.ToLower(strings.TrimSpace(header.Get("Content-Transfer-Encoding"))) { + case "", "7bit", "8bit", "binary": + return header.Get("Content-Encoding") != "" + } + return true +} + +// hasDuplicateKeys reports whether a JSON object in body repeats a key. +func hasDuplicateKeys(body []byte) bool { + type object struct { + keys map[string]bool + expectKey bool + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var stack []*object + for { + token, err := dec.Token() + if err != nil { + return false + } + switch t := token.(type) { + case json.Delim: + switch t { + case '{': + stack = append(stack, &object{keys: map[string]bool{}, expectKey: true}) + continue + case '[': + stack = append(stack, nil) + continue + } + stack = stack[:len(stack)-1] + case string: + if n := len(stack); n > 0 && stack[n-1] != nil && stack[n-1].expectKey { + if stack[n-1].keys[t] { + return true + } + stack[n-1].keys[t] = true + stack[n-1].expectKey = false + continue + } + } + if n := len(stack); n > 0 && stack[n-1] != nil { + stack[n-1].expectKey = true + } + } +} + func isJSONStructure(value []byte) bool { trimmed := bytes.TrimLeft(value, " \t\r\n") return len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '[') From c025b40cb421d8ff749bae0893c5bf0dccedcd81 Mon Sep 17 00:00:00 2001 From: 2ynn Date: Thu, 8 Oct 2026 15:40:39 -0400 Subject: [PATCH 12/23] fix(go): ignore leading whitespace when detecting JSON labelled as a form --- .../go/auxiliary/{{getRedactLocation}}/redact.go.stmpl | 2 +- .../go/tests/primary/redact_additional_test.go.stmpl | 4 +++- zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/sdk/redact/redact.go | 2 +- zSDKs/sdk-go/.speakeasy/gen.lock | 2 +- zSDKs/sdk-go/redact/redact.go | 2 +- zSDKs/terraform-provider-testing/.speakeasy/gen.lock | 2 +- .../terraform-provider-testing/internal/sdk/redact/redact.go | 2 +- 8 files changed, 10 insertions(+), 8 deletions(-) diff --git a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl index 3d7917bde..2574bbb29 100644 --- a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl +++ b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl @@ -164,7 +164,7 @@ func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { for i, pair := range pairs { rawKey, rawValue, _ := strings.Cut(pair, "=") key, err := url.QueryUnescape(rawKey) - if err != nil || strings.HasPrefix(key, "{") || strings.HasPrefix(key, "[") { + if err != nil || isJSONStructure([]byte(key)) { return nil, false, false } value, err := url.QueryUnescape(rawValue) diff --git a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl index ed2a92bb2..8a770c987 100644 --- a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl +++ b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl @@ -216,7 +216,9 @@ func TestRedactFormRequestBody(t *testing.T) { assert.Equal(t, "name=visible-name&metadata="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("name=visible-name&metadata=secret-text"))), "unparseable JSON-encoded fields are masked") assert.Equal(t, "name=visible-name&tokens[]="+redact.Mask+"&tokens[]="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("name=visible-name&tokens[]=secret-tokens-0&tokens[]=secret-tokens-1"))), "arrays sent with a [] suffix are matched by their property name") assert.Equal(t, "tokens="+redact.Mask+"&tokens="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("tokens=null&tokens=secret-tokens-0"))), "every repeated sensitive item is masked") - assert.Equal(t, redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte(`{"token": "secret-json", "name": "visible-name"}`))), "a JSON body labelled as a form is masked as a whole") + for _, body := range []string{`{"token": "secret-json", "name": "visible-name"}`, " \n\t{\"token\": \"secret-json\"}", `[{"token": "secret-json"}]`} { + assert.Equal(t, redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte(body))), "a JSON body labelled as a form is masked as a whole: %q", body) + } assert.Equal(t, "token%5B%5D=visible-bracketed&token="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("token%5B%5D=visible-bracketed&token=secret-token"))), "a plain property whose name ends in brackets is not mistaken for a sensitive one") responseCtx := redact.WithOperation(context.Background(), "sensitiveBodyRedaction") diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index 95b2cf4ac..4e196c236 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -566,7 +566,7 @@ trackedFiles: internal/sdk/optionalnullable/optionalnullable_test.go: last_write_checksum: sha1:d3991667754ed67db803aeab4a6239f6f9a1be37 internal/sdk/redact/redact.go: - last_write_checksum: sha1:c1d7f8de55abd1651db1e9e5ed38ca01d2c595dd + last_write_checksum: sha1:6325bc07070b86ba40da3bbd8489ca30cacf8a1d internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/sdk-cli/internal/sdk/redact/redact.go b/zSDKs/sdk-cli/internal/sdk/redact/redact.go index 355368704..6900ddbcd 100644 --- a/zSDKs/sdk-cli/internal/sdk/redact/redact.go +++ b/zSDKs/sdk-cli/internal/sdk/redact/redact.go @@ -166,7 +166,7 @@ func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { for i, pair := range pairs { rawKey, rawValue, _ := strings.Cut(pair, "=") key, err := url.QueryUnescape(rawKey) - if err != nil || strings.HasPrefix(key, "{") || strings.HasPrefix(key, "[") { + if err != nil || isJSONStructure([]byte(key)) { return nil, false, false } value, err := url.QueryUnescape(rawValue) diff --git a/zSDKs/sdk-go/.speakeasy/gen.lock b/zSDKs/sdk-go/.speakeasy/gen.lock index 86cebce7e..32bb002b5 100644 --- a/zSDKs/sdk-go/.speakeasy/gen.lock +++ b/zSDKs/sdk-go/.speakeasy/gen.lock @@ -783,7 +783,7 @@ trackedFiles: recursiveformfield.go: last_write_checksum: sha1:2e06532b8c5445500bd2ae8ae8586d24e911a3e2 redact/redact.go: - last_write_checksum: sha1:c1d7f8de55abd1651db1e9e5ed38ca01d2c595dd + last_write_checksum: sha1:6325bc07070b86ba40da3bbd8489ca30cacf8a1d renderassetop.go: last_write_checksum: sha1:a788e389967e88581337759b330af30c4e562bf8 requesttimeouterror.go: diff --git a/zSDKs/sdk-go/redact/redact.go b/zSDKs/sdk-go/redact/redact.go index 355368704..6900ddbcd 100644 --- a/zSDKs/sdk-go/redact/redact.go +++ b/zSDKs/sdk-go/redact/redact.go @@ -166,7 +166,7 @@ func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { for i, pair := range pairs { rawKey, rawValue, _ := strings.Cut(pair, "=") key, err := url.QueryUnescape(rawKey) - if err != nil || strings.HasPrefix(key, "{") || strings.HasPrefix(key, "[") { + if err != nil || isJSONStructure([]byte(key)) { return nil, false, false } value, err := url.QueryUnescape(rawValue) diff --git a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock index db77d9811..21fd34115 100644 --- a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock +++ b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock @@ -3574,7 +3574,7 @@ trackedFiles: internal/sdk/polling/option.go: last_write_checksum: sha1:69d4557bf43a1a97b2d3735c442d2d25be53e56f internal/sdk/redact/redact.go: - last_write_checksum: sha1:6defebe94d439ac2404f7ade5fc0b5d4ebab7a83 + last_write_checksum: sha1:044ee0368f730759744ff3c46a5134acc076a8a3 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go index 0b4a63674..a1b23a058 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go @@ -166,7 +166,7 @@ func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { for i, pair := range pairs { rawKey, rawValue, _ := strings.Cut(pair, "=") key, err := url.QueryUnescape(rawKey) - if err != nil || strings.HasPrefix(key, "{") || strings.HasPrefix(key, "[") { + if err != nil || isJSONStructure([]byte(key)) { return nil, false, false } value, err := url.QueryUnescape(rawValue) From fdb2f7613027225b60a9aa5dac4341b4128ec1cc Mon Sep 17 00:00:00 2001 From: 2ynn Date: Thu, 8 Oct 2026 15:59:25 -0400 Subject: [PATCH 13/23] fix(go): keep repeated plain scalar items readable when redacting A form field or multipart part that is not a JSON object or array may be one item of an array sent as repeated fields, so it is now matched against the item schemas too: integer and boolean items of a union with a sensitive object stay readable, as string items already did. Cases for property names ending in brackets move to dedicated operations tested only by the Go SDK, since other targets do not generate them yet. The declared-fields test preallocates its slice. --- internal/ast/sensitive_body_test.go | 2 +- .../{{getRedactLocation}}/redact.go.stmpl | 4 + .../primary/redact_additional_test.go.stmpl | 43 ++++-- .../primary/sensitive-body-redaction.yaml | 133 +++++++++++++++--- zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/sdk/redact/redact.go | 4 + zSDKs/sdk-go/.speakeasy/gen.lock | 2 +- zSDKs/sdk-go/redact/redact.go | 4 + .../.speakeasy/gen.lock | 2 +- .../internal/sdk/redact/redact.go | 4 + 10 files changed, 169 insertions(+), 31 deletions(-) diff --git a/internal/ast/sensitive_body_test.go b/internal/ast/sensitive_body_test.go index b2f44c76c..0c9f1ca37 100644 --- a/internal/ast/sensitive_body_test.go +++ b/internal/ast/sensitive_body_test.go @@ -214,7 +214,7 @@ func TestSensitiveBodyGraphPlainUnionVariants(t *testing.T) { assert.False(t, g.Nodes[mixed[0]].PlainArray || g.Nodes[mixed[0]].PlainString || g.Nodes[mixed[0]].PlainNumber || g.Nodes[mixed[0]].PlainBoolean) assert.Len(t, g.Nodes[mixed[0]].Variants, 1) - var declared []string + declared := make([]string, 0, len(g.Nodes[g.Roots[0]].Fields)) for _, f := range g.Nodes[g.Roots[0]].Fields { declared = append(declared, f.Name) } diff --git a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl index 2574bbb29..faca7241b 100644 --- a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl +++ b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl @@ -265,6 +265,10 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !isJSONStructure(value) { return value, false } + // A scalar may also be one item of an array sent as repeated fields. + if !isJSONStructure(value) { + roots = append(append([]int{}, roots...), items...) + } if masked, changed, ok := g.redactJSON(value, roots); ok { return masked, changed } diff --git a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl index 8a770c987..4782cf4be 100644 --- a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl +++ b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl @@ -162,6 +162,9 @@ func TestRedactUnionVariantsWithoutSensitiveValues(t *testing.T) { assert.Equal(t, "true", redactField("secretOrFlag", "true"), "%s: a plain boolean member stays readable", name) assert.Equal(t, "visible-name", redactField("name", "visible-name"), "%s: fields without sensitive values stay readable", name) assert.Equal(t, "visible-item-name", redactField("secretsOrNames", "visible-item-name"), "%s: a repeated item matching a plain string member stays readable", name) + assert.Equal(t, "4242", redactField("secretsOrCounts", "4242"), "%s: a repeated item matching a plain integer member stays readable", name) + assert.Equal(t, "true", redactField("secretsOrFlags", "true"), "%s: a repeated item matching a plain boolean member stays readable", name) + assert.Equal(t, redact.Mask, redactField("secretsOrCounts", "secret-text"), "%s: a repeated text item without a plain string member is masked", name) assert.NotContains(t, redactField("secretsOrNames", `{"label": "visible-label", "secret": "secret-item"}`), "secret-item", "%s: a repeated object item is masked", name) } } @@ -194,7 +197,6 @@ func TestRedactFormRequestBody(t *testing.T) { Tokens: []string{"secret-tokens-0", "secret-tokens-1"}, Nested: &shared.SensitiveBodyFormNested{VisibleProperty: sdk.String("visible-nested"), SecretProperty: sdk.String("secret-nested")}, Metadata: &shared.SensitiveBodyNested{Label: sdk.String("visible-metadata"), Secret: sdk.String("secret-metadata")}, - BracketedPayload: sdk.String("secret-brackets"), }) require.NoError(t, err) assert.Equal(t, "sensitiveFormRedaction", client.operation) @@ -202,13 +204,12 @@ func TestRedactFormRequestBody(t *testing.T) { masked := string(client.masked) assertMasked(t, masked, - []string{"secret-token", "secret-tokens-0", "secret-tokens-1", "secret-nested", "secret-metadata", "secret-brackets"}, + []string{"secret-token", "secret-tokens-0", "secret-tokens-1", "secret-nested", "secret-metadata"}, []string{"name=visible-name", "visible-nested", "visible-metadata"}) values, err := url.ParseQuery(masked) require.NoError(t, err, "the masked body is a form") assert.Equal(t, []string{redact.Mask, redact.Mask}, values["tokens"]) assert.JSONEq(t, `{"label": "visible-metadata", "secret": "`+redact.Mask+`"}`, values.Get("metadata")) - assert.Equal(t, []string{redact.Mask}, values["payload[]"], "a property whose name ends in brackets is matched by its exact name") plainCtx := redact.WithOperation(context.Background(), "sensitiveFormRedaction") plain := []byte("name=visible-name&metadata=%7B%22label%22%3A%22visible%22%7D") @@ -219,10 +220,36 @@ func TestRedactFormRequestBody(t *testing.T) { for _, body := range []string{`{"token": "secret-json", "name": "visible-name"}`, " \n\t{\"token\": \"secret-json\"}", `[{"token": "secret-json"}]`} { assert.Equal(t, redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte(body))), "a JSON body labelled as a form is masked as a whole: %q", body) } - assert.Equal(t, "token%5B%5D=visible-bracketed&token="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("token%5B%5D=visible-bracketed&token=secret-token"))), "a plain property whose name ends in brackets is not mistaken for a sensitive one") +} + +func TestRedactBracketedFieldNames(t *testing.T) { + client := &redactCaptureClient{status: http.StatusOK} + s := sdk.New(sdk.WithServerURL("http://localhost"), sdk.WithClient(client)) - responseCtx := redact.WithOperation(context.Background(), "sensitiveBodyRedaction") - assert.Equal(t, "tokens[]="+redact.Mask, string(redact.ResponseBody(responseCtx, "application/x-www-form-urlencoded", []byte("tokens[]=secret-tokens-0"))), "a bracketed name declared by one response schema still matches a sensitive array of another") + _, _ = s.SensitiveBodies.BracketedFormRedaction(context.Background(), shared.SensitiveBodyBracketedForm{ + Name: "visible-name", + BracketedPayload: sdk.String("secret-brackets"), + Token: sdk.String("secret-token"), + BracketedToken: sdk.String("visible-bracketed"), + }) + assert.Equal(t, "sensitiveBracketedFormRedaction", client.operation) + values, err := url.ParseQuery(string(client.masked)) + require.NoError(t, err, "the masked body is a form") + assert.Equal(t, []string{redact.Mask}, values["payload[]"], "a property whose name ends in brackets is matched by its exact name") + assert.Equal(t, []string{redact.Mask}, values["token"]) + assert.Equal(t, []string{"visible-bracketed"}, values["token[]"], "a plain bracketed property is not mistaken for a sensitive one") + assert.Equal(t, []string{"visible-name"}, values["name"]) + + ctx := redact.WithOperation(context.Background(), "sensitiveBracketedFormRedaction") + assert.Equal(t, "name=visible-name&tokens[]="+redact.Mask+"&tokens[]="+redact.Mask, string(redact.RequestBody(ctx, "application/x-www-form-urlencoded", []byte("name=visible-name&tokens[]=secret-tokens-0&tokens[]=secret-tokens-1"))), "arrays sent with a [] suffix are matched by their property name") + assert.Equal(t, "tokens[]="+redact.Mask, string(redact.ResponseBody(ctx, "application/x-www-form-urlencoded", []byte("tokens[]=secret-tokens-0"))), "a bracketed name declared by one response schema still matches a sensitive array of another") + + _, _ = s.SensitiveBodies.BracketedMultipartRedaction(context.Background(), shared.SensitiveBodyBracketedMultipart{ + Name: "visible-name", + BracketedPayload: sdk.String("secret-brackets"), + }) + assert.Equal(t, "sensitiveBracketedMultipartRedaction", client.operation) + assertMasked(t, string(client.masked), []string{"secret-brackets"}, []string{"visible-name", `name="payload[]"`}) } func TestRedactMultipartRequestBody(t *testing.T) { @@ -235,14 +262,13 @@ func TestRedactMultipartRequestBody(t *testing.T) { Tokens: []string{"secret-tokens-0", "secret-tokens-1"}, Metadata: &shared.SensitiveBodyNested{Label: sdk.String("visible-metadata"), Secret: sdk.String("secret-metadata")}, File: &shared.File{FileName: "visible.txt", Content: []byte("visible-file-content")}, - BracketedPayload: sdk.String("secret-brackets"), }) require.NoError(t, err) assert.Equal(t, "sensitiveMultipartRedaction", client.operation) assert.Contains(t, string(client.body), "secret-token", "the body sent is not masked") assertMasked(t, string(client.masked), - []string{"secret-token", "secret-tokens-0", "secret-tokens-1", "secret-metadata", "secret-brackets"}, + []string{"secret-token", "secret-tokens-0", "secret-tokens-1", "secret-metadata"}, []string{"visible-name", "visible-metadata", "visible.txt", "visible-file-content"}) _, params, err := mime.ParseMediaType(client.contentType) @@ -262,7 +288,6 @@ func TestRedactMultipartRequestBody(t *testing.T) { assert.Equal(t, []string{"visible-name"}, parts["name"]) assert.Equal(t, []string{redact.Mask}, parts["token"]) assert.Equal(t, []string{redact.Mask, redact.Mask}, parts["tokens"]) - assert.Equal(t, []string{redact.Mask}, parts["payload[]"], "a property whose name ends in brackets is matched by its exact name") assert.Equal(t, []string{"visible-file-content"}, parts["file"]) ctx := redact.WithOperation(context.Background(), "sensitiveMultipartRedaction") diff --git a/tests/specs/fragments/primary/sensitive-body-redaction.yaml b/tests/specs/fragments/primary/sensitive-body-redaction.yaml index e7298e8a7..a6b48dfce 100644 --- a/tests/specs/fragments/primary/sensitive-body-redaction.yaml +++ b/tests/specs/fragments/primary/sensitive-body-redaction.yaml @@ -137,6 +137,56 @@ paths: responses: "200": description: OK + /anything/sensitiveBracketedFormRedaction: + post: + operationId: sensitiveBracketedFormRedaction + x-speakeasy-group: sensitiveBodies + x-speakeasy-name-override: bracketedFormRedaction + x-speakeasy-exclude-test-sdk: [cli, csharp, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + description: >- + Exercises form field names ending in brackets: a sensitive property + whose wire name ends in brackets, a plain bracketed property beside a + sensitive one of the same base name, and an array of sensitive values + sent as repeated fields with a bracket suffix. A response schema + declaring a plain bracketed property must not hide the sensitive array + of another response schema. + requestBody: + required: true + content: + application/x-www-form-urlencoded: + schema: + $ref: "#/components/schemas/SensitiveBodyBracketedForm" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/SensitiveBodyBracketedTokens" + "400": + description: Bad request + content: + application/json: + schema: + $ref: "#/components/schemas/SensitiveBodyBracketedError" + /anything/sensitiveBracketedMultipartRedaction: + post: + operationId: sensitiveBracketedMultipartRedaction + x-speakeasy-group: sensitiveBodies + x-speakeasy-name-override: bracketedMultipartRedaction + x-speakeasy-exclude-test-sdk: [cli, csharp, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + description: >- + Exercises a multipart part whose name ends in brackets and holds a + sensitive value, beside a plain part. + requestBody: + required: true + content: + multipart/form-data: + schema: + $ref: "#/components/schemas/SensitiveBodyBracketedMultipart" + responses: + "200": + description: OK components: schemas: SensitiveBodyPayload: @@ -216,6 +266,20 @@ components: oneOf: - $ref: "#/components/schemas/SensitiveBodyNested" - type: string + secretsOrCounts: + description: Array of a union of an object with a sensitive value and an integer; integer items sent as repeated fields stay readable. + type: array + items: + oneOf: + - $ref: "#/components/schemas/SensitiveBodyNested" + - type: integer + secretsOrFlags: + description: Array of a union of an object with a sensitive value and a boolean; boolean items sent as repeated fields stay readable. + type: array + items: + oneOf: + - $ref: "#/components/schemas/SensitiveBodyNested" + - type: boolean SensitiveBodyPlainLabel: type: object properties: @@ -226,10 +290,6 @@ components: properties: message: type: string - tokens[]: - description: Property whose wire name ends in brackets, holding nothing sensitive, while another response schema of the operation has a sensitive tokens array. - type: string - x-speakeasy-name-override: bracketedTokens secret: type: string x-speakeasy-param-sensitive: true @@ -295,15 +355,6 @@ components: required: - name properties: - token[]: - description: Property whose wire name ends in brackets, holding nothing sensitive, beside a sensitive token property. - type: string - x-speakeasy-name-override: bracketedToken - payload[]: - description: Property whose wire name ends in brackets; it is not an array. - type: string - x-speakeasy-name-override: bracketedPayload - x-speakeasy-param-sensitive: true name: type: string token: @@ -331,11 +382,6 @@ components: required: - name properties: - payload[]: - description: Property whose wire name ends in brackets; it is not an array. - type: string - x-speakeasy-name-override: bracketedPayload - x-speakeasy-param-sensitive: true name: type: string token: @@ -351,3 +397,54 @@ components: file: type: string format: binary + SensitiveBodyBracketedForm: + type: object + required: + - name + properties: + name: + type: string + payload[]: + description: Property whose wire name ends in brackets; it is not an array. + type: string + x-speakeasy-name-override: bracketedPayload + x-speakeasy-param-sensitive: true + token: + type: string + x-speakeasy-param-sensitive: true + token[]: + description: Property whose wire name ends in brackets, holding nothing sensitive, beside a sensitive token property. + type: string + x-speakeasy-name-override: bracketedToken + tokens: + type: array + items: + type: string + x-speakeasy-param-sensitive: true + SensitiveBodyBracketedTokens: + type: object + properties: + tokens: + type: array + items: + type: string + x-speakeasy-param-sensitive: true + SensitiveBodyBracketedError: + type: object + properties: + tokens[]: + description: Property whose wire name ends in brackets, holding nothing sensitive, while another response schema has a sensitive tokens array. + type: string + x-speakeasy-name-override: bracketedTokens + SensitiveBodyBracketedMultipart: + type: object + required: + - name + properties: + name: + type: string + payload[]: + description: Property whose wire name ends in brackets; it is not an array. + type: string + x-speakeasy-name-override: bracketedPayload + x-speakeasy-param-sensitive: true diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index 4e196c236..949102087 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -566,7 +566,7 @@ trackedFiles: internal/sdk/optionalnullable/optionalnullable_test.go: last_write_checksum: sha1:d3991667754ed67db803aeab4a6239f6f9a1be37 internal/sdk/redact/redact.go: - last_write_checksum: sha1:6325bc07070b86ba40da3bbd8489ca30cacf8a1d + last_write_checksum: sha1:c034b68429958375c94f272c0ec5523e5a5b009f internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/sdk-cli/internal/sdk/redact/redact.go b/zSDKs/sdk-cli/internal/sdk/redact/redact.go index 6900ddbcd..4d890914d 100644 --- a/zSDKs/sdk-cli/internal/sdk/redact/redact.go +++ b/zSDKs/sdk-cli/internal/sdk/redact/redact.go @@ -267,6 +267,10 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !isJSONStructure(value) { return value, false } + // A scalar may also be one item of an array sent as repeated fields. + if !isJSONStructure(value) { + roots = append(append([]int{}, roots...), items...) + } if masked, changed, ok := g.redactJSON(value, roots); ok { return masked, changed } diff --git a/zSDKs/sdk-go/.speakeasy/gen.lock b/zSDKs/sdk-go/.speakeasy/gen.lock index 32bb002b5..568101f9a 100644 --- a/zSDKs/sdk-go/.speakeasy/gen.lock +++ b/zSDKs/sdk-go/.speakeasy/gen.lock @@ -783,7 +783,7 @@ trackedFiles: recursiveformfield.go: last_write_checksum: sha1:2e06532b8c5445500bd2ae8ae8586d24e911a3e2 redact/redact.go: - last_write_checksum: sha1:6325bc07070b86ba40da3bbd8489ca30cacf8a1d + last_write_checksum: sha1:c034b68429958375c94f272c0ec5523e5a5b009f renderassetop.go: last_write_checksum: sha1:a788e389967e88581337759b330af30c4e562bf8 requesttimeouterror.go: diff --git a/zSDKs/sdk-go/redact/redact.go b/zSDKs/sdk-go/redact/redact.go index 6900ddbcd..4d890914d 100644 --- a/zSDKs/sdk-go/redact/redact.go +++ b/zSDKs/sdk-go/redact/redact.go @@ -267,6 +267,10 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !isJSONStructure(value) { return value, false } + // A scalar may also be one item of an array sent as repeated fields. + if !isJSONStructure(value) { + roots = append(append([]int{}, roots...), items...) + } if masked, changed, ok := g.redactJSON(value, roots); ok { return masked, changed } diff --git a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock index 21fd34115..885da2a81 100644 --- a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock +++ b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock @@ -3574,7 +3574,7 @@ trackedFiles: internal/sdk/polling/option.go: last_write_checksum: sha1:69d4557bf43a1a97b2d3735c442d2d25be53e56f internal/sdk/redact/redact.go: - last_write_checksum: sha1:044ee0368f730759744ff3c46a5134acc076a8a3 + last_write_checksum: sha1:90d5f1a7d4f0176deedf3d6a905eb39968a4ec38 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go index a1b23a058..4da057659 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go @@ -267,6 +267,10 @@ func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !isJSONStructure(value) { return value, false } + // A scalar may also be one item of an array sent as repeated fields. + if !isJSONStructure(value) { + roots = append(append([]int{}, roots...), items...) + } if masked, changed, ok := g.redactJSON(value, roots); ok { return masked, changed } From f79abd862a65ea6afdd11ac0e52f2098ef7f2e13 Mon Sep 17 00:00:00 2001 From: 2ynn Date: Thu, 1 Oct 2026 14:03:21 -0400 Subject: [PATCH 14/23] fix(cli): redact request-body values marked sensitive in diagnostics --dry-run and --debug request previews redacted body fields only by a fixed list of key names, so a secret under a generic name (for example "value") was printed in plain text. - templateSensitiveBodyFields collects, per operation, the JSON paths of request-body values whose schema sets x-speakeasy-param-sensitive or format: password ("*" for array items and map values, "**" for any depth of a recursive schema, union variants at their parent's path) - diagnostics looks the paths up by the command's speakeasy_operation annotation and redacts them in JSON bodies, form fields and multipart parts (including repeated and JSON-encoded ones), on top of the name-based rules; response bodies and the wire request are unchanged - same-named fields of other schemas stay readable --- .changesets/1790872792-577bf8cd.yaml | 10 + templates/templates/cli/README.md | 1 + .../internal/client/diagnostics.go.stmpl | 138 +++++++-- templates/templates/cli/includes/security.ts | 100 +++++++ .../tests/primary/diagnostics_test.go.stmpl | 107 +++++++ .../primary/cli-sensitive-body-fields.yaml | 272 ++++++++++++++++++ zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/client/diagnostics.go | 134 +++++++-- 8 files changed, 721 insertions(+), 43 deletions(-) create mode 100644 .changesets/1790872792-577bf8cd.yaml create mode 100644 tests/specs/fragments/primary/cli-sensitive-body-fields.yaml diff --git a/.changesets/1790872792-577bf8cd.yaml b/.changesets/1790872792-577bf8cd.yaml new file mode 100644 index 000000000..dee934c90 --- /dev/null +++ b/.changesets/1790872792-577bf8cd.yaml @@ -0,0 +1,10 @@ +id: 1790872792-577bf8cd +features: + - core +targets: + - cli +type: fix +bump: patch +description: '--dry-run and --debug request previews redact request-body values whose schema is marked x-speakeasy-param-sensitive or format: password, in JSON bodies, form fields and multipart parts (including JSON-encoded ones), so secrets under generic names are no longer printed; same-named unmarked fields of other schemas stay readable' +author: 2ynn +date: "2026-10-01" diff --git a/templates/templates/cli/README.md b/templates/templates/cli/README.md index fe8baae85..22d7742ce 100644 --- a/templates/templates/cli/README.md +++ b/templates/templates/cli/README.md @@ -1491,6 +1491,7 @@ cli users list --debug - **Headers**: Case-insensitive denylist (`authorization`, `cookie`, `set-cookie`, `x-api-key`, `x-session-token`, `*-secret`, `*-token` suffixes) → `[REDACTED]` - **URL query**: Credential parameters (`api_key`, `token`, `client_secret`, `signature`, etc.) are redacted without reordering the query +- **Request body values marked sensitive**: `templateSensitiveBodyFields` (`includes/security.ts`) collects, per operation, the JSON paths of request-body values whose schema sets `x-speakeasy-param-sensitive: true` or `format: password` (object fields, array items and map values as `*`, union variants at their parent's path, and `**` for any depth of a recursive schema) into `sensitiveBodyFields`. Dry-run and debug request previews redact those values, keyed by the command's `speakeasy_operation` annotation, so a secret under a generic name (`value`) is hidden while a same-named field of another schema stays readable. Form fields and multipart parts match on their names (repeated ones as array items), and JSON-encoded ones on the paths inside them; a form field that holds JSON is shown re-encoded (compact, keys sorted) when a sensitive path may lie inside it. Response bodies use the name-based rules only. The marker must sit on the value's own schema: an extension next to a `$ref` is not read - **JSON body**: Separator/case-normalized sensitive keys and canonical base64 strings (encoded length ≥128, excluding all-hex IDs) are redacted; numbers retain their exact lexemes and depth overflow becomes an explicit marker - **Other bodies**: Binary media becomes `` and multipart payloads become a deterministic inventory; human dry-run is not truncated. Debug applies the same redactor, then caps the rendered preview at 4 KiB - **Credentials**: Generated request construction uses request-scoped resolvers. Dry-run reads flags, environment and config, but never probes the OS keychain; whoami also uses the request-scoped resolvers (so `whoami --dry-run` skips the keychain), while configuration retains keychain access diff --git a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl index 67bd3f125..ab50bf6d3 100644 --- a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl +++ b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl @@ -84,6 +84,16 @@ var sensitiveJSONKeys = map[string]bool{ "clientsecret": true, } +// sensitiveBodyFields lists, per operation id, the request-body fields the API +// marks sensitive (x-speakeasy-param-sensitive or format: password) as JSON +// paths, where "*" stands for any array index or map key and "**" for any run +// of segments inside a recursive schema. +var sensitiveBodyFields = map[string][][]string{ + {{- range templateSensitiveBodyFields}} + "{{escapeGoString .OperationID}}": {{.Paths}}, + {{- end}} +} + var sensitiveQueryKeys = map[string]bool{ "apikey": true, "accesstoken": true, @@ -161,8 +171,73 @@ func isSensitiveName(key string) bool { return false } -// redactJSON recursively redacts sensitive keys in a JSON structure. -func redactJSON(v interface{}, depth int) interface{} { +// sensitiveBodyFieldsFor returns the sensitive request-body paths of the +// operation that cmd (an operation or intent command) invokes. +func sensitiveBodyFieldsFor(cmd *cobra.Command) [][]string { + if cmd == nil { + return nil + } + return sensitiveBodyFields[cmd.Annotations["speakeasy_operation"]] +} + +func isSensitiveBodyPath(sensitive [][]string, path []string) bool { + for _, candidate := range sensitive { + if matchesBodyPath(candidate, path) { + return true + } + } + return false +} + +func matchesBodyPath(pattern, path []string) bool { + if len(pattern) == 0 { + return len(path) == 0 + } + if pattern[0] == "**" { + for i := 0; i <= len(path); i++ { + if matchesBodyPath(pattern[1:], path[i:]) { + return true + } + } + return false + } + if len(path) == 0 || (pattern[0] != "*" && pattern[0] != path[0]) { + return false + } + return matchesBodyPath(pattern[1:], path[1:]) +} + +// isSensitiveEncodedField reports whether a form field or multipart part is +// sensitive; repeated fields carry the items of an array. +func isSensitiveEncodedField(sensitive [][]string, name string) bool { + return isSensitiveBodyPath(sensitive, []string{name}) || isSensitiveBodyPath(sensitive, []string{name, "*"}) +} + +// sensitiveBodySubpaths returns the sensitive paths below the top-level field +// name, relative to it, for fields sent as an encoded JSON value (a multipart +// part or a form field). +func sensitiveBodySubpaths(sensitive [][]string, name string) [][]string { + var out [][]string + var collect func(pattern []string) + collect = func(pattern []string) { + switch { + case len(pattern) == 0: + case pattern[0] == "**": + out = append(out, pattern) + collect(pattern[1:]) + case (pattern[0] == "*" || pattern[0] == name) && len(pattern) > 1: + out = append(out, pattern[1:]) + } + } + for _, pattern := range sensitive { + collect(pattern) + } + return out +} + +// redactJSON recursively redacts sensitive keys in a JSON structure, plus the +// values at the sensitive request-body paths. +func redactJSON(v interface{}, depth int, path []string, sensitive [][]string) interface{} { if depth > maxRedactDepth { return redactionDepthMarker } @@ -170,17 +245,24 @@ func redactJSON(v interface{}, depth int) interface{} { case map[string]interface{}: out := make(map[string]interface{}, len(val)) for k, child := range val { - if sensitiveJSONKeys[normalizeSensitiveKey(k)] { + childPath := append(path[:len(path):len(path)], k) + if sensitiveJSONKeys[normalizeSensitiveKey(k)] || isSensitiveBodyPath(sensitive, childPath) { out[k] = "[REDACTED]" } else { - out[k] = redactJSON(child, depth+1) + out[k] = redactJSON(child, depth+1, childPath, sensitive) } } return out case []interface{}: out := make([]interface{}, len(val)) + childPath := append(path[:len(path):len(path)], "*") + redactItems := isSensitiveBodyPath(sensitive, childPath) for i, child := range val { - out[i] = redactJSON(child, depth+1) + if redactItems { + out[i] = "[REDACTED]" + } else { + out[i] = redactJSON(child, depth+1, childPath, sensitive) + } } return out case string: @@ -256,7 +338,7 @@ func isAllHex(value string) bool { return true } -func decodeJSON(body []byte) (interface{}, bool) { +func decodeJSON(body []byte, sensitive [][]string) (interface{}, bool) { dec := json.NewDecoder(bytes.NewReader(body)) dec.UseNumber() var parsed interface{} @@ -266,7 +348,7 @@ func decodeJSON(body []byte) (interface{}, bool) { if err := dec.Decode(&struct{}{}); err != io.EOF { return nil, false } - return redactJSON(parsed, 0), true + return redactJSON(parsed, 0, nil, sensitive), true } func isTextMediaType(mediaType string) bool { @@ -277,7 +359,9 @@ func isTextMediaType(mediaType string) bool { mediaType == "application/x-www-form-urlencoded" } -func previewBody(body []byte, contentType string) interface{} { +// previewBody renders a body for diagnostics; sensitive holds the request-body +// paths to redact (nil for response bodies). +func previewBody(body []byte, contentType string, sensitive [][]string) interface{} { if len(body) == 0 { return nil } @@ -286,16 +370,16 @@ func previewBody(body []byte, contentType string) interface{} { mediaType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) } if isJSONMediaType(mediaType) || mediaType == "" { - if parsed, ok := decodeJSON(body); ok { + if parsed, ok := decodeJSON(body, sensitive); ok { return parsed } } if mediaType == "application/x-www-form-urlencoded" { - return previewForm(body) + return previewForm(body, sensitive) } if strings.HasPrefix(mediaType, "multipart/") { if boundary := params["boundary"]; boundary != "" { - if summary, ok := previewMultipart(body, boundary); ok { + if summary, ok := previewMultipart(body, boundary, sensitive); ok { return summary } } @@ -307,7 +391,7 @@ func previewBody(body []byte, contentType string) interface{} { return redactBase64String(string(body)) } -func previewForm(body []byte) string { +func previewForm(body []byte, sensitive [][]string) string { raw := string(body) if _, err := url.ParseQuery(raw); err != nil { return fmt.Sprintf("", len(body)) @@ -321,10 +405,18 @@ func previewForm(body []byte) string { parts[i] = key + "=[REDACTED]" continue } - if isSensitiveName(decodedKey) { + if isSensitiveName(decodedKey) || isSensitiveEncodedField(sensitive, decodedKey) { parts[i] = key + "=[REDACTED]" continue } + if nested := sensitiveBodySubpaths(sensitive, decodedKey); len(nested) > 0 { + if parsed, ok := decodeJSON([]byte(decodedValue), nested); ok { + if encoded, err := encodeJSON(parsed, ""); err == nil { + parts[i] = key + "=" + url.QueryEscape(encoded) + continue + } + } + } redacted := redactBase64String(decodedValue) if redacted != decodedValue { parts[i] = key + "=" + redacted @@ -335,7 +427,7 @@ func previewForm(body []byte) string { return strings.Join(parts, "&") } -func previewMultipart(body []byte, boundary string) (string, bool) { +func previewMultipart(body []byte, boundary string, sensitive [][]string) (string, bool) { r := multipart.NewReader(bytes.NewReader(body), boundary) var lines []string for { @@ -358,8 +450,8 @@ func previewMultipart(body []byte, boundary string) (string, bool) { lines = append(lines, fmt.Sprintf("%s: ", name, len(data))) continue } - value := previewBody(data, part.Header.Get("Content-Type")) - if isSensitiveName(name) { + value := previewBody(data, part.Header.Get("Content-Type"), sensitiveBodySubpaths(sensitive, name)) + if isSensitiveName(name) || isSensitiveEncodedField(sensitive, name) { value = "[REDACTED]" } lines = append(lines, fmt.Sprintf("%s: %s", name, formatBodyPreview(value, 0))) @@ -454,6 +546,8 @@ func redactURL(u string) string { type DebugClient struct { Inner HTTPClient Stderr io.Writer + // SensitiveBodyFields lists request-body paths to redact; see sensitiveBodyFields. + SensitiveBodyFields [][]string } // HTTPClient is the interface that SDK clients implement. @@ -472,7 +566,7 @@ func (c *DebugClient) Do(req *http.Request) (*http.Response, error) { if readErr != nil { fmt.Fprintf(c.Stderr, "[DEBUG] Request Body Read Error: %v\n", readErr) } else { - body := previewBody(bodyData, req.Header.Get("Content-Type")) + body := previewBody(bodyData, req.Header.Get("Content-Type"), c.SensitiveBodyFields) fmt.Fprintf(c.Stderr, "[DEBUG] Request Body:\n %s\n", formatBodyPreview(body, maxBodyPreview)) } } @@ -497,7 +591,7 @@ func (c *DebugClient) Do(req *http.Request) (*http.Response, error) { if readErr != nil { fmt.Fprintf(c.Stderr, "[DEBUG] Response Body Read Error: %v\n", readErr) } else if len(bodyData) > 0 { - body := previewBody(bodyData, resp.Header.Get("Content-Type")) + body := previewBody(bodyData, resp.Header.Get("Content-Type"), nil) fmt.Fprintf(c.Stderr, "[DEBUG] Response Body:\n %s\n", formatBodyPreview(body, maxBodyPreview)) } } @@ -526,6 +620,8 @@ type DryRunClient struct { Stdout io.Writer JSON bool Cmd *cobra.Command + // SensitiveBodyFields lists request-body paths to redact; see sensitiveBodyFields. + SensitiveBodyFields [][]string } type dryRunBody struct{ length int64 } @@ -562,7 +658,7 @@ func (c *DryRunClient) Do(req *http.Request) (*http.Response, error) { if err != nil { return nil, fmt.Errorf("read request body for dry-run preview: %w", err) } - body = previewBody(bodyData, req.Header.Get("Content-Type")) + body = previewBody(bodyData, req.Header.Get("Content-Type"), c.SensitiveBodyFields) } requestURL := redactURL(req.URL.String()) headers := redactHeaders(req.Header) @@ -701,10 +797,10 @@ func dryRunResponseShape(accept string) (string, []byte) { func WrapClientForDiagnostics(cmd *cobra.Command, inner HTTPClient) HTTPClient { stderr := cmd.ErrOrStderr() if IsDryRun(cmd) { - return &DryRunClient{Stderr: stderr, Stdout: cmd.OutOrStdout(), JSON: IsJSONDryRun(cmd), Cmd: cmd} + return &DryRunClient{Stderr: stderr, Stdout: cmd.OutOrStdout(), JSON: IsJSONDryRun(cmd), Cmd: cmd, SensitiveBodyFields: sensitiveBodyFieldsFor(cmd)} } if IsDebug(cmd) { - return &DebugClient{Inner: inner, Stderr: stderr} + return &DebugClient{Inner: inner, Stderr: stderr, SensitiveBodyFields: sensitiveBodyFieldsFor(cmd)} } return inner } diff --git a/templates/templates/cli/includes/security.ts b/templates/templates/cli/includes/security.ts index fcc491da3..bbfacec80 100644 --- a/templates/templates/cli/includes/security.ts +++ b/templates/templates/cli/includes/security.ts @@ -369,6 +369,106 @@ registerTemplateFunc( templateSensitiveSecurityHeaders, ); +function isSensitiveBodyType(type: TypeDef): boolean { + const marker = type.Extensions?.All?.["x-speakeasy-param-sensitive"]; + return marker === true || marker === "true" || type.Format === "password"; +} + +function requestBodyType(op: Operation): TypeDef | undefined { + if (!op.Request) return undefined; + if (op.Request.IsRequestBody) return op.Request.RequestBody?.Type; + return op.Request.Field?.Type?.Fields?.find( + (f: FieldDef) => f.Annotations?.Has("request"), + )?.Type; +} + +/** + * Request-body values each operation marks sensitive + * (x-speakeasy-param-sensitive or format: password), as Go literals of JSON + * paths keyed by operation id. A "*" segment stands for any array index or map + * key and "**" for any run of segments inside a recursive schema; union + * variants share their parent's path. Diagnostics redacts these in dry-run and + * debug request previews on top of its name-based list, so a secret under a + * generic name is hidden while same-named fields of other schemas stay + * readable. + */ +function templateSensitiveBodyFields(): { + OperationID: string; + Paths: string; +}[] { + const entries: { OperationID: string; Paths: string }[] = []; + for (const op of allOperations()) { + const paths = new Map(); + const add = (path: string[]): void => { + paths.set(JSON.stringify(path), path); + }; + // Components on the current walk, flagged once they recur below themselves. + const stack = new Map(); + const walk = (type: TypeDef | undefined, path: string[]): void => { + if (!type) return; + if (path.length > 0 && isSensitiveBodyType(type)) { + add(path); + return; + } + const component = type.IsComponent && type.Name ? type.Name : ""; + if (component) { + if (stack.has(component)) { + stack.set(component, true); + return; + } + stack.set(component, false); + } + const before = new Set(paths.keys()); + switch (type.Type.toString()) { + case "class": + for (const field of type.Fields || []) { + if (field.Const) continue; + walk( + field.Type, + field.IsAdditionalProperties + ? path + : [...path, originalFieldName(field)], + ); + } + break; + case "array": + case "set": + case "map": + walk(type.ItemType, [...path, "*"]); + break; + case "union": + for (const variant of type.AssociatedTypes || []) { + walk(variant, path); + } + break; + } + if (!component) return; + if (stack.get(component)) { + const found = [...paths].filter(([key]) => !before.has(key)); + for (const [, foundPath] of found) { + const rel = foundPath.slice(path.length); + add([...path, "**", ...(rel[0] === "**" ? rel.slice(1) : rel)]); + } + } + stack.delete(component); + }; + walk(requestBodyType(op), []); + if (paths.size === 0) continue; + entries.push({ + OperationID: op.OriginalID, + // Spaced braces: this file renders twice (recurse), so "{{" must not appear. + Paths: `{ ${[...paths.values()] + .map((p) => `{ ${p.map((s) => `"${escapeGoString(s)}"`).join(", ")} }`) + .join(", ")} }`, + }); + } + return entries.sort((a, b) => a.OperationID.localeCompare(b.OperationID)); +} +registerTemplateFunc( + "templateSensitiveBodyFields", + templateSensitiveBodyFields, +); + function primaryCLISecurityField( fields: CLISecurityFieldInfo[], ): CLISecurityFieldInfo | undefined { diff --git a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl index 886a5d923..76609bf97 100644 --- a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl +++ b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl @@ -339,6 +339,113 @@ func TestVersionAndWhoamiMachineOutput(t *testing.T) { assert.False(t, json.Valid([]byte(strings.TrimSpace(h.GetStdout()))), "pretty version output turned into JSON") } + +// diagnosticsOutputModes runs a command under each diagnostics mode that +// previews the request body and returns the stream holding that preview. +var diagnosticsOutputModes = []struct { + name string + run func(t *testing.T, args []string) string +}{ + {"dry-run", func(t *testing.T, args []string) string { + h := NewCLITestHarness(t) + require.NoError(t, h.RunWithServer("http://127.0.0.1:1", append([]string{"--dry-run", "--output-format", "pretty"}, args...))) + return h.GetStderr() + }}, + {"dry-run json", func(t *testing.T, args []string) string { + h := NewCLITestHarness(t) + require.NoError(t, h.RunWithServer("http://127.0.0.1:1", append([]string{"--dry-run"}, args...))) + return h.GetStdout() + }}, + {"debug", func(t *testing.T, args []string) string { + s := newDiagnosticsServer() + defer s.Close() + h := NewCLITestHarness(t) + require.NoError(t, h.RunWithServer(s.URL, append([]string{"--debug"}, args...))) + return h.GetStderr() + }}, +} + +func TestDiagnosticsRedactSensitiveBodyFields(t *testing.T) { + cases := []struct { + name string + args []string + secrets []string + visible []string + }{ + { + name: "marked fields at every depth", + args: []string{"request-bodies", "cli-sensitive-body-fields-post", "--body", + `{"name":"visible-name","value":"canary-top","passphrase":"canary-password",` + + `"nested":{"label":"visible-nested","value":"canary-nested"},` + + `"entries":[{"label":"visible-entry","value":"canary-entry"}],` + + `"byName":{"first":{"label":"visible-map","value":"canary-map"}},` + + `"codes":["canary-item"],"codesByName":{"visible-code":"canary-map-value"},` + + `"tree":{"label":"visible-root","value":"canary-root","child":{"label":"visible-child","value":"canary-child",` + + `"child":{"value":"canary-grandchild"},"branch":{"label":"visible-branch","value":"canary-branch",` + + `"next":{"value":"canary-next"}}}}}`}, + secrets: []string{"canary-top", "canary-password", "canary-nested", "canary-entry", "canary-map", + "canary-item", "canary-map-value", "canary-root", "canary-child", "canary-grandchild", + "canary-branch", "canary-next"}, + visible: []string{"visible-name", "visible-nested", "visible-entry", "visible-map", "visible-code", + "visible-root", "visible-child", "visible-branch"}, + }, + { + name: "union variant field", + args: []string{"request-bodies", "cli-sensitive-union-body-post", "--scope", "visible-scope", + "--cli-sensitive-secret", `{"kind":"static","id":"visible-id","value":"canary-static"}`}, + secrets: []string{"canary-static"}, + visible: []string{"visible-id"}, + }, + { + name: "expanded union variant flags", + args: []string{"request-bodies", "cli-sensitive-union-body-post", "--scope", "visible-scope", + "--cli-sensitive-secret.basic.id", "visible-id", "--cli-sensitive-secret.basic.username", "visible-user", + "--cli-sensitive-secret.basic.passphrase", "canary-basic"}, + secrets: []string{"canary-basic"}, + visible: []string{"visible-id", "visible-user"}, + }, + { + name: "multipart parts", + args: []string{"request-bodies", "cli-sensitive-multipart-post", "--name", "visible-name", "--value", "canary-top", + "--config-param", `{"label":"visible-label","value":"canary-config"}`, + "--codes", "canary-code-1", "--codes", "canary-code-2"}, + secrets: []string{"canary-top", "canary-config", "canary-code-1", "canary-code-2"}, + visible: []string{"visible-name", "visible-label"}, + }, + { + name: "form fields", + args: []string{"request-bodies", "cli-sensitive-form-put", "--name", "visible-name", "--value", "canary-top", + "--config-param", `{"label":"visible-label","value":"canary-config"}`, + "--codes", "canary-code-1", "--codes", "canary-code-2"}, + secrets: []string{"canary-top", "canary-config", "canary-code-1", "canary-code-2"}, + visible: []string{"visible-name", "visible-label"}, + }, + } + for _, mode := range diagnosticsOutputModes { + for _, tc := range cases { + t.Run(mode.name+"/"+tc.name, func(t *testing.T) { + output := mode.run(t, tc.args) + for _, secret := range tc.secrets { + assert.NotContains(t, output, secret) + } + for _, value := range tc.visible { + assert.Contains(t, output, value) + } + }) + } + } +} + +func TestDiagnosticsKeepUnmarkedFieldSharingSensitiveName(t *testing.T) { + for _, mode := range diagnosticsOutputModes { + t.Run(mode.name, func(t *testing.T) { + output := mode.run(t, []string{"request-bodies", "cli-plain-body-fields-post", + "--name", "visible-name", "--value", "visible-value"}) + assert.Contains(t, output, "visible-value") + assert.NotRegexp(t, `"value":\s*"\[REDACTED\]"`, output) + }) + } +} {{- if hasRootVersionFlag}} func TestRootVersionFlagMatchesVersionCommand(t *testing.T) { diff --git a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml new file mode 100644 index 000000000..bc19292b4 --- /dev/null +++ b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml @@ -0,0 +1,272 @@ +openapi: 3.1.0 +security: + - apiKeyAuth: [] + - oauth2: [] + - clientCredentials: [read, write] + - customSchemeAppId: [] + - basicHttp: [] + - accessToken: [] + - {} +paths: + /anything/cli/sensitiveBodyFields: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveBodyFieldsPost + description: >- + Request body whose secrets have generic names, marked sensitive with + x-speakeasy-param-sensitive or format: password at the top level, in + a nested object, in array items and map values (as object fields or as + the items and values themselves), and at every depth of nested + recursive schemas. Diagnostics output must redact each of them while + keeping their unmarked siblings readable. + tags: + - requestBodies + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CliSensitiveBodyFields" + responses: + "200": + description: OK + /anything/cli/sensitiveUnionBody/{scope}: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveUnionBodyPost + description: >- + Discriminated-union request body next to a path parameter, whose + variants mark a generically named field sensitive. Diagnostics output + must redact it whichever variant is sent. + tags: + - requestBodies + parameters: + - name: scope + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CliSensitiveSecret" + responses: + "200": + description: OK + /anything/cli/sensitiveEncodedFields: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveMultipartPost + description: >- + Multipart request body with a sensitive top-level part, a sensitive + array sent as repeated parts, and a JSON part that nests a sensitive + field. Diagnostics output must redact all of them. + tags: + - requestBodies + requestBody: + required: true + content: + multipart/form-data: + schema: + $ref: "#/components/schemas/CliSensitiveEncodedFields" + responses: + "200": + description: OK + put: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveFormPut + description: >- + Form-encoded request body with a sensitive top-level field, a sensitive + array sent as repeated fields, and a JSON-encoded field that nests a + sensitive field. Diagnostics output must redact all of them. + tags: + - requestBodies + requestBody: + required: true + content: + application/x-www-form-urlencoded: + schema: + $ref: "#/components/schemas/CliSensitiveEncodedFields" + encoding: + config: + contentType: application/json + responses: + "200": + description: OK + /anything/cli/plainBodyFields: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliPlainBodyFieldsPost + description: >- + Request body with a field that shares the name of a sensitive field in + another schema but is not marked sensitive. Diagnostics output must + keep it readable. + tags: + - requestBodies + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CliPlainBodyFields" + responses: + "200": + description: OK +components: + schemas: + CliSensitiveBodyFields: + type: object + required: + - name + - value + properties: + name: + type: string + value: + type: string + x-speakeasy-param-sensitive: true + passphrase: + type: string + format: password + nested: + type: object + properties: + label: + type: string + value: + type: string + x-speakeasy-param-sensitive: true + entries: + type: array + items: + type: object + properties: + label: + type: string + value: + type: string + format: password + byName: + type: object + additionalProperties: + type: object + properties: + label: + type: string + value: + type: string + x-speakeasy-param-sensitive: true + codes: + type: array + items: + type: string + format: password + codesByName: + type: object + additionalProperties: + type: string + x-speakeasy-param-sensitive: true + tree: + $ref: "#/components/schemas/CliSensitiveNode" + CliSensitiveNode: + type: object + properties: + label: + type: string + value: + type: string + x-speakeasy-param-sensitive: true + child: + $ref: "#/components/schemas/CliSensitiveNode" + branch: + $ref: "#/components/schemas/CliSensitiveBranch" + CliSensitiveBranch: + type: object + properties: + label: + type: string + value: + type: string + format: password + next: + $ref: "#/components/schemas/CliSensitiveBranch" + CliSensitiveEncodedFields: + type: object + required: + - name + - value + properties: + name: + type: string + value: + type: string + x-speakeasy-param-sensitive: true + config: + type: object + properties: + label: + type: string + value: + type: string + format: password + codes: + type: array + items: + type: string + format: password + CliSensitiveSecret: + oneOf: + - $ref: "#/components/schemas/CliSensitiveStaticSecret" + - $ref: "#/components/schemas/CliSensitiveBasicSecret" + discriminator: + propertyName: kind + mapping: + static: "#/components/schemas/CliSensitiveStaticSecret" + basic: "#/components/schemas/CliSensitiveBasicSecret" + CliSensitiveStaticSecret: + type: object + required: + - kind + - id + - value + properties: + kind: + type: string + const: static + id: + type: string + value: + type: string + writeOnly: true + x-speakeasy-param-sensitive: true + CliSensitiveBasicSecret: + type: object + required: + - kind + - id + - username + - passphrase + properties: + kind: + type: string + const: basic + id: + type: string + username: + type: string + passphrase: + type: string + writeOnly: true + format: password + CliPlainBodyFields: + type: object + required: + - name + - value + properties: + name: + type: string + value: + type: string diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index 949102087..bc01fae73 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -222,7 +222,7 @@ trackedFiles: internal/client/client.go: last_write_checksum: sha1:bca25435230238b7194798699e137ef94d983844 internal/client/diagnostics.go: - last_write_checksum: sha1:36245f95381109b1b8e1b0a041acd7b89737cd2e + last_write_checksum: sha1:627dbba858530ec9190853b44162674e3b1a506c internal/clierrors/clierrors.go: last_write_checksum: sha1:e17fb3e33e439c6997834182011540427d760857 internal/config/config.go: diff --git a/zSDKs/sdk-cli/internal/client/diagnostics.go b/zSDKs/sdk-cli/internal/client/diagnostics.go index 1c44a0921..e638d3003 100644 --- a/zSDKs/sdk-cli/internal/client/diagnostics.go +++ b/zSDKs/sdk-cli/internal/client/diagnostics.go @@ -83,6 +83,12 @@ var sensitiveJSONKeys = map[string]bool{ "clientsecret": true, } +// sensitiveBodyFields lists, per operation id, the request-body fields the API +// marks sensitive (x-speakeasy-param-sensitive or format: password) as JSON +// paths, where "*" stands for any array index or map key and "**" for any run +// of segments inside a recursive schema. +var sensitiveBodyFields = map[string][][]string{} + var sensitiveQueryKeys = map[string]bool{ "apikey": true, "accesstoken": true, @@ -160,8 +166,73 @@ func isSensitiveName(key string) bool { return false } -// redactJSON recursively redacts sensitive keys in a JSON structure. -func redactJSON(v interface{}, depth int) interface{} { +// sensitiveBodyFieldsFor returns the sensitive request-body paths of the +// operation that cmd (an operation or intent command) invokes. +func sensitiveBodyFieldsFor(cmd *cobra.Command) [][]string { + if cmd == nil { + return nil + } + return sensitiveBodyFields[cmd.Annotations["speakeasy_operation"]] +} + +func isSensitiveBodyPath(sensitive [][]string, path []string) bool { + for _, candidate := range sensitive { + if matchesBodyPath(candidate, path) { + return true + } + } + return false +} + +func matchesBodyPath(pattern, path []string) bool { + if len(pattern) == 0 { + return len(path) == 0 + } + if pattern[0] == "**" { + for i := 0; i <= len(path); i++ { + if matchesBodyPath(pattern[1:], path[i:]) { + return true + } + } + return false + } + if len(path) == 0 || (pattern[0] != "*" && pattern[0] != path[0]) { + return false + } + return matchesBodyPath(pattern[1:], path[1:]) +} + +// isSensitiveEncodedField reports whether a form field or multipart part is +// sensitive; repeated fields carry the items of an array. +func isSensitiveEncodedField(sensitive [][]string, name string) bool { + return isSensitiveBodyPath(sensitive, []string{name}) || isSensitiveBodyPath(sensitive, []string{name, "*"}) +} + +// sensitiveBodySubpaths returns the sensitive paths below the top-level field +// name, relative to it, for fields sent as an encoded JSON value (a multipart +// part or a form field). +func sensitiveBodySubpaths(sensitive [][]string, name string) [][]string { + var out [][]string + var collect func(pattern []string) + collect = func(pattern []string) { + switch { + case len(pattern) == 0: + case pattern[0] == "**": + out = append(out, pattern) + collect(pattern[1:]) + case (pattern[0] == "*" || pattern[0] == name) && len(pattern) > 1: + out = append(out, pattern[1:]) + } + } + for _, pattern := range sensitive { + collect(pattern) + } + return out +} + +// redactJSON recursively redacts sensitive keys in a JSON structure, plus the +// values at the sensitive request-body paths. +func redactJSON(v interface{}, depth int, path []string, sensitive [][]string) interface{} { if depth > maxRedactDepth { return redactionDepthMarker } @@ -169,17 +240,24 @@ func redactJSON(v interface{}, depth int) interface{} { case map[string]interface{}: out := make(map[string]interface{}, len(val)) for k, child := range val { - if sensitiveJSONKeys[normalizeSensitiveKey(k)] { + childPath := append(path[:len(path):len(path)], k) + if sensitiveJSONKeys[normalizeSensitiveKey(k)] || isSensitiveBodyPath(sensitive, childPath) { out[k] = "[REDACTED]" } else { - out[k] = redactJSON(child, depth+1) + out[k] = redactJSON(child, depth+1, childPath, sensitive) } } return out case []interface{}: out := make([]interface{}, len(val)) + childPath := append(path[:len(path):len(path)], "*") + redactItems := isSensitiveBodyPath(sensitive, childPath) for i, child := range val { - out[i] = redactJSON(child, depth+1) + if redactItems { + out[i] = "[REDACTED]" + } else { + out[i] = redactJSON(child, depth+1, childPath, sensitive) + } } return out case string: @@ -255,7 +333,7 @@ func isAllHex(value string) bool { return true } -func decodeJSON(body []byte) (interface{}, bool) { +func decodeJSON(body []byte, sensitive [][]string) (interface{}, bool) { dec := json.NewDecoder(bytes.NewReader(body)) dec.UseNumber() var parsed interface{} @@ -265,7 +343,7 @@ func decodeJSON(body []byte) (interface{}, bool) { if err := dec.Decode(&struct{}{}); err != io.EOF { return nil, false } - return redactJSON(parsed, 0), true + return redactJSON(parsed, 0, nil, sensitive), true } func isTextMediaType(mediaType string) bool { @@ -276,7 +354,9 @@ func isTextMediaType(mediaType string) bool { mediaType == "application/x-www-form-urlencoded" } -func previewBody(body []byte, contentType string) interface{} { +// previewBody renders a body for diagnostics; sensitive holds the request-body +// paths to redact (nil for response bodies). +func previewBody(body []byte, contentType string, sensitive [][]string) interface{} { if len(body) == 0 { return nil } @@ -285,16 +365,16 @@ func previewBody(body []byte, contentType string) interface{} { mediaType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) } if isJSONMediaType(mediaType) || mediaType == "" { - if parsed, ok := decodeJSON(body); ok { + if parsed, ok := decodeJSON(body, sensitive); ok { return parsed } } if mediaType == "application/x-www-form-urlencoded" { - return previewForm(body) + return previewForm(body, sensitive) } if strings.HasPrefix(mediaType, "multipart/") { if boundary := params["boundary"]; boundary != "" { - if summary, ok := previewMultipart(body, boundary); ok { + if summary, ok := previewMultipart(body, boundary, sensitive); ok { return summary } } @@ -306,7 +386,7 @@ func previewBody(body []byte, contentType string) interface{} { return redactBase64String(string(body)) } -func previewForm(body []byte) string { +func previewForm(body []byte, sensitive [][]string) string { raw := string(body) if _, err := url.ParseQuery(raw); err != nil { return fmt.Sprintf("", len(body)) @@ -320,10 +400,18 @@ func previewForm(body []byte) string { parts[i] = key + "=[REDACTED]" continue } - if isSensitiveName(decodedKey) { + if isSensitiveName(decodedKey) || isSensitiveEncodedField(sensitive, decodedKey) { parts[i] = key + "=[REDACTED]" continue } + if nested := sensitiveBodySubpaths(sensitive, decodedKey); len(nested) > 0 { + if parsed, ok := decodeJSON([]byte(decodedValue), nested); ok { + if encoded, err := encodeJSON(parsed, ""); err == nil { + parts[i] = key + "=" + url.QueryEscape(encoded) + continue + } + } + } redacted := redactBase64String(decodedValue) if redacted != decodedValue { parts[i] = key + "=" + redacted @@ -334,7 +422,7 @@ func previewForm(body []byte) string { return strings.Join(parts, "&") } -func previewMultipart(body []byte, boundary string) (string, bool) { +func previewMultipart(body []byte, boundary string, sensitive [][]string) (string, bool) { r := multipart.NewReader(bytes.NewReader(body), boundary) var lines []string for { @@ -357,8 +445,8 @@ func previewMultipart(body []byte, boundary string) (string, bool) { lines = append(lines, fmt.Sprintf("%s: ", name, len(data))) continue } - value := previewBody(data, part.Header.Get("Content-Type")) - if isSensitiveName(name) { + value := previewBody(data, part.Header.Get("Content-Type"), sensitiveBodySubpaths(sensitive, name)) + if isSensitiveName(name) || isSensitiveEncodedField(sensitive, name) { value = "[REDACTED]" } lines = append(lines, fmt.Sprintf("%s: %s", name, formatBodyPreview(value, 0))) @@ -453,6 +541,8 @@ func redactURL(u string) string { type DebugClient struct { Inner HTTPClient Stderr io.Writer + // SensitiveBodyFields lists request-body paths to redact; see sensitiveBodyFields. + SensitiveBodyFields [][]string } // HTTPClient is the interface that SDK clients implement. @@ -471,7 +561,7 @@ func (c *DebugClient) Do(req *http.Request) (*http.Response, error) { if readErr != nil { fmt.Fprintf(c.Stderr, "[DEBUG] Request Body Read Error: %v\n", readErr) } else { - body := previewBody(bodyData, req.Header.Get("Content-Type")) + body := previewBody(bodyData, req.Header.Get("Content-Type"), c.SensitiveBodyFields) fmt.Fprintf(c.Stderr, "[DEBUG] Request Body:\n %s\n", formatBodyPreview(body, maxBodyPreview)) } } @@ -496,7 +586,7 @@ func (c *DebugClient) Do(req *http.Request) (*http.Response, error) { if readErr != nil { fmt.Fprintf(c.Stderr, "[DEBUG] Response Body Read Error: %v\n", readErr) } else if len(bodyData) > 0 { - body := previewBody(bodyData, resp.Header.Get("Content-Type")) + body := previewBody(bodyData, resp.Header.Get("Content-Type"), nil) fmt.Fprintf(c.Stderr, "[DEBUG] Response Body:\n %s\n", formatBodyPreview(body, maxBodyPreview)) } } @@ -525,6 +615,8 @@ type DryRunClient struct { Stdout io.Writer JSON bool Cmd *cobra.Command + // SensitiveBodyFields lists request-body paths to redact; see sensitiveBodyFields. + SensitiveBodyFields [][]string } type dryRunBody struct{ length int64 } @@ -561,7 +653,7 @@ func (c *DryRunClient) Do(req *http.Request) (*http.Response, error) { if err != nil { return nil, fmt.Errorf("read request body for dry-run preview: %w", err) } - body = previewBody(bodyData, req.Header.Get("Content-Type")) + body = previewBody(bodyData, req.Header.Get("Content-Type"), c.SensitiveBodyFields) } requestURL := redactURL(req.URL.String()) headers := redactHeaders(req.Header) @@ -700,10 +792,10 @@ func dryRunResponseShape(accept string) (string, []byte) { func WrapClientForDiagnostics(cmd *cobra.Command, inner HTTPClient) HTTPClient { stderr := cmd.ErrOrStderr() if IsDryRun(cmd) { - return &DryRunClient{Stderr: stderr, Stdout: cmd.OutOrStdout(), JSON: IsJSONDryRun(cmd), Cmd: cmd} + return &DryRunClient{Stderr: stderr, Stdout: cmd.OutOrStdout(), JSON: IsJSONDryRun(cmd), Cmd: cmd, SensitiveBodyFields: sensitiveBodyFieldsFor(cmd)} } if IsDebug(cmd) { - return &DebugClient{Inner: inner, Stderr: stderr} + return &DebugClient{Inner: inner, Stderr: stderr, SensitiveBodyFields: sensitiveBodyFieldsFor(cmd)} } return inner } From 67bb3a5066683dd47a31c7a53dbf28a75b5ed6ce Mon Sep 17 00:00:00 2001 From: 2ynn Date: Wed, 7 Oct 2026 11:44:37 -0400 Subject: [PATCH 15/23] fix(cli): merge sensitive paths for split request content types --- templates/templates/cli/includes/security.ts | 10 +++-- .../tests/primary/diagnostics_test.go.stmpl | 18 ++++++++ .../primary/cli-sensitive-body-fields.yaml | 42 +++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/templates/templates/cli/includes/security.ts b/templates/templates/cli/includes/security.ts index bbfacec80..f51889af9 100644 --- a/templates/templates/cli/includes/security.ts +++ b/templates/templates/cli/includes/security.ts @@ -396,9 +396,10 @@ function templateSensitiveBodyFields(): { OperationID: string; Paths: string; }[] { - const entries: { OperationID: string; Paths: string }[] = []; + const operations = new Map>(); for (const op of allOperations()) { - const paths = new Map(); + const paths = operations.get(op.OriginalID) || new Map(); + operations.set(op.OriginalID, paths); const add = (path: string[]): void => { paths.set(JSON.stringify(path), path); }; @@ -453,9 +454,12 @@ function templateSensitiveBodyFields(): { stack.delete(component); }; walk(requestBodyType(op), []); + } + const entries: { OperationID: string; Paths: string }[] = []; + for (const [operationID, paths] of operations) { if (paths.size === 0) continue; entries.push({ - OperationID: op.OriginalID, + OperationID: operationID, // Spaced braces: this file renders twice (recurse), so "{{" must not appear. Paths: `{ ${[...paths.values()] .map((p) => `{ ${p.map((s) => `"${escapeGoString(s)}"`).join(", ")} }`) diff --git a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl index 76609bf97..8f2549fef 100644 --- a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl +++ b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl @@ -372,6 +372,24 @@ func TestDiagnosticsRedactSensitiveBodyFields(t *testing.T) { secrets []string visible []string }{ + { + name: "split json body", + args: []string{"request-bodies", "cli-sensitive-split-body-post-json", "--json-value", "canary-json", "--name", "visible-name"}, + secrets: []string{"canary-json"}, + visible: []string{"visible-name"}, + }, + { + name: "split multipart body", + args: []string{"request-bodies", "cli-sensitive-split-body-post-multipart", "--multipart-value", "canary-multipart", "--name", "visible-name"}, + secrets: []string{"canary-multipart"}, + visible: []string{"visible-name"}, + }, + { + name: "split form body", + args: []string{"request-bodies", "cli-sensitive-split-body-post-form", "--form-value", "canary-form", "--name", "visible-name"}, + secrets: []string{"canary-form"}, + visible: []string{"visible-name"}, + }, { name: "marked fields at every depth", args: []string{"request-bodies", "cli-sensitive-body-fields-post", "--body", diff --git a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml index bc19292b4..0c3cb74a1 100644 --- a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml +++ b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml @@ -114,6 +114,48 @@ paths: responses: "200": description: OK + /anything/cli/sensitiveSplitBody: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveSplitBodyPost + description: Distinct request content types each contain a sensitive value. + tags: [requestBodies] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [jsonValue] + properties: + jsonValue: + type: string + x-speakeasy-param-sensitive: true + name: + type: string + multipart/form-data: + schema: + type: object + required: [multipartValue] + properties: + multipartValue: + type: string + x-speakeasy-param-sensitive: true + name: + type: string + application/x-www-form-urlencoded: + schema: + type: object + required: [formValue] + properties: + formValue: + type: string + x-speakeasy-param-sensitive: true + name: + type: string + responses: + "200": + description: OK components: schemas: CliSensitiveBodyFields: From 12e958d76a019d63212c4b4888116c3e83581b79 Mon Sep 17 00:00:00 2001 From: 2ynn Date: Wed, 7 Oct 2026 11:48:20 -0400 Subject: [PATCH 16/23] fix(cli): redact request bodies marked sensitive at the root --- .../internal/client/diagnostics.go.stmpl | 3 + templates/templates/cli/includes/security.ts | 2 +- .../tests/primary/diagnostics_test.go.stmpl | 24 +++++++ .../primary/cli-sensitive-body-fields.yaml | 70 +++++++++++++++++++ zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/client/diagnostics.go | 3 + 6 files changed, 102 insertions(+), 2 deletions(-) diff --git a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl index ab50bf6d3..9adfe535c 100644 --- a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl +++ b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl @@ -365,6 +365,9 @@ func previewBody(body []byte, contentType string, sensitive [][]string) interfac if len(body) == 0 { return nil } + if isSensitiveBodyPath(sensitive, nil) { + return "[REDACTED]" + } mediaType, params, err := mime.ParseMediaType(contentType) if err != nil { mediaType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) diff --git a/templates/templates/cli/includes/security.ts b/templates/templates/cli/includes/security.ts index f51889af9..9e02829a3 100644 --- a/templates/templates/cli/includes/security.ts +++ b/templates/templates/cli/includes/security.ts @@ -407,7 +407,7 @@ function templateSensitiveBodyFields(): { const stack = new Map(); const walk = (type: TypeDef | undefined, path: string[]): void => { if (!type) return; - if (path.length > 0 && isSensitiveBodyType(type)) { + if (isSensitiveBodyType(type)) { add(path); return; } diff --git a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl index 8f2549fef..a34213149 100644 --- a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl +++ b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl @@ -390,6 +390,30 @@ func TestDiagnosticsRedactSensitiveBodyFields(t *testing.T) { secrets: []string{"canary-form"}, visible: []string{"visible-name"}, }, + { + name: "whole json body", + args: []string{"request-bodies", "cli-sensitive-whole-json-post", "--value", "canary-whole-json"}, + secrets: []string{"canary-whole-json"}, + visible: []string{"[REDACTED]"}, + }, + { + name: "whole form body", + args: []string{"request-bodies", "cli-sensitive-whole-form-post", "--value", "canary-whole-form"}, + secrets: []string{"canary-whole-form"}, + visible: []string{"[REDACTED]"}, + }, + { + name: "whole multipart body", + args: []string{"request-bodies", "cli-sensitive-whole-multipart-post", "--value", "canary-whole-multipart"}, + secrets: []string{"canary-whole-multipart"}, + visible: []string{"[REDACTED]"}, + }, + { + name: "whole text body", + args: []string{"request-bodies", "cli-sensitive-whole-text-post", "--request", `"canary-whole-text"`}, + secrets: []string{"canary-whole-text"}, + visible: []string{"[REDACTED]"}, + }, { name: "marked fields at every depth", args: []string{"request-bodies", "cli-sensitive-body-fields-post", "--body", diff --git a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml index 0c3cb74a1..857c53d52 100644 --- a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml +++ b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml @@ -156,6 +156,66 @@ paths: responses: "200": description: OK + /anything/cli/sensitiveWholeJson: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveWholeJsonPost + description: The entire request body is marked sensitive on its schema. + tags: [requestBodies] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CliSensitiveWholeBody" + responses: + "200": + description: OK + /anything/cli/sensitiveWholeForm: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveWholeFormPost + description: The entire request body is marked sensitive on its schema. + tags: [requestBodies] + requestBody: + required: true + content: + application/x-www-form-urlencoded: + schema: + $ref: "#/components/schemas/CliSensitiveWholeBody" + responses: + "200": + description: OK + /anything/cli/sensitiveWholeMultipart: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveWholeMultipartPost + description: The entire request body is marked sensitive on its schema. + tags: [requestBodies] + requestBody: + required: true + content: + multipart/form-data: + schema: + $ref: "#/components/schemas/CliSensitiveWholeBody" + responses: + "200": + description: OK + /anything/cli/sensitiveWholeText: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveWholeTextPost + description: The entire request body is marked sensitive on its schema. + tags: [requestBodies] + requestBody: + required: true + content: + text/plain: + schema: + $ref: "#/components/schemas/CliSensitiveWholeValue" + responses: + "200": + description: OK components: schemas: CliSensitiveBodyFields: @@ -312,3 +372,13 @@ components: type: string value: type: string + CliSensitiveWholeBody: + type: object + x-speakeasy-param-sensitive: true + required: [value] + properties: + value: + type: string + CliSensitiveWholeValue: + type: string + format: password diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index bc01fae73..495055fc1 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -222,7 +222,7 @@ trackedFiles: internal/client/client.go: last_write_checksum: sha1:bca25435230238b7194798699e137ef94d983844 internal/client/diagnostics.go: - last_write_checksum: sha1:627dbba858530ec9190853b44162674e3b1a506c + last_write_checksum: sha1:7e2f2af20793241ffbc7d977de1e921736b4d194 internal/clierrors/clierrors.go: last_write_checksum: sha1:e17fb3e33e439c6997834182011540427d760857 internal/config/config.go: diff --git a/zSDKs/sdk-cli/internal/client/diagnostics.go b/zSDKs/sdk-cli/internal/client/diagnostics.go index e638d3003..37486700b 100644 --- a/zSDKs/sdk-cli/internal/client/diagnostics.go +++ b/zSDKs/sdk-cli/internal/client/diagnostics.go @@ -360,6 +360,9 @@ func previewBody(body []byte, contentType string, sensitive [][]string) interfac if len(body) == 0 { return nil } + if isSensitiveBodyPath(sensitive, nil) { + return "[REDACTED]" + } mediaType, params, err := mime.ParseMediaType(contentType) if err != nil { mediaType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) From 6b3d5da31df3995523a0a9cc873eafc6c490edb2 Mon Sep 17 00:00:00 2001 From: 2ynn Date: Wed, 7 Oct 2026 11:57:48 -0400 Subject: [PATCH 17/23] fix(cli): follow schema references when redacting recursive bodies --- .changesets/1790872792-577bf8cd.yaml | 2 +- templates/templates/cli/README.md | 2 +- .../internal/client/diagnostics.go.stmpl | 158 ++++++++-------- templates/templates/cli/includes/security.ts | 162 ++++++++++------- .../tests/primary/diagnostics_test.go.stmpl | 64 ++++++- .../primary/cli-sensitive-body-fields.yaml | 93 ++++++++++ zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/client/diagnostics.go | 168 +++++++++--------- 8 files changed, 416 insertions(+), 235 deletions(-) diff --git a/.changesets/1790872792-577bf8cd.yaml b/.changesets/1790872792-577bf8cd.yaml index dee934c90..2e76350e2 100644 --- a/.changesets/1790872792-577bf8cd.yaml +++ b/.changesets/1790872792-577bf8cd.yaml @@ -5,6 +5,6 @@ targets: - cli type: fix bump: patch -description: '--dry-run and --debug request previews redact request-body values whose schema is marked x-speakeasy-param-sensitive or format: password, in JSON bodies, form fields and multipart parts (including JSON-encoded ones), so secrets under generic names are no longer printed; same-named unmarked fields of other schemas stay readable' +description: '--dry-run and --debug redact sensitive request-body values and whole bodies across JSON, form, multipart and text previews, including recursive schemas and operations with multiple request content types; unrelated unmarked fields remain readable' author: 2ynn date: "2026-10-01" diff --git a/templates/templates/cli/README.md b/templates/templates/cli/README.md index 22d7742ce..c315cfe4f 100644 --- a/templates/templates/cli/README.md +++ b/templates/templates/cli/README.md @@ -1491,7 +1491,7 @@ cli users list --debug - **Headers**: Case-insensitive denylist (`authorization`, `cookie`, `set-cookie`, `x-api-key`, `x-session-token`, `*-secret`, `*-token` suffixes) → `[REDACTED]` - **URL query**: Credential parameters (`api_key`, `token`, `client_secret`, `signature`, etc.) are redacted without reordering the query -- **Request body values marked sensitive**: `templateSensitiveBodyFields` (`includes/security.ts`) collects, per operation, the JSON paths of request-body values whose schema sets `x-speakeasy-param-sensitive: true` or `format: password` (object fields, array items and map values as `*`, union variants at their parent's path, and `**` for any depth of a recursive schema) into `sensitiveBodyFields`. Dry-run and debug request previews redact those values, keyed by the command's `speakeasy_operation` annotation, so a secret under a generic name (`value`) is hidden while a same-named field of another schema stays readable. Form fields and multipart parts match on their names (repeated ones as array items), and JSON-encoded ones on the paths inside them; a form field that holds JSON is shown re-encoded (compact, keys sorted) when a sensitive path may lie inside it. Response bodies use the name-based rules only. The marker must sit on the value's own schema: an extension next to a `$ref` is not read +- **Request body values marked sensitive**: `templateSensitiveBodyFields` (`includes/security.ts`) builds a schema graph per operation for values marked `x-speakeasy-param-sensitive: true` or `format: password`, including whole bodies. Graph edges follow object fields, array items, map values, union variants, and recursive component references. Dry-run and debug request previews select the graph through the command's `speakeasy_operation` annotation; split request content types contribute roots to the same graph. Unmarked fields in unrelated descendant schemas stay readable. Form fields and multipart parts are matched by name, with JSON-encoded fields traversed from that field's schema and repeated scalar fields treated as array items. Response bodies use the name-based rules only. The marker must sit on the value's own schema: an extension next to a `$ref` is not read - **JSON body**: Separator/case-normalized sensitive keys and canonical base64 strings (encoded length ≥128, excluding all-hex IDs) are redacted; numbers retain their exact lexemes and depth overflow becomes an explicit marker - **Other bodies**: Binary media becomes `` and multipart payloads become a deterministic inventory; human dry-run is not truncated. Debug applies the same redactor, then caps the rendered preview at 4 KiB - **Credentials**: Generated request construction uses request-scoped resolvers. Dry-run reads flags, environment and config, but never probes the OS keychain; whoami also uses the request-scoped resolvers (so `whoami --dry-run` skips the keychain), while configuration retains keychain access diff --git a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl index 9adfe535c..32ae84b2b 100644 --- a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl +++ b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl @@ -84,13 +84,21 @@ var sensitiveJSONKeys = map[string]bool{ "clientsecret": true, } -// sensitiveBodyFields lists, per operation id, the request-body fields the API -// marks sensitive (x-speakeasy-param-sensitive or format: password) as JSON -// paths, where "*" stands for any array index or map key and "**" for any run -// of segments inside a recursive schema. -var sensitiveBodyFields = map[string][][]string{ +type sensitiveBodyNode struct { + Sensitive bool + Fields map[string]int + Item int + Variants []int +} + +type sensitiveBodySchema struct { + Nodes []sensitiveBodyNode + Roots []int +} + +var sensitiveBodyFields = map[string]sensitiveBodySchema{ {{- range templateSensitiveBodyFields}} - "{{escapeGoString .OperationID}}": {{.Paths}}, + "{{escapeGoString .OperationID}}": {{.Schema}}, {{- end}} } @@ -171,98 +179,94 @@ func isSensitiveName(key string) bool { return false } -// sensitiveBodyFieldsFor returns the sensitive request-body paths of the -// operation that cmd (an operation or intent command) invokes. -func sensitiveBodyFieldsFor(cmd *cobra.Command) [][]string { +func sensitiveBodyFieldsFor(cmd *cobra.Command) sensitiveBodySchema { if cmd == nil { - return nil + return sensitiveBodySchema{} } return sensitiveBodyFields[cmd.Annotations["speakeasy_operation"]] } -func isSensitiveBodyPath(sensitive [][]string, path []string) bool { - for _, candidate := range sensitive { - if matchesBodyPath(candidate, path) { - return true +func (s sensitiveBodySchema) expandedRoots() []int { + var roots []int + seen := make(map[int]bool) + var visit func(int) + visit = func(id int) { + if id <= 0 || id >= len(s.Nodes) || seen[id] { + return + } + seen[id] = true + roots = append(roots, id) + for _, variant := range s.Nodes[id].Variants { + visit(variant) } } - return false + for _, root := range s.Roots { + visit(root) + } + return roots } -func matchesBodyPath(pattern, path []string) bool { - if len(pattern) == 0 { - return len(path) == 0 - } - if pattern[0] == "**" { - for i := 0; i <= len(path); i++ { - if matchesBodyPath(pattern[1:], path[i:]) { - return true - } +func (s sensitiveBodySchema) isSensitive() bool { + for _, id := range s.expandedRoots() { + if s.Nodes[id].Sensitive { + return true } - return false } - if len(path) == 0 || (pattern[0] != "*" && pattern[0] != path[0]) { - return false - } - return matchesBodyPath(pattern[1:], path[1:]) + return false } -// isSensitiveEncodedField reports whether a form field or multipart part is -// sensitive; repeated fields carry the items of an array. -func isSensitiveEncodedField(sensitive [][]string, name string) bool { - return isSensitiveBodyPath(sensitive, []string{name}) || isSensitiveBodyPath(sensitive, []string{name, "*"}) +func (s sensitiveBodySchema) child(name string) sensitiveBodySchema { + var roots []int + for _, id := range s.expandedRoots() { + node := s.Nodes[id] + if child := node.Fields[name]; child != 0 { + roots = append(roots, child) + } + if node.Item != 0 { + roots = append(roots, node.Item) + } + } + return sensitiveBodySchema{Nodes: s.Nodes, Roots: roots} } -// sensitiveBodySubpaths returns the sensitive paths below the top-level field -// name, relative to it, for fields sent as an encoded JSON value (a multipart -// part or a form field). -func sensitiveBodySubpaths(sensitive [][]string, name string) [][]string { - var out [][]string - var collect func(pattern []string) - collect = func(pattern []string) { - switch { - case len(pattern) == 0: - case pattern[0] == "**": - out = append(out, pattern) - collect(pattern[1:]) - case (pattern[0] == "*" || pattern[0] == name) && len(pattern) > 1: - out = append(out, pattern[1:]) +func (s sensitiveBodySchema) items() sensitiveBodySchema { + var roots []int + for _, id := range s.expandedRoots() { + if item := s.Nodes[id].Item; item != 0 { + roots = append(roots, item) } } - for _, pattern := range sensitive { - collect(pattern) - } - return out + return sensitiveBodySchema{Nodes: s.Nodes, Roots: roots} +} + +func isSensitiveEncodedField(sensitive sensitiveBodySchema, name string) bool { + nested := sensitive.child(name) + return nested.isSensitive() || nested.items().isSensitive() } -// redactJSON recursively redacts sensitive keys in a JSON structure, plus the -// values at the sensitive request-body paths. -func redactJSON(v interface{}, depth int, path []string, sensitive [][]string) interface{} { +func redactJSON(v interface{}, depth int, sensitive sensitiveBodySchema) interface{} { if depth > maxRedactDepth { return redactionDepthMarker } + if sensitive.isSensitive() { + return "[REDACTED]" + } switch val := v.(type) { case map[string]interface{}: out := make(map[string]interface{}, len(val)) for k, child := range val { - childPath := append(path[:len(path):len(path)], k) - if sensitiveJSONKeys[normalizeSensitiveKey(k)] || isSensitiveBodyPath(sensitive, childPath) { + if sensitiveJSONKeys[normalizeSensitiveKey(k)] { out[k] = "[REDACTED]" } else { - out[k] = redactJSON(child, depth+1, childPath, sensitive) + out[k] = redactJSON(child, depth+1, sensitive.child(k)) } } return out case []interface{}: out := make([]interface{}, len(val)) - childPath := append(path[:len(path):len(path)], "*") - redactItems := isSensitiveBodyPath(sensitive, childPath) + items := sensitive.items() for i, child := range val { - if redactItems { - out[i] = "[REDACTED]" - } else { - out[i] = redactJSON(child, depth+1, childPath, sensitive) - } + out[i] = redactJSON(child, depth+1, items) } return out case string: @@ -338,7 +342,7 @@ func isAllHex(value string) bool { return true } -func decodeJSON(body []byte, sensitive [][]string) (interface{}, bool) { +func decodeJSON(body []byte, sensitive sensitiveBodySchema) (interface{}, bool) { dec := json.NewDecoder(bytes.NewReader(body)) dec.UseNumber() var parsed interface{} @@ -348,7 +352,7 @@ func decodeJSON(body []byte, sensitive [][]string) (interface{}, bool) { if err := dec.Decode(&struct{}{}); err != io.EOF { return nil, false } - return redactJSON(parsed, 0, nil, sensitive), true + return redactJSON(parsed, 0, sensitive), true } func isTextMediaType(mediaType string) bool { @@ -359,13 +363,11 @@ func isTextMediaType(mediaType string) bool { mediaType == "application/x-www-form-urlencoded" } -// previewBody renders a body for diagnostics; sensitive holds the request-body -// paths to redact (nil for response bodies). -func previewBody(body []byte, contentType string, sensitive [][]string) interface{} { +func previewBody(body []byte, contentType string, sensitive sensitiveBodySchema) interface{} { if len(body) == 0 { return nil } - if isSensitiveBodyPath(sensitive, nil) { + if sensitive.isSensitive() { return "[REDACTED]" } mediaType, params, err := mime.ParseMediaType(contentType) @@ -394,7 +396,7 @@ func previewBody(body []byte, contentType string, sensitive [][]string) interfac return redactBase64String(string(body)) } -func previewForm(body []byte, sensitive [][]string) string { +func previewForm(body []byte, sensitive sensitiveBodySchema) string { raw := string(body) if _, err := url.ParseQuery(raw); err != nil { return fmt.Sprintf("", len(body)) @@ -412,7 +414,7 @@ func previewForm(body []byte, sensitive [][]string) string { parts[i] = key + "=[REDACTED]" continue } - if nested := sensitiveBodySubpaths(sensitive, decodedKey); len(nested) > 0 { + if nested := sensitive.child(decodedKey); len(nested.Roots) > 0 { if parsed, ok := decodeJSON([]byte(decodedValue), nested); ok { if encoded, err := encodeJSON(parsed, ""); err == nil { parts[i] = key + "=" + url.QueryEscape(encoded) @@ -430,7 +432,7 @@ func previewForm(body []byte, sensitive [][]string) string { return strings.Join(parts, "&") } -func previewMultipart(body []byte, boundary string, sensitive [][]string) (string, bool) { +func previewMultipart(body []byte, boundary string, sensitive sensitiveBodySchema) (string, bool) { r := multipart.NewReader(bytes.NewReader(body), boundary) var lines []string for { @@ -453,7 +455,7 @@ func previewMultipart(body []byte, boundary string, sensitive [][]string) (strin lines = append(lines, fmt.Sprintf("%s: ", name, len(data))) continue } - value := previewBody(data, part.Header.Get("Content-Type"), sensitiveBodySubpaths(sensitive, name)) + value := previewBody(data, part.Header.Get("Content-Type"), sensitive.child(name)) if isSensitiveName(name) || isSensitiveEncodedField(sensitive, name) { value = "[REDACTED]" } @@ -549,8 +551,7 @@ func redactURL(u string) string { type DebugClient struct { Inner HTTPClient Stderr io.Writer - // SensitiveBodyFields lists request-body paths to redact; see sensitiveBodyFields. - SensitiveBodyFields [][]string + SensitiveBodyFields sensitiveBodySchema } // HTTPClient is the interface that SDK clients implement. @@ -594,7 +595,7 @@ func (c *DebugClient) Do(req *http.Request) (*http.Response, error) { if readErr != nil { fmt.Fprintf(c.Stderr, "[DEBUG] Response Body Read Error: %v\n", readErr) } else if len(bodyData) > 0 { - body := previewBody(bodyData, resp.Header.Get("Content-Type"), nil) + body := previewBody(bodyData, resp.Header.Get("Content-Type"), sensitiveBodySchema{}) fmt.Fprintf(c.Stderr, "[DEBUG] Response Body:\n %s\n", formatBodyPreview(body, maxBodyPreview)) } } @@ -623,8 +624,7 @@ type DryRunClient struct { Stdout io.Writer JSON bool Cmd *cobra.Command - // SensitiveBodyFields lists request-body paths to redact; see sensitiveBodyFields. - SensitiveBodyFields [][]string + SensitiveBodyFields sensitiveBodySchema } type dryRunBody struct{ length int64 } diff --git a/templates/templates/cli/includes/security.ts b/templates/templates/cli/includes/security.ts index 9e02829a3..9d862c07e 100644 --- a/templates/templates/cli/includes/security.ts +++ b/templates/templates/cli/includes/security.ts @@ -382,92 +382,128 @@ function requestBodyType(op: Operation): TypeDef | undefined { )?.Type; } -/** - * Request-body values each operation marks sensitive - * (x-speakeasy-param-sensitive or format: password), as Go literals of JSON - * paths keyed by operation id. A "*" segment stands for any array index or map - * key and "**" for any run of segments inside a recursive schema; union - * variants share their parent's path. Diagnostics redacts these in dry-run and - * debug request previews on top of its name-based list, so a secret under a - * generic name is hidden while same-named fields of other schemas stay - * readable. - */ +interface CLISensitiveBodyNode { + Sensitive?: boolean; + Fields?: Map; + Item?: number; + Variants?: number[]; +} + +interface CLISensitiveBodySchema { + Nodes: CLISensitiveBodyNode[]; + Roots: number[]; +} + +function renderSensitiveBodySchema( + schema: CLISensitiveBodySchema, +): string | undefined { + const reachable = new Set(); + schema.Nodes.forEach((node, i) => { + if (node.Sensitive) reachable.add(i); + }); + let changed = true; + while (changed) { + changed = false; + schema.Nodes.forEach((node, i) => { + if (reachable.has(i)) return; + const children = [ + ...(node.Fields?.values() || []), + ...(node.Variants || []), + node.Item || 0, + ]; + if (children.some((child) => reachable.has(child))) { + reachable.add(i); + changed = true; + } + }); + } + const roots = schema.Roots.filter((root) => reachable.has(root)); + if (roots.length === 0) return undefined; + const indices = [...reachable].sort((a, b) => a - b); + const remap = new Map(indices.map((id, i) => [id, i + 1])); + const nodes = indices.map((id) => { + const node = schema.Nodes[id]; + const fields: string[] = []; + if (node.Sensitive) fields.push("Sensitive: true"); + const children = [...(node.Fields || [])] + .filter(([, child]) => reachable.has(child)) + .map(([name, child]) => `"${escapeGoString(name)}": ${remap.get(child)}`); + if (children.length) + fields.push(`Fields: map[string]int{ ${children.join(", ")} }`); + if (node.Item && reachable.has(node.Item)) + fields.push(`Item: ${remap.get(node.Item)}`); + const variants = (node.Variants || []).filter((child) => + reachable.has(child), + ); + if (variants.length) + fields.push( + `Variants: []int{ ${variants + .map((child) => remap.get(child)) + .join(", ")} }`, + ); + return `{ ${fields.join(", ")} }`; + }); + // Spaced braces: this file renders twice (recurse), so "{{" must not appear. + return `{ Nodes: []sensitiveBodyNode{ {}, ${nodes.join( + ", ", + )} }, Roots: []int{ ${roots.map((id) => remap.get(id)).join(", ")} } }`; +} + function templateSensitiveBodyFields(): { OperationID: string; - Paths: string; + Schema: string; }[] { - const operations = new Map>(); + const operations = new Map(); for (const op of allOperations()) { - const paths = operations.get(op.OriginalID) || new Map(); - operations.set(op.OriginalID, paths); - const add = (path: string[]): void => { - paths.set(JSON.stringify(path), path); - }; - // Components on the current walk, flagged once they recur below themselves. - const stack = new Map(); - const walk = (type: TypeDef | undefined, path: string[]): void => { - if (!type) return; + const schema = operations.get(op.OriginalID) || { Nodes: [{}], Roots: [] }; + operations.set(op.OriginalID, schema); + const components = new Map(); + const walk = (type: TypeDef | undefined): number => { + if (!type) return 0; + const component = + type.IsComponent && type.Name ? type.GetRegistrationID() : ""; + if (component && components.has(component)) + return components.get(component)!; + const id = schema.Nodes.length; + const node: CLISensitiveBodyNode = {}; + schema.Nodes.push(node); + if (component) components.set(component, id); if (isSensitiveBodyType(type)) { - add(path); - return; - } - const component = type.IsComponent && type.Name ? type.Name : ""; - if (component) { - if (stack.has(component)) { - stack.set(component, true); - return; - } - stack.set(component, false); + node.Sensitive = true; + return id; } - const before = new Set(paths.keys()); switch (type.Type.toString()) { case "class": + node.Fields = new Map(); + node.Variants = []; for (const field of type.Fields || []) { if (field.Const) continue; - walk( - field.Type, - field.IsAdditionalProperties - ? path - : [...path, originalFieldName(field)], - ); + const child = walk(field.Type); + if (field.IsAdditionalProperties) node.Variants.push(child); + else node.Fields.set(originalFieldName(field), child); } break; case "array": case "set": case "map": - walk(type.ItemType, [...path, "*"]); + node.Item = walk(type.ItemType); break; case "union": - for (const variant of type.AssociatedTypes || []) { - walk(variant, path); - } + node.Variants = (type.AssociatedTypes || []).map(walk); break; } - if (!component) return; - if (stack.get(component)) { - const found = [...paths].filter(([key]) => !before.has(key)); - for (const [, foundPath] of found) { - const rel = foundPath.slice(path.length); - add([...path, "**", ...(rel[0] === "**" ? rel.slice(1) : rel)]); - } - } - stack.delete(component); + return id; }; - walk(requestBodyType(op), []); - } - const entries: { OperationID: string; Paths: string }[] = []; - for (const [operationID, paths] of operations) { - if (paths.size === 0) continue; - entries.push({ - OperationID: operationID, - // Spaced braces: this file renders twice (recurse), so "{{" must not appear. - Paths: `{ ${[...paths.values()] - .map((p) => `{ ${p.map((s) => `"${escapeGoString(s)}"`).join(", ")} }`) - .join(", ")} }`, - }); + schema.Roots.push(walk(requestBodyType(op))); + } + const entries: { OperationID: string; Schema: string }[] = []; + for (const [operationID, schema] of operations) { + const rendered = renderSensitiveBodySchema(schema); + if (rendered) entries.push({ OperationID: operationID, Schema: rendered }); } return entries.sort((a, b) => a.OperationID.localeCompare(b.OperationID)); } + registerTemplateFunc( "templateSensitiveBodyFields", templateSensitiveBodyFields, diff --git a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl index a34213149..5d1cb5dca 100644 --- a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl +++ b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl @@ -2,6 +2,7 @@ package tests import ( "encoding/json" + "io" "net/http" "net/http/httptest" "strings" @@ -414,6 +415,37 @@ func TestDiagnosticsRedactSensitiveBodyFields(t *testing.T) { secrets: []string{"canary-whole-text"}, visible: []string{"[REDACTED]"}, }, + { + name: "recursive union", + args: []string{"request-bodies", "cli-sensitive-recursive-union-post", "--cli-sensitive-recursive-union", + `{"kind":"branch","value":"canary-root","plain":{"name":"visible-name","value":"visible-root"},` + + `"child":{"kind":"branch","value":"canary-child","plain":{"name":"visible-name","value":"visible-child"},` + + `"child":{"kind":"branch","value":"canary-grandchild"}}}`}, + secrets: []string{"canary-root", "canary-child", "canary-grandchild"}, + visible: []string{"visible-root", "visible-child"}, + }, + { + name: "recursive union-reversed", + args: []string{"request-bodies", "cli-sensitive-recursive-union-reversed-post", "--cli-sensitive-recursive-union-reversed", + `{"kind":"branch","value":"canary-root","plain":{"name":"visible-name","value":"visible-root"},` + + `"child":{"kind":"branch","value":"canary-child","plain":{"name":"visible-name","value":"visible-child"},` + + `"child":{"kind":"branch","value":"canary-grandchild"}}}`}, + secrets: []string{"canary-root", "canary-child", "canary-grandchild"}, + visible: []string{"visible-root", "visible-child"}, + }, + { + name: "recursive schema edges", + args: []string{"request-bodies", "cli-sensitive-body-fields-post", "--body", + `{"name":"visible-name","value":"canary-top","tree":{"value":"canary-root",` + + `"plain":{"name":"visible-name","value":"visible-root"},` + + `"children":[{"value":"canary-array","plain":{"name":"visible-name","value":"visible-array"}}],` + + `"byName":{"first":{"value":"canary-map","child":{"value":"canary-map-child"},` + + `"plain":{"name":"visible-name","value":"visible-map"}}},` + + `"link":{"value":"visible-link","next":{"value":"canary-mutual",` + + `"link":{"value":"visible-inner-link","next":{"value":"canary-inner-mutual"}}}}}}`}, + secrets: []string{"canary-top", "canary-root", "canary-array", "canary-map", "canary-map-child", "canary-mutual", "canary-inner-mutual"}, + visible: []string{"visible-root", "visible-array", "visible-map", "visible-link", "visible-inner-link"}, + }, { name: "marked fields at every depth", args: []string{"request-bodies", "cli-sensitive-body-fields-post", "--body", @@ -450,17 +482,19 @@ func TestDiagnosticsRedactSensitiveBodyFields(t *testing.T) { name: "multipart parts", args: []string{"request-bodies", "cli-sensitive-multipart-post", "--name", "visible-name", "--value", "canary-top", "--config-param", `{"label":"visible-label","value":"canary-config"}`, - "--codes", "canary-code-1", "--codes", "canary-code-2"}, - secrets: []string{"canary-top", "canary-config", "canary-code-1", "canary-code-2"}, - visible: []string{"visible-name", "visible-label"}, + "--codes", "canary-code-1", "--codes", "canary-code-2", + "--tree", `{"value":"canary-tree","child":{"value":"canary-tree-child","plain":{"name":"visible-name","value":"visible-tree-plain"}}}`}, + secrets: []string{"canary-top", "canary-config", "canary-code-1", "canary-code-2", "canary-tree", "canary-tree-child"}, + visible: []string{"visible-name", "visible-label", "visible-tree-plain"}, }, { name: "form fields", args: []string{"request-bodies", "cli-sensitive-form-put", "--name", "visible-name", "--value", "canary-top", "--config-param", `{"label":"visible-label","value":"canary-config"}`, - "--codes", "canary-code-1", "--codes", "canary-code-2"}, - secrets: []string{"canary-top", "canary-config", "canary-code-1", "canary-code-2"}, - visible: []string{"visible-name", "visible-label"}, + "--codes", "canary-code-1", "--codes", "canary-code-2", + "--tree", `{"value":"canary-tree","child":{"value":"canary-tree-child","plain":{"name":"visible-name","value":"visible-tree-plain"}}}`}, + secrets: []string{"canary-top", "canary-config", "canary-code-1", "canary-code-2", "canary-tree", "canary-tree-child"}, + visible: []string{"visible-name", "visible-label", "visible-tree-plain"}, }, } for _, mode := range diagnosticsOutputModes { @@ -478,6 +512,24 @@ func TestDiagnosticsRedactSensitiveBodyFields(t *testing.T) { } } +func TestDiagnosticsSensitiveBodyPreservesWireAndResponse(t *testing.T) { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + assert.NoError(t, err) + assert.Contains(t, string(body), "canary-child") + assert.Contains(t, string(body), "canary-root") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"value":"visible-response"}`)) + })) + defer s.Close() + h := NewCLITestHarness(t) + require.NoError(t, h.RunWithServer(s.URL, []string{"--debug", "request-bodies", "cli-sensitive-recursive-union-post", + "--cli-sensitive-recursive-union", `{"kind":"branch","value":"canary-root","child":{"kind":"branch","value":"canary-child"}}`})) + assert.NotContains(t, h.GetStderr(), "canary-root") + assert.NotContains(t, h.GetStderr(), "canary-child") + assert.Contains(t, h.GetStderr(), "visible-response") +} + func TestDiagnosticsKeepUnmarkedFieldSharingSensitiveName(t *testing.T) { for _, mode := range diagnosticsOutputModes { t.Run(mode.name, func(t *testing.T) { diff --git a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml index 857c53d52..d5ee69bd7 100644 --- a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml +++ b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml @@ -216,6 +216,36 @@ paths: responses: "200": description: OK + /anything/cli/sensitiveRecursiveUnion: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveRecursiveUnionPost + description: Union variants share a sensitive field and one variant is recursive. + tags: [requestBodies] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CliSensitiveRecursiveUnion" + responses: + "200": + description: OK + /anything/cli/sensitiveRecursiveUnionReversed: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveRecursiveUnionReversedPost + description: Union variants share a sensitive field and one variant is recursive. + tags: [requestBodies] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CliSensitiveRecursiveUnionReversed" + responses: + "200": + description: OK components: schemas: CliSensitiveBodyFields: @@ -284,6 +314,18 @@ components: $ref: "#/components/schemas/CliSensitiveNode" branch: $ref: "#/components/schemas/CliSensitiveBranch" + plain: + $ref: "#/components/schemas/CliPlainBodyFields" + children: + type: array + items: + $ref: "#/components/schemas/CliSensitiveNode" + byName: + type: object + additionalProperties: + $ref: "#/components/schemas/CliSensitiveNode" + link: + $ref: "#/components/schemas/CliSensitiveLink" CliSensitiveBranch: type: object properties: @@ -318,6 +360,8 @@ components: items: type: string format: password + tree: + $ref: "#/components/schemas/CliSensitiveNode" CliSensitiveSecret: oneOf: - $ref: "#/components/schemas/CliSensitiveStaticSecret" @@ -382,3 +426,52 @@ components: CliSensitiveWholeValue: type: string format: password + CliSensitiveLink: + type: object + properties: + value: + type: string + next: + $ref: "#/components/schemas/CliSensitiveNode" + CliSensitiveRecursiveLeaf: + type: object + required: [kind, value] + properties: + kind: + type: string + const: leaf + value: + type: string + x-speakeasy-param-sensitive: true + CliSensitiveRecursiveBranch: + type: object + required: [kind, value] + properties: + kind: + type: string + const: branch + value: + type: string + x-speakeasy-param-sensitive: true + child: + $ref: "#/components/schemas/CliSensitiveRecursiveBranch" + plain: + $ref: "#/components/schemas/CliPlainBodyFields" + CliSensitiveRecursiveUnion: + oneOf: + - $ref: "#/components/schemas/CliSensitiveRecursiveLeaf" + - $ref: "#/components/schemas/CliSensitiveRecursiveBranch" + discriminator: + propertyName: kind + mapping: + leaf: "#/components/schemas/CliSensitiveRecursiveLeaf" + branch: "#/components/schemas/CliSensitiveRecursiveBranch" + CliSensitiveRecursiveUnionReversed: + oneOf: + - $ref: "#/components/schemas/CliSensitiveRecursiveBranch" + - $ref: "#/components/schemas/CliSensitiveRecursiveLeaf" + discriminator: + propertyName: kind + mapping: + leaf: "#/components/schemas/CliSensitiveRecursiveLeaf" + branch: "#/components/schemas/CliSensitiveRecursiveBranch" diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index 495055fc1..ae7c9ea84 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -222,7 +222,7 @@ trackedFiles: internal/client/client.go: last_write_checksum: sha1:bca25435230238b7194798699e137ef94d983844 internal/client/diagnostics.go: - last_write_checksum: sha1:7e2f2af20793241ffbc7d977de1e921736b4d194 + last_write_checksum: sha1:37f290dbeea89b304f388a982c5713c3b2596e65 internal/clierrors/clierrors.go: last_write_checksum: sha1:e17fb3e33e439c6997834182011540427d760857 internal/config/config.go: diff --git a/zSDKs/sdk-cli/internal/client/diagnostics.go b/zSDKs/sdk-cli/internal/client/diagnostics.go index 37486700b..b5bc33b6b 100644 --- a/zSDKs/sdk-cli/internal/client/diagnostics.go +++ b/zSDKs/sdk-cli/internal/client/diagnostics.go @@ -83,11 +83,19 @@ var sensitiveJSONKeys = map[string]bool{ "clientsecret": true, } -// sensitiveBodyFields lists, per operation id, the request-body fields the API -// marks sensitive (x-speakeasy-param-sensitive or format: password) as JSON -// paths, where "*" stands for any array index or map key and "**" for any run -// of segments inside a recursive schema. -var sensitiveBodyFields = map[string][][]string{} +type sensitiveBodyNode struct { + Sensitive bool + Fields map[string]int + Item int + Variants []int +} + +type sensitiveBodySchema struct { + Nodes []sensitiveBodyNode + Roots []int +} + +var sensitiveBodyFields = map[string]sensitiveBodySchema{} var sensitiveQueryKeys = map[string]bool{ "apikey": true, @@ -166,98 +174,94 @@ func isSensitiveName(key string) bool { return false } -// sensitiveBodyFieldsFor returns the sensitive request-body paths of the -// operation that cmd (an operation or intent command) invokes. -func sensitiveBodyFieldsFor(cmd *cobra.Command) [][]string { +func sensitiveBodyFieldsFor(cmd *cobra.Command) sensitiveBodySchema { if cmd == nil { - return nil + return sensitiveBodySchema{} } return sensitiveBodyFields[cmd.Annotations["speakeasy_operation"]] } -func isSensitiveBodyPath(sensitive [][]string, path []string) bool { - for _, candidate := range sensitive { - if matchesBodyPath(candidate, path) { - return true +func (s sensitiveBodySchema) expandedRoots() []int { + var roots []int + seen := make(map[int]bool) + var visit func(int) + visit = func(id int) { + if id <= 0 || id >= len(s.Nodes) || seen[id] { + return + } + seen[id] = true + roots = append(roots, id) + for _, variant := range s.Nodes[id].Variants { + visit(variant) } } - return false + for _, root := range s.Roots { + visit(root) + } + return roots } -func matchesBodyPath(pattern, path []string) bool { - if len(pattern) == 0 { - return len(path) == 0 - } - if pattern[0] == "**" { - for i := 0; i <= len(path); i++ { - if matchesBodyPath(pattern[1:], path[i:]) { - return true - } +func (s sensitiveBodySchema) isSensitive() bool { + for _, id := range s.expandedRoots() { + if s.Nodes[id].Sensitive { + return true } - return false } - if len(path) == 0 || (pattern[0] != "*" && pattern[0] != path[0]) { - return false - } - return matchesBodyPath(pattern[1:], path[1:]) + return false } -// isSensitiveEncodedField reports whether a form field or multipart part is -// sensitive; repeated fields carry the items of an array. -func isSensitiveEncodedField(sensitive [][]string, name string) bool { - return isSensitiveBodyPath(sensitive, []string{name}) || isSensitiveBodyPath(sensitive, []string{name, "*"}) +func (s sensitiveBodySchema) child(name string) sensitiveBodySchema { + var roots []int + for _, id := range s.expandedRoots() { + node := s.Nodes[id] + if child := node.Fields[name]; child != 0 { + roots = append(roots, child) + } + if node.Item != 0 { + roots = append(roots, node.Item) + } + } + return sensitiveBodySchema{Nodes: s.Nodes, Roots: roots} } -// sensitiveBodySubpaths returns the sensitive paths below the top-level field -// name, relative to it, for fields sent as an encoded JSON value (a multipart -// part or a form field). -func sensitiveBodySubpaths(sensitive [][]string, name string) [][]string { - var out [][]string - var collect func(pattern []string) - collect = func(pattern []string) { - switch { - case len(pattern) == 0: - case pattern[0] == "**": - out = append(out, pattern) - collect(pattern[1:]) - case (pattern[0] == "*" || pattern[0] == name) && len(pattern) > 1: - out = append(out, pattern[1:]) +func (s sensitiveBodySchema) items() sensitiveBodySchema { + var roots []int + for _, id := range s.expandedRoots() { + if item := s.Nodes[id].Item; item != 0 { + roots = append(roots, item) } } - for _, pattern := range sensitive { - collect(pattern) - } - return out + return sensitiveBodySchema{Nodes: s.Nodes, Roots: roots} +} + +func isSensitiveEncodedField(sensitive sensitiveBodySchema, name string) bool { + nested := sensitive.child(name) + return nested.isSensitive() || nested.items().isSensitive() } -// redactJSON recursively redacts sensitive keys in a JSON structure, plus the -// values at the sensitive request-body paths. -func redactJSON(v interface{}, depth int, path []string, sensitive [][]string) interface{} { +func redactJSON(v interface{}, depth int, sensitive sensitiveBodySchema) interface{} { if depth > maxRedactDepth { return redactionDepthMarker } + if sensitive.isSensitive() { + return "[REDACTED]" + } switch val := v.(type) { case map[string]interface{}: out := make(map[string]interface{}, len(val)) for k, child := range val { - childPath := append(path[:len(path):len(path)], k) - if sensitiveJSONKeys[normalizeSensitiveKey(k)] || isSensitiveBodyPath(sensitive, childPath) { + if sensitiveJSONKeys[normalizeSensitiveKey(k)] { out[k] = "[REDACTED]" } else { - out[k] = redactJSON(child, depth+1, childPath, sensitive) + out[k] = redactJSON(child, depth+1, sensitive.child(k)) } } return out case []interface{}: out := make([]interface{}, len(val)) - childPath := append(path[:len(path):len(path)], "*") - redactItems := isSensitiveBodyPath(sensitive, childPath) + items := sensitive.items() for i, child := range val { - if redactItems { - out[i] = "[REDACTED]" - } else { - out[i] = redactJSON(child, depth+1, childPath, sensitive) - } + out[i] = redactJSON(child, depth+1, items) } return out case string: @@ -333,7 +337,7 @@ func isAllHex(value string) bool { return true } -func decodeJSON(body []byte, sensitive [][]string) (interface{}, bool) { +func decodeJSON(body []byte, sensitive sensitiveBodySchema) (interface{}, bool) { dec := json.NewDecoder(bytes.NewReader(body)) dec.UseNumber() var parsed interface{} @@ -343,7 +347,7 @@ func decodeJSON(body []byte, sensitive [][]string) (interface{}, bool) { if err := dec.Decode(&struct{}{}); err != io.EOF { return nil, false } - return redactJSON(parsed, 0, nil, sensitive), true + return redactJSON(parsed, 0, sensitive), true } func isTextMediaType(mediaType string) bool { @@ -354,13 +358,11 @@ func isTextMediaType(mediaType string) bool { mediaType == "application/x-www-form-urlencoded" } -// previewBody renders a body for diagnostics; sensitive holds the request-body -// paths to redact (nil for response bodies). -func previewBody(body []byte, contentType string, sensitive [][]string) interface{} { +func previewBody(body []byte, contentType string, sensitive sensitiveBodySchema) interface{} { if len(body) == 0 { return nil } - if isSensitiveBodyPath(sensitive, nil) { + if sensitive.isSensitive() { return "[REDACTED]" } mediaType, params, err := mime.ParseMediaType(contentType) @@ -389,7 +391,7 @@ func previewBody(body []byte, contentType string, sensitive [][]string) interfac return redactBase64String(string(body)) } -func previewForm(body []byte, sensitive [][]string) string { +func previewForm(body []byte, sensitive sensitiveBodySchema) string { raw := string(body) if _, err := url.ParseQuery(raw); err != nil { return fmt.Sprintf("", len(body)) @@ -407,7 +409,7 @@ func previewForm(body []byte, sensitive [][]string) string { parts[i] = key + "=[REDACTED]" continue } - if nested := sensitiveBodySubpaths(sensitive, decodedKey); len(nested) > 0 { + if nested := sensitive.child(decodedKey); len(nested.Roots) > 0 { if parsed, ok := decodeJSON([]byte(decodedValue), nested); ok { if encoded, err := encodeJSON(parsed, ""); err == nil { parts[i] = key + "=" + url.QueryEscape(encoded) @@ -425,7 +427,7 @@ func previewForm(body []byte, sensitive [][]string) string { return strings.Join(parts, "&") } -func previewMultipart(body []byte, boundary string, sensitive [][]string) (string, bool) { +func previewMultipart(body []byte, boundary string, sensitive sensitiveBodySchema) (string, bool) { r := multipart.NewReader(bytes.NewReader(body), boundary) var lines []string for { @@ -448,7 +450,7 @@ func previewMultipart(body []byte, boundary string, sensitive [][]string) (strin lines = append(lines, fmt.Sprintf("%s: ", name, len(data))) continue } - value := previewBody(data, part.Header.Get("Content-Type"), sensitiveBodySubpaths(sensitive, name)) + value := previewBody(data, part.Header.Get("Content-Type"), sensitive.child(name)) if isSensitiveName(name) || isSensitiveEncodedField(sensitive, name) { value = "[REDACTED]" } @@ -542,10 +544,9 @@ func redactURL(u string) string { // DebugClient wraps an HTTP client and logs request/response diagnostics to stderr. type DebugClient struct { - Inner HTTPClient - Stderr io.Writer - // SensitiveBodyFields lists request-body paths to redact; see sensitiveBodyFields. - SensitiveBodyFields [][]string + Inner HTTPClient + Stderr io.Writer + SensitiveBodyFields sensitiveBodySchema } // HTTPClient is the interface that SDK clients implement. @@ -589,7 +590,7 @@ func (c *DebugClient) Do(req *http.Request) (*http.Response, error) { if readErr != nil { fmt.Fprintf(c.Stderr, "[DEBUG] Response Body Read Error: %v\n", readErr) } else if len(bodyData) > 0 { - body := previewBody(bodyData, resp.Header.Get("Content-Type"), nil) + body := previewBody(bodyData, resp.Header.Get("Content-Type"), sensitiveBodySchema{}) fmt.Fprintf(c.Stderr, "[DEBUG] Response Body:\n %s\n", formatBodyPreview(body, maxBodyPreview)) } } @@ -614,12 +615,11 @@ func isStreamingResponse(resp *http.Response) bool { // DryRunClient intercepts HTTP requests and returns a synthetic response // without making any network calls. type DryRunClient struct { - Stderr io.Writer - Stdout io.Writer - JSON bool - Cmd *cobra.Command - // SensitiveBodyFields lists request-body paths to redact; see sensitiveBodyFields. - SensitiveBodyFields [][]string + Stderr io.Writer + Stdout io.Writer + JSON bool + Cmd *cobra.Command + SensitiveBodyFields sensitiveBodySchema } type dryRunBody struct{ length int64 } From 0aa9a11d80c774d2ad065907970ba9164ff5bcaf Mon Sep 17 00:00:00 2001 From: 2ynn Date: Wed, 7 Oct 2026 15:07:22 -0400 Subject: [PATCH 18/23] fix(cli): keep the redactJSON doc comment --- .../templates/cli/auxiliary/internal/client/diagnostics.go.stmpl | 1 + zSDKs/sdk-cli/internal/client/diagnostics.go | 1 + 2 files changed, 2 insertions(+) diff --git a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl index 32ae84b2b..3d896bd94 100644 --- a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl +++ b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl @@ -244,6 +244,7 @@ func isSensitiveEncodedField(sensitive sensitiveBodySchema, name string) bool { return nested.isSensitive() || nested.items().isSensitive() } +// redactJSON recursively redacts sensitive keys in a JSON structure. func redactJSON(v interface{}, depth int, sensitive sensitiveBodySchema) interface{} { if depth > maxRedactDepth { return redactionDepthMarker diff --git a/zSDKs/sdk-cli/internal/client/diagnostics.go b/zSDKs/sdk-cli/internal/client/diagnostics.go index b5bc33b6b..13936f53f 100644 --- a/zSDKs/sdk-cli/internal/client/diagnostics.go +++ b/zSDKs/sdk-cli/internal/client/diagnostics.go @@ -239,6 +239,7 @@ func isSensitiveEncodedField(sensitive sensitiveBodySchema, name string) bool { return nested.isSensitive() || nested.items().isSensitive() } +// redactJSON recursively redacts sensitive keys in a JSON structure. func redactJSON(v interface{}, depth int, sensitive sensitiveBodySchema) interface{} { if depth > maxRedactDepth { return redactionDepthMarker From 34723492d4bc40956c98e67180407408cfb947fd Mon Sep 17 00:00:00 2001 From: 2ynn Date: Wed, 7 Oct 2026 15:22:35 -0400 Subject: [PATCH 19/23] fix(cli): keep declared fields readable beside sensitive map values The sensitive-body graph used one edge for array items and map values, and modelled additional properties as a union variant. A sensitive additionalProperties schema therefore redacted every declared field of the object, and a union of a sensitive array and an object redacted every field of the object variant. Array items and map values now use separate edges. Additional properties become the object's map-value edge, consulted only for keys the object does not declare. Recursive types are memoized through the shared custom-type ID used by the help renderer. --- templates/templates/cli/README.md | 2 +- .../internal/client/diagnostics.go.stmpl | 28 ++++++----- templates/templates/cli/includes/security.ts | 37 +++++++++------ .../tests/primary/diagnostics_test.go.stmpl | 14 ++++++ .../primary/cli-sensitive-body-fields.yaml | 47 +++++++++++++++++++ zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/client/diagnostics.go | 10 ++-- 7 files changed, 107 insertions(+), 33 deletions(-) diff --git a/templates/templates/cli/README.md b/templates/templates/cli/README.md index c315cfe4f..4f1920af8 100644 --- a/templates/templates/cli/README.md +++ b/templates/templates/cli/README.md @@ -1491,7 +1491,7 @@ cli users list --debug - **Headers**: Case-insensitive denylist (`authorization`, `cookie`, `set-cookie`, `x-api-key`, `x-session-token`, `*-secret`, `*-token` suffixes) → `[REDACTED]` - **URL query**: Credential parameters (`api_key`, `token`, `client_secret`, `signature`, etc.) are redacted without reordering the query -- **Request body values marked sensitive**: `templateSensitiveBodyFields` (`includes/security.ts`) builds a schema graph per operation for values marked `x-speakeasy-param-sensitive: true` or `format: password`, including whole bodies. Graph edges follow object fields, array items, map values, union variants, and recursive component references. Dry-run and debug request previews select the graph through the command's `speakeasy_operation` annotation; split request content types contribute roots to the same graph. Unmarked fields in unrelated descendant schemas stay readable. Form fields and multipart parts are matched by name, with JSON-encoded fields traversed from that field's schema and repeated scalar fields treated as array items. Response bodies use the name-based rules only. The marker must sit on the value's own schema: an extension next to a `$ref` is not read +- **Request body values marked sensitive**: `templateSensitiveBodyFields` (`includes/security.ts`) builds a schema graph per operation for values marked `x-speakeasy-param-sensitive: true` or `format: password`, including whole bodies. Graph edges follow object fields, array items, map and additional-properties values, union variants, and recursive references. A declared field never inherits the schema of its object's additional properties, so a sensitive `additionalProperties` leaves declared siblings readable. Dry-run and debug request previews select the graph through the command's `speakeasy_operation` annotation; split request content types contribute roots to the same graph. Unmarked fields in unrelated descendant schemas stay readable. Form fields and multipart parts are matched by name, with JSON-encoded fields traversed from that field's schema and repeated scalar fields treated as array items. Response bodies use the name-based rules only. The marker must sit on the value's own schema: an extension next to a `$ref` is not read - **JSON body**: Separator/case-normalized sensitive keys and canonical base64 strings (encoded length ≥128, excluding all-hex IDs) are redacted; numbers retain their exact lexemes and depth overflow becomes an explicit marker - **Other bodies**: Binary media becomes `` and multipart payloads become a deterministic inventory; human dry-run is not truncated. Debug applies the same redactor, then caps the rendered preview at 4 KiB - **Credentials**: Generated request construction uses request-scoped resolvers. Dry-run reads flags, environment and config, but never probes the OS keychain; whoami also uses the request-scoped resolvers (so `whoami --dry-run` skips the keychain), while configuration retains keychain access diff --git a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl index 3d896bd94..1eafd5c65 100644 --- a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl +++ b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl @@ -86,9 +86,10 @@ var sensitiveJSONKeys = map[string]bool{ type sensitiveBodyNode struct { Sensitive bool - Fields map[string]int - Item int - Variants []int + Fields map[string]int + Item int + Values int + Variants []int } type sensitiveBodySchema struct { @@ -219,11 +220,12 @@ func (s sensitiveBodySchema) child(name string) sensitiveBodySchema { var roots []int for _, id := range s.expandedRoots() { node := s.Nodes[id] - if child := node.Fields[name]; child != 0 { - roots = append(roots, child) + child, declared := node.Fields[name] + if !declared { + child = node.Values } - if node.Item != 0 { - roots = append(roots, node.Item) + if child != 0 { + roots = append(roots, child) } } return sensitiveBodySchema{Nodes: s.Nodes, Roots: roots} @@ -550,8 +552,8 @@ func redactURL(u string) string { // DebugClient wraps an HTTP client and logs request/response diagnostics to stderr. type DebugClient struct { - Inner HTTPClient - Stderr io.Writer + Inner HTTPClient + Stderr io.Writer SensitiveBodyFields sensitiveBodySchema } @@ -621,10 +623,10 @@ func isStreamingResponse(resp *http.Response) bool { // DryRunClient intercepts HTTP requests and returns a synthetic response // without making any network calls. type DryRunClient struct { - Stderr io.Writer - Stdout io.Writer - JSON bool - Cmd *cobra.Command + Stderr io.Writer + Stdout io.Writer + JSON bool + Cmd *cobra.Command SensitiveBodyFields sensitiveBodySchema } diff --git a/templates/templates/cli/includes/security.ts b/templates/templates/cli/includes/security.ts index 9d862c07e..29c4f74aa 100644 --- a/templates/templates/cli/includes/security.ts +++ b/templates/templates/cli/includes/security.ts @@ -386,6 +386,7 @@ interface CLISensitiveBodyNode { Sensitive?: boolean; Fields?: Map; Item?: number; + Values?: number; Variants?: number[]; } @@ -410,6 +411,7 @@ function renderSensitiveBodySchema( ...(node.Fields?.values() || []), ...(node.Variants || []), node.Item || 0, + node.Values || 0, ]; if (children.some((child) => reachable.has(child))) { reachable.add(i); @@ -425,13 +427,18 @@ function renderSensitiveBodySchema( const node = schema.Nodes[id]; const fields: string[] = []; if (node.Sensitive) fields.push("Sensitive: true"); + const values = node.Values && reachable.has(node.Values) ? node.Values : 0; const children = [...(node.Fields || [])] - .filter(([, child]) => reachable.has(child)) - .map(([name, child]) => `"${escapeGoString(name)}": ${remap.get(child)}`); + .filter(([, child]) => values || reachable.has(child)) + .map( + ([name, child]) => + `"${escapeGoString(name)}": ${remap.get(child) || 0}`, + ); if (children.length) fields.push(`Fields: map[string]int{ ${children.join(", ")} }`); if (node.Item && reachable.has(node.Item)) fields.push(`Item: ${remap.get(node.Item)}`); + if (values) fields.push(`Values: ${remap.get(values)}`); const variants = (node.Variants || []).filter((child) => reachable.has(child), ); @@ -457,17 +464,15 @@ function templateSensitiveBodyFields(): { for (const op of allOperations()) { const schema = operations.get(op.OriginalID) || { Nodes: [{}], Roots: [] }; operations.set(op.OriginalID, schema); - const components = new Map(); + const walked = new Map(); const walk = (type: TypeDef | undefined): number => { if (!type) return 0; - const component = - type.IsComponent && type.Name ? type.GetRegistrationID() : ""; - if (component && components.has(component)) - return components.get(component)!; + const typeID = type.IsCustomType() ? getUniqueID(type) : ""; + if (typeID && walked.has(typeID)) return walked.get(typeID)!; const id = schema.Nodes.length; const node: CLISensitiveBodyNode = {}; schema.Nodes.push(node); - if (component) components.set(component, id); + if (typeID) walked.set(typeID, id); if (isSensitiveBodyType(type)) { node.Sensitive = true; return id; @@ -475,19 +480,23 @@ function templateSensitiveBodyFields(): { switch (type.Type.toString()) { case "class": node.Fields = new Map(); - node.Variants = []; for (const field of type.Fields || []) { - if (field.Const) continue; - const child = walk(field.Type); - if (field.IsAdditionalProperties) node.Variants.push(child); - else node.Fields.set(originalFieldName(field), child); + if (field.IsAdditionalProperties) + node.Values = walk(field.Type.ItemType); + else + node.Fields.set( + originalFieldName(field), + field.Const ? 0 : walk(field.Type), + ); } break; case "array": case "set": - case "map": node.Item = walk(type.ItemType); break; + case "map": + node.Values = walk(type.ItemType); + break; case "union": node.Variants = (type.AssociatedTypes || []).map(walk); break; diff --git a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl index 5d1cb5dca..8de3f1a9e 100644 --- a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl +++ b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl @@ -463,6 +463,20 @@ func TestDiagnosticsRedactSensitiveBodyFields(t *testing.T) { visible: []string{"visible-name", "visible-nested", "visible-entry", "visible-map", "visible-code", "visible-root", "visible-child", "visible-branch"}, }, + { + name: "additional properties beside declared fields", + args: []string{"request-bodies", "cli-sensitive-open-object-post", "--body", + `{"name":"visible-name","value":"visible-value","extra":"canary-extra","entries":{"label":"visible-label"}}`}, + secrets: []string{"canary-extra"}, + visible: []string{"visible-name", "visible-value", "visible-label", "visible-const"}, + }, + { + name: "array variant beside object variant", + args: []string{"request-bodies", "cli-sensitive-open-object-post", "--body", + `{"name":"visible-name","entries":["canary-list"]}`}, + secrets: []string{"canary-list"}, + visible: []string{"visible-name", "visible-const"}, + }, { name: "union variant field", args: []string{"request-bodies", "cli-sensitive-union-body-post", "--scope", "visible-scope", diff --git a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml index d5ee69bd7..837bb891a 100644 --- a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml +++ b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml @@ -246,6 +246,26 @@ paths: responses: "200": description: OK + /anything/cli/sensitiveOpenObject: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveOpenObjectPost + description: >- + Request body whose additional properties are marked sensitive next to + declared fields, one of them a union of a sensitive array and a plain + object. Diagnostics output must redact the additional properties and + the array items while keeping the declared fields, the constant and + the plain object readable. + tags: [requestBodies] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CliSensitiveOpenObject" + responses: + "200": + description: OK components: schemas: CliSensitiveBodyFields: @@ -475,3 +495,30 @@ components: mapping: leaf: "#/components/schemas/CliSensitiveRecursiveLeaf" branch: "#/components/schemas/CliSensitiveRecursiveBranch" + CliSensitiveOpenObject: + type: object + required: [kind, name] + properties: + kind: + type: string + const: visible-const + name: + type: string + value: + type: string + entries: + $ref: "#/components/schemas/CliSensitiveListOrLabel" + additionalProperties: + type: string + x-speakeasy-param-sensitive: true + CliSensitiveListOrLabel: + oneOf: + - type: array + items: + type: string + format: password + - type: object + required: [label] + properties: + label: + type: string diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index ae7c9ea84..0755856ff 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -222,7 +222,7 @@ trackedFiles: internal/client/client.go: last_write_checksum: sha1:bca25435230238b7194798699e137ef94d983844 internal/client/diagnostics.go: - last_write_checksum: sha1:37f290dbeea89b304f388a982c5713c3b2596e65 + last_write_checksum: sha1:b5a1c38793fb785f3c59bf637e28eb8cd2f8e87d internal/clierrors/clierrors.go: last_write_checksum: sha1:e17fb3e33e439c6997834182011540427d760857 internal/config/config.go: diff --git a/zSDKs/sdk-cli/internal/client/diagnostics.go b/zSDKs/sdk-cli/internal/client/diagnostics.go index 13936f53f..1c0530330 100644 --- a/zSDKs/sdk-cli/internal/client/diagnostics.go +++ b/zSDKs/sdk-cli/internal/client/diagnostics.go @@ -87,6 +87,7 @@ type sensitiveBodyNode struct { Sensitive bool Fields map[string]int Item int + Values int Variants []int } @@ -214,11 +215,12 @@ func (s sensitiveBodySchema) child(name string) sensitiveBodySchema { var roots []int for _, id := range s.expandedRoots() { node := s.Nodes[id] - if child := node.Fields[name]; child != 0 { - roots = append(roots, child) + child, declared := node.Fields[name] + if !declared { + child = node.Values } - if node.Item != 0 { - roots = append(roots, node.Item) + if child != 0 { + roots = append(roots, child) } } return sensitiveBodySchema{Nodes: s.Nodes, Roots: roots} From 3d2c4c4d500c46eb80de5c77bce8259da692906d Mon Sep 17 00:00:00 2001 From: 2ynn Date: Wed, 7 Oct 2026 16:33:50 -0400 Subject: [PATCH 20/23] fix(cli): show JSON-encoded form fields as sent when nothing is redacted A JSON-encoded form field on a path to a sensitive value was always decoded and re-encoded for the preview, which dropped the HTML escaping the request used. The preview now keeps the original bytes unless redaction changed the value. The request body type is resolved through the shared getRequestBody helper, and the README notes that the graph also applies to OAuth2 token requests and that union variants sharing a field name are redacted together. --- templates/templates/cli/README.md | 2 +- .../internal/client/diagnostics.go.stmpl | 19 ++++++++++++++++--- templates/templates/cli/includes/security.ts | 8 +++----- .../tests/primary/diagnostics_test.go.stmpl | 7 +++++++ zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/client/diagnostics.go | 19 ++++++++++++++++--- 6 files changed, 44 insertions(+), 13 deletions(-) diff --git a/templates/templates/cli/README.md b/templates/templates/cli/README.md index 4f1920af8..c3fc1e592 100644 --- a/templates/templates/cli/README.md +++ b/templates/templates/cli/README.md @@ -1491,7 +1491,7 @@ cli users list --debug - **Headers**: Case-insensitive denylist (`authorization`, `cookie`, `set-cookie`, `x-api-key`, `x-session-token`, `*-secret`, `*-token` suffixes) → `[REDACTED]` - **URL query**: Credential parameters (`api_key`, `token`, `client_secret`, `signature`, etc.) are redacted without reordering the query -- **Request body values marked sensitive**: `templateSensitiveBodyFields` (`includes/security.ts`) builds a schema graph per operation for values marked `x-speakeasy-param-sensitive: true` or `format: password`, including whole bodies. Graph edges follow object fields, array items, map and additional-properties values, union variants, and recursive references. A declared field never inherits the schema of its object's additional properties, so a sensitive `additionalProperties` leaves declared siblings readable. Dry-run and debug request previews select the graph through the command's `speakeasy_operation` annotation; split request content types contribute roots to the same graph. Unmarked fields in unrelated descendant schemas stay readable. Form fields and multipart parts are matched by name, with JSON-encoded fields traversed from that field's schema and repeated scalar fields treated as array items. Response bodies use the name-based rules only. The marker must sit on the value's own schema: an extension next to a `$ref` is not read +- **Request body values marked sensitive**: `templateSensitiveBodyFields` (`includes/security.ts`) builds a schema graph per operation for values marked `x-speakeasy-param-sensitive: true` or `format: password`, including whole bodies. Graph edges follow object fields, array items, map and additional-properties values, union variants, and recursive references. A declared field never inherits the schema of its object's additional properties, so a sensitive `additionalProperties` leaves declared siblings readable. Dry-run and debug request previews select the graph through the command's `speakeasy_operation` annotation; split request content types contribute roots to the same graph. The graph applies to every request sent through the command's client, including an OAuth2 token request, so a body marked sensitive as a whole also hides that request's form body. Union variants are not told apart at runtime: a field sensitive in one variant is redacted in every variant that declares the same name. Unmarked fields in unrelated descendant schemas stay readable. Form fields and multipart parts are matched by name, with JSON-encoded fields traversed from that field's schema and repeated scalar fields treated as array items; a JSON-encoded form field with nothing to redact is shown exactly as sent. Response bodies use the name-based rules only. The marker must sit on the value's own schema: an extension next to a `$ref` is not read - **JSON body**: Separator/case-normalized sensitive keys and canonical base64 strings (encoded length ≥128, excluding all-hex IDs) are redacted; numbers retain their exact lexemes and depth overflow becomes an explicit marker - **Other bodies**: Binary media becomes `` and multipart payloads become a deterministic inventory; human dry-run is not truncated. Debug applies the same redactor, then caps the rendered preview at 4 KiB - **Credentials**: Generated request construction uses request-scoped resolvers. Dry-run reads flags, environment and config, but never probes the OS keychain; whoami also uses the request-scoped resolvers (so `whoami --dry-run` skips the keychain), while configuration retains keychain access diff --git a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl index 1eafd5c65..1879fe790 100644 --- a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl +++ b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl @@ -11,6 +11,7 @@ import ( "mime/multipart" "net/http" "net/url" + "reflect" "sort" "strings" @@ -345,7 +346,7 @@ func isAllHex(value string) bool { return true } -func decodeJSON(body []byte, sensitive sensitiveBodySchema) (interface{}, bool) { +func parseJSON(body []byte) (interface{}, bool) { dec := json.NewDecoder(bytes.NewReader(body)) dec.UseNumber() var parsed interface{} @@ -355,6 +356,14 @@ func decodeJSON(body []byte, sensitive sensitiveBodySchema) (interface{}, bool) if err := dec.Decode(&struct{}{}); err != io.EOF { return nil, false } + return parsed, true +} + +func decodeJSON(body []byte, sensitive sensitiveBodySchema) (interface{}, bool) { + parsed, ok := parseJSON(body) + if !ok { + return nil, false + } return redactJSON(parsed, 0, sensitive), true } @@ -418,8 +427,12 @@ func previewForm(body []byte, sensitive sensitiveBodySchema) string { continue } if nested := sensitive.child(decodedKey); len(nested.Roots) > 0 { - if parsed, ok := decodeJSON([]byte(decodedValue), nested); ok { - if encoded, err := encodeJSON(parsed, ""); err == nil { + if parsed, ok := parseJSON([]byte(decodedValue)); ok { + redacted := redactJSON(parsed, 0, nested) + if reflect.DeepEqual(parsed, redacted) { + continue + } + if encoded, err := encodeJSON(redacted, ""); err == nil { parts[i] = key + "=" + url.QueryEscape(encoded) continue } diff --git a/templates/templates/cli/includes/security.ts b/templates/templates/cli/includes/security.ts index 29c4f74aa..deba583b2 100644 --- a/templates/templates/cli/includes/security.ts +++ b/templates/templates/cli/includes/security.ts @@ -375,11 +375,9 @@ function isSensitiveBodyType(type: TypeDef): boolean { } function requestBodyType(op: Operation): TypeDef | undefined { - if (!op.Request) return undefined; - if (op.Request.IsRequestBody) return op.Request.RequestBody?.Type; - return op.Request.Field?.Type?.Fields?.find( - (f: FieldDef) => f.Annotations?.Has("request"), - )?.Type; + const body = getRequestBody(op); + if (body?.Type === "request") return body.Value.RequestBody?.Type; + return body?.Value.Type; } interface CLISensitiveBodyNode { diff --git a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl index 8de3f1a9e..86ebd262a 100644 --- a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl +++ b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl @@ -510,6 +510,13 @@ func TestDiagnosticsRedactSensitiveBodyFields(t *testing.T) { secrets: []string{"canary-top", "canary-config", "canary-code-1", "canary-code-2", "canary-tree", "canary-tree-child"}, visible: []string{"visible-name", "visible-label", "visible-tree-plain"}, }, + { + name: "form json field without redactions kept as sent", + args: []string{"request-bodies", "cli-sensitive-form-put", "--name", "visible-name", "--value", "canary-top", + "--tree", `{"label":"visible 0 { - if parsed, ok := decodeJSON([]byte(decodedValue), nested); ok { - if encoded, err := encodeJSON(parsed, ""); err == nil { + if parsed, ok := parseJSON([]byte(decodedValue)); ok { + redacted := redactJSON(parsed, 0, nested) + if reflect.DeepEqual(parsed, redacted) { + continue + } + if encoded, err := encodeJSON(redacted, ""); err == nil { parts[i] = key + "=" + url.QueryEscape(encoded) continue } From ace147a492a7fcb37acb47e64fab83f7ec6d76c5 Mon Sep 17 00:00:00 2001 From: 2ynn Date: Wed, 7 Oct 2026 18:02:32 -0400 Subject: [PATCH 21/23] test(cli): cover arrays of objects in multipart and form diagnostics previews --- .../tests/primary/diagnostics_test.go.stmpl | 10 ++++++---- .../primary/cli-sensitive-body-fields.yaml | 20 +++++++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl index 86ebd262a..c240f0ac0 100644 --- a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl +++ b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl @@ -497,18 +497,20 @@ func TestDiagnosticsRedactSensitiveBodyFields(t *testing.T) { args: []string{"request-bodies", "cli-sensitive-multipart-post", "--name", "visible-name", "--value", "canary-top", "--config-param", `{"label":"visible-label","value":"canary-config"}`, "--codes", "canary-code-1", "--codes", "canary-code-2", + "--entries", `[{"label":"visible-entry","value":"canary-entry"}]`, "--tree", `{"value":"canary-tree","child":{"value":"canary-tree-child","plain":{"name":"visible-name","value":"visible-tree-plain"}}}`}, - secrets: []string{"canary-top", "canary-config", "canary-code-1", "canary-code-2", "canary-tree", "canary-tree-child"}, - visible: []string{"visible-name", "visible-label", "visible-tree-plain"}, + secrets: []string{"canary-top", "canary-config", "canary-code-1", "canary-code-2", "canary-entry", "canary-tree", "canary-tree-child"}, + visible: []string{"visible-name", "visible-label", "visible-entry", "visible-tree-plain"}, }, { name: "form fields", args: []string{"request-bodies", "cli-sensitive-form-put", "--name", "visible-name", "--value", "canary-top", "--config-param", `{"label":"visible-label","value":"canary-config"}`, "--codes", "canary-code-1", "--codes", "canary-code-2", + "--entries", `[{"label":"visible-entry","value":"canary-entry"}]`, "--tree", `{"value":"canary-tree","child":{"value":"canary-tree-child","plain":{"name":"visible-name","value":"visible-tree-plain"}}}`}, - secrets: []string{"canary-top", "canary-config", "canary-code-1", "canary-code-2", "canary-tree", "canary-tree-child"}, - visible: []string{"visible-name", "visible-label", "visible-tree-plain"}, + secrets: []string{"canary-top", "canary-config", "canary-code-1", "canary-code-2", "canary-entry", "canary-tree", "canary-tree-child"}, + visible: []string{"visible-name", "visible-label", "visible-entry", "visible-tree-plain"}, }, { name: "form json field without redactions kept as sent", diff --git a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml index 837bb891a..cbe28e25f 100644 --- a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml +++ b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml @@ -61,8 +61,9 @@ paths: operationId: cliSensitiveMultipartPost description: >- Multipart request body with a sensitive top-level part, a sensitive - array sent as repeated parts, and a JSON part that nests a sensitive - field. Diagnostics output must redact all of them. + array sent as repeated parts, a JSON part that nests a sensitive + field, and an array of objects with a sensitive field. Diagnostics + output must redact all of them. tags: - requestBodies requestBody: @@ -79,8 +80,9 @@ paths: operationId: cliSensitiveFormPut description: >- Form-encoded request body with a sensitive top-level field, a sensitive - array sent as repeated fields, and a JSON-encoded field that nests a - sensitive field. Diagnostics output must redact all of them. + array sent as repeated fields, a JSON-encoded field that nests a + sensitive field, and an array of objects with a sensitive field. + Diagnostics output must redact all of them. tags: - requestBodies requestBody: @@ -380,6 +382,16 @@ components: items: type: string format: password + entries: + type: array + items: + type: object + properties: + label: + type: string + value: + type: string + x-speakeasy-param-sensitive: true tree: $ref: "#/components/schemas/CliSensitiveNode" CliSensitiveSecret: From 432cfabaefb29d29a0292b6233f9a1320c1fc785 Mon Sep 17 00:00:00 2001 From: 2ynn Date: Thu, 8 Oct 2026 14:22:56 -0400 Subject: [PATCH 22/23] refactor(cli): mask sensitive body values through the SDK redact package Dry-run and debug previews now pass request and response bodies through the embedded SDK's redact package, which selects the operation's schema graph from the request context, before applying the name-based preview rules. This replaces the CLI's own request-body graph and traversal, and adds schema-based masking to debug response previews. A body masked as a whole is shown as the mask. Name-based rules still apply inside JSON-encoded form fields, so a property named like a credential stays hidden next to a schema-marked one. --- .changesets/1790872792-577bf8cd.yaml | 2 +- templates/templates/cli/README.md | 2 +- .../internal/client/diagnostics.go.stmpl | 174 +++++------------- templates/templates/cli/includes/security.ts | 147 --------------- .../tests/primary/diagnostics_test.go.stmpl | 36 +++- .../primary/cli-sensitive-body-fields.yaml | 34 ++++ zSDKs/sdk-cli/.speakeasy/gen.lock | 2 +- zSDKs/sdk-cli/internal/client/diagnostics.go | 170 +++++------------ 8 files changed, 161 insertions(+), 406 deletions(-) diff --git a/.changesets/1790872792-577bf8cd.yaml b/.changesets/1790872792-577bf8cd.yaml index 2e76350e2..376587221 100644 --- a/.changesets/1790872792-577bf8cd.yaml +++ b/.changesets/1790872792-577bf8cd.yaml @@ -5,6 +5,6 @@ targets: - cli type: fix bump: patch -description: '--dry-run and --debug redact sensitive request-body values and whole bodies across JSON, form, multipart and text previews, including recursive schemas and operations with multiple request content types; unrelated unmarked fields remain readable' +description: '--dry-run and --debug mask request and response body values marked sensitive (x-speakeasy-param-sensitive or format: password) through the SDK redact package, including whole bodies, recursive schemas and form or multipart fields' author: 2ynn date: "2026-10-01" diff --git a/templates/templates/cli/README.md b/templates/templates/cli/README.md index c3fc1e592..317c79e27 100644 --- a/templates/templates/cli/README.md +++ b/templates/templates/cli/README.md @@ -1491,7 +1491,7 @@ cli users list --debug - **Headers**: Case-insensitive denylist (`authorization`, `cookie`, `set-cookie`, `x-api-key`, `x-session-token`, `*-secret`, `*-token` suffixes) → `[REDACTED]` - **URL query**: Credential parameters (`api_key`, `token`, `client_secret`, `signature`, etc.) are redacted without reordering the query -- **Request body values marked sensitive**: `templateSensitiveBodyFields` (`includes/security.ts`) builds a schema graph per operation for values marked `x-speakeasy-param-sensitive: true` or `format: password`, including whole bodies. Graph edges follow object fields, array items, map and additional-properties values, union variants, and recursive references. A declared field never inherits the schema of its object's additional properties, so a sensitive `additionalProperties` leaves declared siblings readable. Dry-run and debug request previews select the graph through the command's `speakeasy_operation` annotation; split request content types contribute roots to the same graph. The graph applies to every request sent through the command's client, including an OAuth2 token request, so a body marked sensitive as a whole also hides that request's form body. Union variants are not told apart at runtime: a field sensitive in one variant is redacted in every variant that declares the same name. Unmarked fields in unrelated descendant schemas stay readable. Form fields and multipart parts are matched by name, with JSON-encoded fields traversed from that field's schema and repeated scalar fields treated as array items; a JSON-encoded form field with nothing to redact is shown exactly as sent. Response bodies use the name-based rules only. The marker must sit on the value's own schema: an extension next to a `$ref` is not read +- **Body values marked sensitive**: request and response previews first pass through the embedded SDK's `redact` package (`redact.RequestBody` / `redact.ResponseBody`), which masks values marked `x-speakeasy-param-sensitive: true` or `format: password` with `redact.Mask`. The SDK stamps the operation ID on each request context, so the schema graph (built by `Operation.SensitiveRequestBodyGraph` / `SensitiveResponseBodyGraph`) is selected per SDK method, and split request content types each use their own graph. Form fields and multipart parts are matched by name and JSON-encoded fields are masked within; other media types, and bodies that do not match their schema, are masked as a whole. Union variants are not told apart: a field sensitive in one variant is masked in every variant declaring the same name. The marker must sit on the value's own schema: an extension next to a `$ref` is not read. The name-based rules below still apply afterwards - **JSON body**: Separator/case-normalized sensitive keys and canonical base64 strings (encoded length ≥128, excluding all-hex IDs) are redacted; numbers retain their exact lexemes and depth overflow becomes an explicit marker - **Other bodies**: Binary media becomes `` and multipart payloads become a deterministic inventory; human dry-run is not truncated. Debug applies the same redactor, then caps the rendered preview at 4 KiB - **Credentials**: Generated request construction uses request-scoped resolvers. Dry-run reads flags, environment and config, but never probes the OS keychain; whoami also uses the request-scoped resolvers (so `whoami --dry-run` skips the keychain), while configuration retains keychain access diff --git a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl index 1879fe790..96fcc9464 100644 --- a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl +++ b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl @@ -19,6 +19,7 @@ import ( "{{getGolangPackage}}/internal/config" "{{getGolangPackage}}/internal/flagutil" + "{{getGolangPackage}}/internal/sdk/redact" ) // maxBodyPreview is the maximum number of bytes to show in body previews. @@ -85,25 +86,6 @@ var sensitiveJSONKeys = map[string]bool{ "clientsecret": true, } -type sensitiveBodyNode struct { - Sensitive bool - Fields map[string]int - Item int - Values int - Variants []int -} - -type sensitiveBodySchema struct { - Nodes []sensitiveBodyNode - Roots []int -} - -var sensitiveBodyFields = map[string]sensitiveBodySchema{ - {{- range templateSensitiveBodyFields}} - "{{escapeGoString .OperationID}}": {{.Schema}}, - {{- end}} -} - var sensitiveQueryKeys = map[string]bool{ "apikey": true, "accesstoken": true, @@ -181,80 +163,11 @@ func isSensitiveName(key string) bool { return false } -func sensitiveBodyFieldsFor(cmd *cobra.Command) sensitiveBodySchema { - if cmd == nil { - return sensitiveBodySchema{} - } - return sensitiveBodyFields[cmd.Annotations["speakeasy_operation"]] -} - -func (s sensitiveBodySchema) expandedRoots() []int { - var roots []int - seen := make(map[int]bool) - var visit func(int) - visit = func(id int) { - if id <= 0 || id >= len(s.Nodes) || seen[id] { - return - } - seen[id] = true - roots = append(roots, id) - for _, variant := range s.Nodes[id].Variants { - visit(variant) - } - } - for _, root := range s.Roots { - visit(root) - } - return roots -} - -func (s sensitiveBodySchema) isSensitive() bool { - for _, id := range s.expandedRoots() { - if s.Nodes[id].Sensitive { - return true - } - } - return false -} - -func (s sensitiveBodySchema) child(name string) sensitiveBodySchema { - var roots []int - for _, id := range s.expandedRoots() { - node := s.Nodes[id] - child, declared := node.Fields[name] - if !declared { - child = node.Values - } - if child != 0 { - roots = append(roots, child) - } - } - return sensitiveBodySchema{Nodes: s.Nodes, Roots: roots} -} - -func (s sensitiveBodySchema) items() sensitiveBodySchema { - var roots []int - for _, id := range s.expandedRoots() { - if item := s.Nodes[id].Item; item != 0 { - roots = append(roots, item) - } - } - return sensitiveBodySchema{Nodes: s.Nodes, Roots: roots} -} - -func isSensitiveEncodedField(sensitive sensitiveBodySchema, name string) bool { - nested := sensitive.child(name) - return nested.isSensitive() || nested.items().isSensitive() -} - // redactJSON recursively redacts sensitive keys in a JSON structure. -func redactJSON(v interface{}, depth int, sensitive sensitiveBodySchema) interface{} { +func redactJSON(v interface{}, depth int) interface{} { if depth > maxRedactDepth { return redactionDepthMarker } - if sensitive.isSensitive() { - return "[REDACTED]" - } switch val := v.(type) { case map[string]interface{}: out := make(map[string]interface{}, len(val)) @@ -262,15 +175,14 @@ func redactJSON(v interface{}, depth int, sensitive sensitiveBodySchema) interfa if sensitiveJSONKeys[normalizeSensitiveKey(k)] { out[k] = "[REDACTED]" } else { - out[k] = redactJSON(child, depth+1, sensitive.child(k)) + out[k] = redactJSON(child, depth+1) } } return out case []interface{}: out := make([]interface{}, len(val)) - items := sensitive.items() for i, child := range val { - out[i] = redactJSON(child, depth+1, items) + out[i] = redactJSON(child, depth+1) } return out case string: @@ -359,12 +271,12 @@ func parseJSON(body []byte) (interface{}, bool) { return parsed, true } -func decodeJSON(body []byte, sensitive sensitiveBodySchema) (interface{}, bool) { +func decodeJSON(body []byte) (interface{}, bool) { parsed, ok := parseJSON(body) if !ok { return nil, false } - return redactJSON(parsed, 0, sensitive), true + return redactJSON(parsed, 0), true } func isTextMediaType(mediaType string) bool { @@ -375,28 +287,25 @@ func isTextMediaType(mediaType string) bool { mediaType == "application/x-www-form-urlencoded" } -func previewBody(body []byte, contentType string, sensitive sensitiveBodySchema) interface{} { +func previewBody(body []byte, contentType string) interface{} { if len(body) == 0 { return nil } - if sensitive.isSensitive() { - return "[REDACTED]" - } mediaType, params, err := mime.ParseMediaType(contentType) if err != nil { mediaType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) } if isJSONMediaType(mediaType) || mediaType == "" { - if parsed, ok := decodeJSON(body, sensitive); ok { + if parsed, ok := decodeJSON(body); ok { return parsed } } if mediaType == "application/x-www-form-urlencoded" { - return previewForm(body, sensitive) + return previewForm(body) } if strings.HasPrefix(mediaType, "multipart/") { if boundary := params["boundary"]; boundary != "" { - if summary, ok := previewMultipart(body, boundary, sensitive); ok { + if summary, ok := previewMultipart(body, boundary); ok { return summary } } @@ -408,7 +317,28 @@ func previewBody(body []byte, contentType string, sensitive sensitiveBodySchema) return redactBase64String(string(body)) } -func previewForm(body []byte, sensitive sensitiveBodySchema) string { +// previewRequestBody masks the values the operation's schema marks sensitive +// before applying the name-based preview rules. +func previewRequestBody(req *http.Request, body []byte) interface{} { + contentType := req.Header.Get("Content-Type") + return previewRedactedBody(redact.RequestBody(req.Context(), contentType, body), contentType) +} + +func previewResponseBody(req *http.Request, resp *http.Response, body []byte) interface{} { + contentType := resp.Header.Get("Content-Type") + return previewRedactedBody(redact.ResponseBody(req.Context(), contentType, body), contentType) +} + +// previewRedactedBody shows a body masked as a whole as the mask itself, +// since it no longer parses as its content type. +func previewRedactedBody(body []byte, contentType string) interface{} { + if string(body) == redact.Mask { + return redact.Mask + } + return previewBody(body, contentType) +} + +func previewForm(body []byte) string { raw := string(body) if _, err := url.ParseQuery(raw); err != nil { return fmt.Sprintf("", len(body)) @@ -422,16 +352,12 @@ func previewForm(body []byte, sensitive sensitiveBodySchema) string { parts[i] = key + "=[REDACTED]" continue } - if isSensitiveName(decodedKey) || isSensitiveEncodedField(sensitive, decodedKey) { + if isSensitiveName(decodedKey) { parts[i] = key + "=[REDACTED]" continue } - if nested := sensitive.child(decodedKey); len(nested.Roots) > 0 { - if parsed, ok := parseJSON([]byte(decodedValue)); ok { - redacted := redactJSON(parsed, 0, nested) - if reflect.DeepEqual(parsed, redacted) { - continue - } + if parsed, ok := parseJSON([]byte(decodedValue)); ok { + if redacted := redactJSON(parsed, 0); !reflect.DeepEqual(parsed, redacted) { if encoded, err := encodeJSON(redacted, ""); err == nil { parts[i] = key + "=" + url.QueryEscape(encoded) continue @@ -448,7 +374,7 @@ func previewForm(body []byte, sensitive sensitiveBodySchema) string { return strings.Join(parts, "&") } -func previewMultipart(body []byte, boundary string, sensitive sensitiveBodySchema) (string, bool) { +func previewMultipart(body []byte, boundary string) (string, bool) { r := multipart.NewReader(bytes.NewReader(body), boundary) var lines []string for { @@ -471,8 +397,8 @@ func previewMultipart(body []byte, boundary string, sensitive sensitiveBodySchem lines = append(lines, fmt.Sprintf("%s: ", name, len(data))) continue } - value := previewBody(data, part.Header.Get("Content-Type"), sensitive.child(name)) - if isSensitiveName(name) || isSensitiveEncodedField(sensitive, name) { + value := previewBody(data, part.Header.Get("Content-Type")) + if isSensitiveName(name) { value = "[REDACTED]" } lines = append(lines, fmt.Sprintf("%s: %s", name, formatBodyPreview(value, 0))) @@ -565,9 +491,8 @@ func redactURL(u string) string { // DebugClient wraps an HTTP client and logs request/response diagnostics to stderr. type DebugClient struct { - Inner HTTPClient - Stderr io.Writer - SensitiveBodyFields sensitiveBodySchema + Inner HTTPClient + Stderr io.Writer } // HTTPClient is the interface that SDK clients implement. @@ -586,7 +511,7 @@ func (c *DebugClient) Do(req *http.Request) (*http.Response, error) { if readErr != nil { fmt.Fprintf(c.Stderr, "[DEBUG] Request Body Read Error: %v\n", readErr) } else { - body := previewBody(bodyData, req.Header.Get("Content-Type"), c.SensitiveBodyFields) + body := previewRequestBody(req, bodyData) fmt.Fprintf(c.Stderr, "[DEBUG] Request Body:\n %s\n", formatBodyPreview(body, maxBodyPreview)) } } @@ -611,7 +536,7 @@ func (c *DebugClient) Do(req *http.Request) (*http.Response, error) { if readErr != nil { fmt.Fprintf(c.Stderr, "[DEBUG] Response Body Read Error: %v\n", readErr) } else if len(bodyData) > 0 { - body := previewBody(bodyData, resp.Header.Get("Content-Type"), sensitiveBodySchema{}) + body := previewResponseBody(req, resp, bodyData) fmt.Fprintf(c.Stderr, "[DEBUG] Response Body:\n %s\n", formatBodyPreview(body, maxBodyPreview)) } } @@ -636,11 +561,10 @@ func isStreamingResponse(resp *http.Response) bool { // DryRunClient intercepts HTTP requests and returns a synthetic response // without making any network calls. type DryRunClient struct { - Stderr io.Writer - Stdout io.Writer - JSON bool - Cmd *cobra.Command - SensitiveBodyFields sensitiveBodySchema + Stderr io.Writer + Stdout io.Writer + JSON bool + Cmd *cobra.Command } type dryRunBody struct{ length int64 } @@ -677,7 +601,7 @@ func (c *DryRunClient) Do(req *http.Request) (*http.Response, error) { if err != nil { return nil, fmt.Errorf("read request body for dry-run preview: %w", err) } - body = previewBody(bodyData, req.Header.Get("Content-Type"), c.SensitiveBodyFields) + body = previewRequestBody(req, bodyData) } requestURL := redactURL(req.URL.String()) headers := redactHeaders(req.Header) @@ -816,10 +740,10 @@ func dryRunResponseShape(accept string) (string, []byte) { func WrapClientForDiagnostics(cmd *cobra.Command, inner HTTPClient) HTTPClient { stderr := cmd.ErrOrStderr() if IsDryRun(cmd) { - return &DryRunClient{Stderr: stderr, Stdout: cmd.OutOrStdout(), JSON: IsJSONDryRun(cmd), Cmd: cmd, SensitiveBodyFields: sensitiveBodyFieldsFor(cmd)} + return &DryRunClient{Stderr: stderr, Stdout: cmd.OutOrStdout(), JSON: IsJSONDryRun(cmd), Cmd: cmd} } if IsDebug(cmd) { - return &DebugClient{Inner: inner, Stderr: stderr, SensitiveBodyFields: sensitiveBodyFieldsFor(cmd)} + return &DebugClient{Inner: inner, Stderr: stderr} } return inner } diff --git a/templates/templates/cli/includes/security.ts b/templates/templates/cli/includes/security.ts index deba583b2..fcc491da3 100644 --- a/templates/templates/cli/includes/security.ts +++ b/templates/templates/cli/includes/security.ts @@ -369,153 +369,6 @@ registerTemplateFunc( templateSensitiveSecurityHeaders, ); -function isSensitiveBodyType(type: TypeDef): boolean { - const marker = type.Extensions?.All?.["x-speakeasy-param-sensitive"]; - return marker === true || marker === "true" || type.Format === "password"; -} - -function requestBodyType(op: Operation): TypeDef | undefined { - const body = getRequestBody(op); - if (body?.Type === "request") return body.Value.RequestBody?.Type; - return body?.Value.Type; -} - -interface CLISensitiveBodyNode { - Sensitive?: boolean; - Fields?: Map; - Item?: number; - Values?: number; - Variants?: number[]; -} - -interface CLISensitiveBodySchema { - Nodes: CLISensitiveBodyNode[]; - Roots: number[]; -} - -function renderSensitiveBodySchema( - schema: CLISensitiveBodySchema, -): string | undefined { - const reachable = new Set(); - schema.Nodes.forEach((node, i) => { - if (node.Sensitive) reachable.add(i); - }); - let changed = true; - while (changed) { - changed = false; - schema.Nodes.forEach((node, i) => { - if (reachable.has(i)) return; - const children = [ - ...(node.Fields?.values() || []), - ...(node.Variants || []), - node.Item || 0, - node.Values || 0, - ]; - if (children.some((child) => reachable.has(child))) { - reachable.add(i); - changed = true; - } - }); - } - const roots = schema.Roots.filter((root) => reachable.has(root)); - if (roots.length === 0) return undefined; - const indices = [...reachable].sort((a, b) => a - b); - const remap = new Map(indices.map((id, i) => [id, i + 1])); - const nodes = indices.map((id) => { - const node = schema.Nodes[id]; - const fields: string[] = []; - if (node.Sensitive) fields.push("Sensitive: true"); - const values = node.Values && reachable.has(node.Values) ? node.Values : 0; - const children = [...(node.Fields || [])] - .filter(([, child]) => values || reachable.has(child)) - .map( - ([name, child]) => - `"${escapeGoString(name)}": ${remap.get(child) || 0}`, - ); - if (children.length) - fields.push(`Fields: map[string]int{ ${children.join(", ")} }`); - if (node.Item && reachable.has(node.Item)) - fields.push(`Item: ${remap.get(node.Item)}`); - if (values) fields.push(`Values: ${remap.get(values)}`); - const variants = (node.Variants || []).filter((child) => - reachable.has(child), - ); - if (variants.length) - fields.push( - `Variants: []int{ ${variants - .map((child) => remap.get(child)) - .join(", ")} }`, - ); - return `{ ${fields.join(", ")} }`; - }); - // Spaced braces: this file renders twice (recurse), so "{{" must not appear. - return `{ Nodes: []sensitiveBodyNode{ {}, ${nodes.join( - ", ", - )} }, Roots: []int{ ${roots.map((id) => remap.get(id)).join(", ")} } }`; -} - -function templateSensitiveBodyFields(): { - OperationID: string; - Schema: string; -}[] { - const operations = new Map(); - for (const op of allOperations()) { - const schema = operations.get(op.OriginalID) || { Nodes: [{}], Roots: [] }; - operations.set(op.OriginalID, schema); - const walked = new Map(); - const walk = (type: TypeDef | undefined): number => { - if (!type) return 0; - const typeID = type.IsCustomType() ? getUniqueID(type) : ""; - if (typeID && walked.has(typeID)) return walked.get(typeID)!; - const id = schema.Nodes.length; - const node: CLISensitiveBodyNode = {}; - schema.Nodes.push(node); - if (typeID) walked.set(typeID, id); - if (isSensitiveBodyType(type)) { - node.Sensitive = true; - return id; - } - switch (type.Type.toString()) { - case "class": - node.Fields = new Map(); - for (const field of type.Fields || []) { - if (field.IsAdditionalProperties) - node.Values = walk(field.Type.ItemType); - else - node.Fields.set( - originalFieldName(field), - field.Const ? 0 : walk(field.Type), - ); - } - break; - case "array": - case "set": - node.Item = walk(type.ItemType); - break; - case "map": - node.Values = walk(type.ItemType); - break; - case "union": - node.Variants = (type.AssociatedTypes || []).map(walk); - break; - } - return id; - }; - schema.Roots.push(walk(requestBodyType(op))); - } - const entries: { OperationID: string; Schema: string }[] = []; - for (const [operationID, schema] of operations) { - const rendered = renderSensitiveBodySchema(schema); - if (rendered) entries.push({ OperationID: operationID, Schema: rendered }); - } - return entries.sort((a, b) => a.OperationID.localeCompare(b.OperationID)); -} - -registerTemplateFunc( - "templateSensitiveBodyFields", - templateSensitiveBodyFields, -); - function primaryCLISecurityField( fields: CLISecurityFieldInfo[], ): CLISecurityFieldInfo | undefined { diff --git a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl index c240f0ac0..6a677c63c 100644 --- a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl +++ b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl @@ -10,6 +10,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "{{getGolangPackage}}/internal/sdk/redact" ) // newDiagnosticsServer creates a test server that echoes requests and includes @@ -375,9 +377,9 @@ func TestDiagnosticsRedactSensitiveBodyFields(t *testing.T) { }{ { name: "split json body", - args: []string{"request-bodies", "cli-sensitive-split-body-post-json", "--json-value", "canary-json", "--name", "visible-name"}, + args: []string{"request-bodies", "cli-sensitive-split-body-post-json", "--json-value", "canary-json", "--name", "visible-name", "--form-value", "visible-form-value"}, secrets: []string{"canary-json"}, - visible: []string{"visible-name"}, + visible: []string{"visible-name", "visible-form-value"}, }, { name: "split multipart body", @@ -395,25 +397,25 @@ func TestDiagnosticsRedactSensitiveBodyFields(t *testing.T) { name: "whole json body", args: []string{"request-bodies", "cli-sensitive-whole-json-post", "--value", "canary-whole-json"}, secrets: []string{"canary-whole-json"}, - visible: []string{"[REDACTED]"}, + visible: []string{redact.Mask}, }, { name: "whole form body", args: []string{"request-bodies", "cli-sensitive-whole-form-post", "--value", "canary-whole-form"}, secrets: []string{"canary-whole-form"}, - visible: []string{"[REDACTED]"}, + visible: []string{redact.Mask}, }, { name: "whole multipart body", args: []string{"request-bodies", "cli-sensitive-whole-multipart-post", "--value", "canary-whole-multipart"}, secrets: []string{"canary-whole-multipart"}, - visible: []string{"[REDACTED]"}, + visible: []string{redact.Mask}, }, { name: "whole text body", args: []string{"request-bodies", "cli-sensitive-whole-text-post", "--request", `"canary-whole-text"`}, secrets: []string{"canary-whole-text"}, - visible: []string{"[REDACTED]"}, + visible: []string{redact.Mask}, }, { name: "recursive union", @@ -495,21 +497,21 @@ func TestDiagnosticsRedactSensitiveBodyFields(t *testing.T) { { name: "multipart parts", args: []string{"request-bodies", "cli-sensitive-multipart-post", "--name", "visible-name", "--value", "canary-top", - "--config-param", `{"label":"visible-label","value":"canary-config"}`, + "--config-param", `{"label":"visible-label","value":"canary-config","password":"canary-config-password"}`, "--codes", "canary-code-1", "--codes", "canary-code-2", "--entries", `[{"label":"visible-entry","value":"canary-entry"}]`, "--tree", `{"value":"canary-tree","child":{"value":"canary-tree-child","plain":{"name":"visible-name","value":"visible-tree-plain"}}}`}, - secrets: []string{"canary-top", "canary-config", "canary-code-1", "canary-code-2", "canary-entry", "canary-tree", "canary-tree-child"}, + secrets: []string{"canary-top", "canary-config", "canary-config-password", "canary-code-1", "canary-code-2", "canary-entry", "canary-tree", "canary-tree-child"}, visible: []string{"visible-name", "visible-label", "visible-entry", "visible-tree-plain"}, }, { name: "form fields", args: []string{"request-bodies", "cli-sensitive-form-put", "--name", "visible-name", "--value", "canary-top", - "--config-param", `{"label":"visible-label","value":"canary-config"}`, + "--config-param", `{"label":"visible-label","value":"canary-config","password":"canary-config-password"}`, "--codes", "canary-code-1", "--codes", "canary-code-2", "--entries", `[{"label":"visible-entry","value":"canary-entry"}]`, "--tree", `{"value":"canary-tree","child":{"value":"canary-tree-child","plain":{"name":"visible-name","value":"visible-tree-plain"}}}`}, - secrets: []string{"canary-top", "canary-config", "canary-code-1", "canary-code-2", "canary-entry", "canary-tree", "canary-tree-child"}, + secrets: []string{"canary-top", "canary-config", "canary-config-password", "canary-code-1", "canary-code-2", "canary-entry", "canary-tree", "canary-tree-child"}, visible: []string{"visible-name", "visible-label", "visible-entry", "visible-tree-plain"}, }, { @@ -559,10 +561,24 @@ func TestDiagnosticsKeepUnmarkedFieldSharingSensitiveName(t *testing.T) { output := mode.run(t, []string{"request-bodies", "cli-plain-body-fields-post", "--name", "visible-name", "--value", "visible-value"}) assert.Contains(t, output, "visible-value") + assert.NotContains(t, output, redact.Mask) assert.NotRegexp(t, `"value":\s*"\[REDACTED\]"`, output) }) } } + +func TestDebugRedactsSensitiveResponseFields(t *testing.T) { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"name":"visible-name","value":"canary-response"}`)) + })) + defer s.Close() + h := NewCLITestHarness(t) + require.NoError(t, h.RunWithServer(s.URL, []string{"--debug", "--output-format", "json", "response-bodies", "cli-sensitive-response-get"})) + assert.NotContains(t, h.GetStderr(), "canary-response") + assert.Contains(t, h.GetStderr(), "visible-name") + assert.Contains(t, h.GetStdout(), "canary-response", "decoded output keeps the value") +} {{- if hasRootVersionFlag}} func TestRootVersionFlagMatchesVersionCommand(t *testing.T) { diff --git a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml index cbe28e25f..60af15fc6 100644 --- a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml +++ b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml @@ -133,6 +133,9 @@ paths: jsonValue: type: string x-speakeasy-param-sensitive: true + formValue: + description: Not marked sensitive in this content type, unlike its form counterpart. + type: string name: type: string multipart/form-data: @@ -268,6 +271,23 @@ paths: responses: "200": description: OK + /anything/cli/sensitiveResponse: + get: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveResponseGet + description: >- + Response body with a field marked sensitive beside an unmarked one. + Debug output must redact the marked value in the logged response while + keeping its sibling readable. + tags: + - responseBodies + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/CliSensitiveResponse" components: schemas: CliSensitiveBodyFields: @@ -377,6 +397,9 @@ components: value: type: string format: password + password: + description: Not marked sensitive; its name alone requires redaction. + type: string codes: type: array items: @@ -534,3 +557,14 @@ components: properties: label: type: string + CliSensitiveResponse: + type: object + required: + - name + - value + properties: + name: + type: string + value: + type: string + x-speakeasy-param-sensitive: true diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index e23dc53b7..28af1a5e0 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -222,7 +222,7 @@ trackedFiles: internal/client/client.go: last_write_checksum: sha1:bca25435230238b7194798699e137ef94d983844 internal/client/diagnostics.go: - last_write_checksum: sha1:3feaa82c501ed582d2754de8aba1e89b8dc55a2d + last_write_checksum: sha1:3f9ea9a986c0a6bf7a487bb53a0eabc36bdaefc1 internal/clierrors/clierrors.go: last_write_checksum: sha1:e17fb3e33e439c6997834182011540427d760857 internal/config/config.go: diff --git a/zSDKs/sdk-cli/internal/client/diagnostics.go b/zSDKs/sdk-cli/internal/client/diagnostics.go index 74b111c36..c92049cd7 100644 --- a/zSDKs/sdk-cli/internal/client/diagnostics.go +++ b/zSDKs/sdk-cli/internal/client/diagnostics.go @@ -20,6 +20,7 @@ import ( "openapi/internal/config" "openapi/internal/flagutil" + "openapi/internal/sdk/redact" ) // maxBodyPreview is the maximum number of bytes to show in body previews. @@ -84,21 +85,6 @@ var sensitiveJSONKeys = map[string]bool{ "clientsecret": true, } -type sensitiveBodyNode struct { - Sensitive bool - Fields map[string]int - Item int - Values int - Variants []int -} - -type sensitiveBodySchema struct { - Nodes []sensitiveBodyNode - Roots []int -} - -var sensitiveBodyFields = map[string]sensitiveBodySchema{} - var sensitiveQueryKeys = map[string]bool{ "apikey": true, "accesstoken": true, @@ -176,80 +162,11 @@ func isSensitiveName(key string) bool { return false } -func sensitiveBodyFieldsFor(cmd *cobra.Command) sensitiveBodySchema { - if cmd == nil { - return sensitiveBodySchema{} - } - return sensitiveBodyFields[cmd.Annotations["speakeasy_operation"]] -} - -func (s sensitiveBodySchema) expandedRoots() []int { - var roots []int - seen := make(map[int]bool) - var visit func(int) - visit = func(id int) { - if id <= 0 || id >= len(s.Nodes) || seen[id] { - return - } - seen[id] = true - roots = append(roots, id) - for _, variant := range s.Nodes[id].Variants { - visit(variant) - } - } - for _, root := range s.Roots { - visit(root) - } - return roots -} - -func (s sensitiveBodySchema) isSensitive() bool { - for _, id := range s.expandedRoots() { - if s.Nodes[id].Sensitive { - return true - } - } - return false -} - -func (s sensitiveBodySchema) child(name string) sensitiveBodySchema { - var roots []int - for _, id := range s.expandedRoots() { - node := s.Nodes[id] - child, declared := node.Fields[name] - if !declared { - child = node.Values - } - if child != 0 { - roots = append(roots, child) - } - } - return sensitiveBodySchema{Nodes: s.Nodes, Roots: roots} -} - -func (s sensitiveBodySchema) items() sensitiveBodySchema { - var roots []int - for _, id := range s.expandedRoots() { - if item := s.Nodes[id].Item; item != 0 { - roots = append(roots, item) - } - } - return sensitiveBodySchema{Nodes: s.Nodes, Roots: roots} -} - -func isSensitiveEncodedField(sensitive sensitiveBodySchema, name string) bool { - nested := sensitive.child(name) - return nested.isSensitive() || nested.items().isSensitive() -} - // redactJSON recursively redacts sensitive keys in a JSON structure. -func redactJSON(v interface{}, depth int, sensitive sensitiveBodySchema) interface{} { +func redactJSON(v interface{}, depth int) interface{} { if depth > maxRedactDepth { return redactionDepthMarker } - if sensitive.isSensitive() { - return "[REDACTED]" - } switch val := v.(type) { case map[string]interface{}: out := make(map[string]interface{}, len(val)) @@ -257,15 +174,14 @@ func redactJSON(v interface{}, depth int, sensitive sensitiveBodySchema) interfa if sensitiveJSONKeys[normalizeSensitiveKey(k)] { out[k] = "[REDACTED]" } else { - out[k] = redactJSON(child, depth+1, sensitive.child(k)) + out[k] = redactJSON(child, depth+1) } } return out case []interface{}: out := make([]interface{}, len(val)) - items := sensitive.items() for i, child := range val { - out[i] = redactJSON(child, depth+1, items) + out[i] = redactJSON(child, depth+1) } return out case string: @@ -354,12 +270,12 @@ func parseJSON(body []byte) (interface{}, bool) { return parsed, true } -func decodeJSON(body []byte, sensitive sensitiveBodySchema) (interface{}, bool) { +func decodeJSON(body []byte) (interface{}, bool) { parsed, ok := parseJSON(body) if !ok { return nil, false } - return redactJSON(parsed, 0, sensitive), true + return redactJSON(parsed, 0), true } func isTextMediaType(mediaType string) bool { @@ -370,28 +286,25 @@ func isTextMediaType(mediaType string) bool { mediaType == "application/x-www-form-urlencoded" } -func previewBody(body []byte, contentType string, sensitive sensitiveBodySchema) interface{} { +func previewBody(body []byte, contentType string) interface{} { if len(body) == 0 { return nil } - if sensitive.isSensitive() { - return "[REDACTED]" - } mediaType, params, err := mime.ParseMediaType(contentType) if err != nil { mediaType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) } if isJSONMediaType(mediaType) || mediaType == "" { - if parsed, ok := decodeJSON(body, sensitive); ok { + if parsed, ok := decodeJSON(body); ok { return parsed } } if mediaType == "application/x-www-form-urlencoded" { - return previewForm(body, sensitive) + return previewForm(body) } if strings.HasPrefix(mediaType, "multipart/") { if boundary := params["boundary"]; boundary != "" { - if summary, ok := previewMultipart(body, boundary, sensitive); ok { + if summary, ok := previewMultipart(body, boundary); ok { return summary } } @@ -403,7 +316,28 @@ func previewBody(body []byte, contentType string, sensitive sensitiveBodySchema) return redactBase64String(string(body)) } -func previewForm(body []byte, sensitive sensitiveBodySchema) string { +// previewRequestBody masks the values the operation's schema marks sensitive +// before applying the name-based preview rules. +func previewRequestBody(req *http.Request, body []byte) interface{} { + contentType := req.Header.Get("Content-Type") + return previewRedactedBody(redact.RequestBody(req.Context(), contentType, body), contentType) +} + +func previewResponseBody(req *http.Request, resp *http.Response, body []byte) interface{} { + contentType := resp.Header.Get("Content-Type") + return previewRedactedBody(redact.ResponseBody(req.Context(), contentType, body), contentType) +} + +// previewRedactedBody shows a body masked as a whole as the mask itself, +// since it no longer parses as its content type. +func previewRedactedBody(body []byte, contentType string) interface{} { + if string(body) == redact.Mask { + return redact.Mask + } + return previewBody(body, contentType) +} + +func previewForm(body []byte) string { raw := string(body) if _, err := url.ParseQuery(raw); err != nil { return fmt.Sprintf("", len(body)) @@ -417,16 +351,12 @@ func previewForm(body []byte, sensitive sensitiveBodySchema) string { parts[i] = key + "=[REDACTED]" continue } - if isSensitiveName(decodedKey) || isSensitiveEncodedField(sensitive, decodedKey) { + if isSensitiveName(decodedKey) { parts[i] = key + "=[REDACTED]" continue } - if nested := sensitive.child(decodedKey); len(nested.Roots) > 0 { - if parsed, ok := parseJSON([]byte(decodedValue)); ok { - redacted := redactJSON(parsed, 0, nested) - if reflect.DeepEqual(parsed, redacted) { - continue - } + if parsed, ok := parseJSON([]byte(decodedValue)); ok { + if redacted := redactJSON(parsed, 0); !reflect.DeepEqual(parsed, redacted) { if encoded, err := encodeJSON(redacted, ""); err == nil { parts[i] = key + "=" + url.QueryEscape(encoded) continue @@ -443,7 +373,7 @@ func previewForm(body []byte, sensitive sensitiveBodySchema) string { return strings.Join(parts, "&") } -func previewMultipart(body []byte, boundary string, sensitive sensitiveBodySchema) (string, bool) { +func previewMultipart(body []byte, boundary string) (string, bool) { r := multipart.NewReader(bytes.NewReader(body), boundary) var lines []string for { @@ -466,8 +396,8 @@ func previewMultipart(body []byte, boundary string, sensitive sensitiveBodySchem lines = append(lines, fmt.Sprintf("%s: ", name, len(data))) continue } - value := previewBody(data, part.Header.Get("Content-Type"), sensitive.child(name)) - if isSensitiveName(name) || isSensitiveEncodedField(sensitive, name) { + value := previewBody(data, part.Header.Get("Content-Type")) + if isSensitiveName(name) { value = "[REDACTED]" } lines = append(lines, fmt.Sprintf("%s: %s", name, formatBodyPreview(value, 0))) @@ -560,9 +490,8 @@ func redactURL(u string) string { // DebugClient wraps an HTTP client and logs request/response diagnostics to stderr. type DebugClient struct { - Inner HTTPClient - Stderr io.Writer - SensitiveBodyFields sensitiveBodySchema + Inner HTTPClient + Stderr io.Writer } // HTTPClient is the interface that SDK clients implement. @@ -581,7 +510,7 @@ func (c *DebugClient) Do(req *http.Request) (*http.Response, error) { if readErr != nil { fmt.Fprintf(c.Stderr, "[DEBUG] Request Body Read Error: %v\n", readErr) } else { - body := previewBody(bodyData, req.Header.Get("Content-Type"), c.SensitiveBodyFields) + body := previewRequestBody(req, bodyData) fmt.Fprintf(c.Stderr, "[DEBUG] Request Body:\n %s\n", formatBodyPreview(body, maxBodyPreview)) } } @@ -606,7 +535,7 @@ func (c *DebugClient) Do(req *http.Request) (*http.Response, error) { if readErr != nil { fmt.Fprintf(c.Stderr, "[DEBUG] Response Body Read Error: %v\n", readErr) } else if len(bodyData) > 0 { - body := previewBody(bodyData, resp.Header.Get("Content-Type"), sensitiveBodySchema{}) + body := previewResponseBody(req, resp, bodyData) fmt.Fprintf(c.Stderr, "[DEBUG] Response Body:\n %s\n", formatBodyPreview(body, maxBodyPreview)) } } @@ -631,11 +560,10 @@ func isStreamingResponse(resp *http.Response) bool { // DryRunClient intercepts HTTP requests and returns a synthetic response // without making any network calls. type DryRunClient struct { - Stderr io.Writer - Stdout io.Writer - JSON bool - Cmd *cobra.Command - SensitiveBodyFields sensitiveBodySchema + Stderr io.Writer + Stdout io.Writer + JSON bool + Cmd *cobra.Command } type dryRunBody struct{ length int64 } @@ -672,7 +600,7 @@ func (c *DryRunClient) Do(req *http.Request) (*http.Response, error) { if err != nil { return nil, fmt.Errorf("read request body for dry-run preview: %w", err) } - body = previewBody(bodyData, req.Header.Get("Content-Type"), c.SensitiveBodyFields) + body = previewRequestBody(req, bodyData) } requestURL := redactURL(req.URL.String()) headers := redactHeaders(req.Header) @@ -811,10 +739,10 @@ func dryRunResponseShape(accept string) (string, []byte) { func WrapClientForDiagnostics(cmd *cobra.Command, inner HTTPClient) HTTPClient { stderr := cmd.ErrOrStderr() if IsDryRun(cmd) { - return &DryRunClient{Stderr: stderr, Stdout: cmd.OutOrStdout(), JSON: IsJSONDryRun(cmd), Cmd: cmd, SensitiveBodyFields: sensitiveBodyFieldsFor(cmd)} + return &DryRunClient{Stderr: stderr, Stdout: cmd.OutOrStdout(), JSON: IsJSONDryRun(cmd), Cmd: cmd} } if IsDebug(cmd) { - return &DebugClient{Inner: inner, Stderr: stderr, SensitiveBodyFields: sensitiveBodyFieldsFor(cmd)} + return &DebugClient{Inner: inner, Stderr: stderr} } return inner } From 27111d873d86eca91c7b8a5fd9d6fe22e77bd7ec Mon Sep 17 00:00:00 2001 From: 2ynn Date: Thu, 8 Oct 2026 15:31:01 -0400 Subject: [PATCH 23/23] test(cli): cover a sensitive form and multipart property whose name ends in brackets --- .../templates/cli/tests/primary/diagnostics_test.go.stmpl | 8 ++++---- .../fragments/primary/cli-sensitive-body-fields.yaml | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl index 6a677c63c..8f2a1960c 100644 --- a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl +++ b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl @@ -496,22 +496,22 @@ func TestDiagnosticsRedactSensitiveBodyFields(t *testing.T) { }, { name: "multipart parts", - args: []string{"request-bodies", "cli-sensitive-multipart-post", "--name", "visible-name", "--value", "canary-top", + args: []string{"request-bodies", "cli-sensitive-multipart-post", "--name", "visible-name", "--value", "canary-top", "--bracketed-payload", "canary-brackets", "--config-param", `{"label":"visible-label","value":"canary-config","password":"canary-config-password"}`, "--codes", "canary-code-1", "--codes", "canary-code-2", "--entries", `[{"label":"visible-entry","value":"canary-entry"}]`, "--tree", `{"value":"canary-tree","child":{"value":"canary-tree-child","plain":{"name":"visible-name","value":"visible-tree-plain"}}}`}, - secrets: []string{"canary-top", "canary-config", "canary-config-password", "canary-code-1", "canary-code-2", "canary-entry", "canary-tree", "canary-tree-child"}, + secrets: []string{"canary-top", "canary-brackets", "canary-config", "canary-config-password", "canary-code-1", "canary-code-2", "canary-entry", "canary-tree", "canary-tree-child"}, visible: []string{"visible-name", "visible-label", "visible-entry", "visible-tree-plain"}, }, { name: "form fields", - args: []string{"request-bodies", "cli-sensitive-form-put", "--name", "visible-name", "--value", "canary-top", + args: []string{"request-bodies", "cli-sensitive-form-put", "--name", "visible-name", "--value", "canary-top", "--bracketed-payload", "canary-brackets", "--config-param", `{"label":"visible-label","value":"canary-config","password":"canary-config-password"}`, "--codes", "canary-code-1", "--codes", "canary-code-2", "--entries", `[{"label":"visible-entry","value":"canary-entry"}]`, "--tree", `{"value":"canary-tree","child":{"value":"canary-tree-child","plain":{"name":"visible-name","value":"visible-tree-plain"}}}`}, - secrets: []string{"canary-top", "canary-config", "canary-config-password", "canary-code-1", "canary-code-2", "canary-entry", "canary-tree", "canary-tree-child"}, + secrets: []string{"canary-top", "canary-brackets", "canary-config", "canary-config-password", "canary-code-1", "canary-code-2", "canary-entry", "canary-tree", "canary-tree-child"}, visible: []string{"visible-name", "visible-label", "visible-entry", "visible-tree-plain"}, }, { diff --git a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml index 60af15fc6..645b47ba7 100644 --- a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml +++ b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml @@ -384,6 +384,11 @@ components: - name - value properties: + payload[]: + description: Property whose wire name ends in brackets; it is not an array. + type: string + x-speakeasy-name-override: bracketedPayload + x-speakeasy-param-sensitive: true name: type: string value: