diff --git a/.changesets/1790872792-577bf8cd.yaml b/.changesets/1790872792-577bf8cd.yaml new file mode 100644 index 000000000..376587221 --- /dev/null +++ b/.changesets/1790872792-577bf8cd.yaml @@ -0,0 +1,10 @@ +id: 1790872792-577bf8cd +features: + - core +targets: + - cli +type: fix +bump: patch +description: '--dry-run and --debug mask request and response body values marked sensitive (x-speakeasy-param-sensitive or format: password) through the SDK redact package, including whole bodies, recursive schemas and form or multipart fields' +author: 2ynn +date: "2026-10-01" diff --git a/.changesets/1791413358-125ef0df.yaml b/.changesets/1791413358-125ef0df.yaml new file mode 100644 index 000000000..ca7614aeb --- /dev/null +++ b/.changesets/1791413358-125ef0df.yaml @@ -0,0 +1,12 @@ +id: 1791413358-125ef0df +features: + - core +targets: + - cli + - go + - terraform +type: feat +bump: minor +description: mask sensitive request and response body values in HTTP logs and API error messages +author: ThomasRooney +date: "2026-10-07" diff --git a/internal/ast/sensitive_body.go b/internal/ast/sensitive_body.go new file mode 100644 index 000000000..08c5798ca --- /dev/null +++ b/internal/ast/sensitive_body.go @@ -0,0 +1,321 @@ +package ast + +import "sort" + +// SensitiveBodyGraph locates the values marked sensitive (x-speakeasy-param-sensitive +// or format: password) in the bodies of an operation, by their JSON location, +// so that generated code can mask them when bodies are logged. +// +// Nodes[0] is a sentinel meaning "nothing sensitive below here"; a node +// reference of 0 can be skipped. Roots are the body schemas: one per request +// content type, or one per response status code and content type. Only nodes +// that can reach a sensitive value are kept. +type SensitiveBodyGraph struct { + Nodes []*SensitiveBodyNode + Roots []int +} + +// SensitiveBodyNode is a schema in a SensitiveBodyGraph. +type SensitiveBodyNode struct { + // Sensitive masks the whole value. Streams are masked as a whole when any + // of their items holds a sensitive value. + Sensitive bool + // Fields are the declared object properties by wire name (JSON property, + // form field or multipart part name), sorted by name, including those + // holding nothing sensitive (with Node 0), so that a declared name is + // never mistaken for another one or for Values. A property that is not + // listed uses Values. + Fields []SensitiveBodyField + // Item is the schema of array items. + Item int + // Values is the schema of map values and additional properties. + Values int + // Variants are union members; the member a value matches is not known. + Variants []int + // PlainObject, PlainArray, PlainString, PlainNumber and PlainBoolean + // report union members accepting JSON values of that type and holding + // nothing sensitive, so a value of that type that matches none of + // Variants may still match the schema. + PlainObject bool + PlainArray bool + PlainString bool + PlainNumber bool + PlainBoolean bool +} + +// SensitiveBodyField is an object property of a SensitiveBodyNode. +type SensitiveBodyField struct { + Name string + Node int +} + +// SensitiveRequestBodyGraph returns the sensitive value locations of the +// operation's request body, or nil when it holds none. +func (o *Operation) SensitiveRequestBodyGraph() *SensitiveBodyGraph { + b := newSensitiveBodyBuilder() + b.addRoot(o.requestBodyType()) + return b.build() +} + +// SensitiveResponseBodyGraph returns the sensitive value locations of all the +// operation's response bodies, including errors, or nil when they hold none. +func (o *Operation) SensitiveResponseBodyGraph() *SensitiveBodyGraph { + b := newSensitiveBodyBuilder() + if o.Response != nil { + for _, response := range o.Response.Responses { + for _, content := range response.Content { + if content != nil && content.Content != nil { + b.addRoot(content.Content.Type) + } + } + } + } + return b.build() +} + +// HasSensitiveBodies reports whether the operation's request or response +// bodies hold values marked sensitive. +func (o *Operation) HasSensitiveBodies() bool { + return o.SensitiveRequestBodyGraph() != nil || o.SensitiveResponseBodyGraph() != nil +} + +func (o *Operation) requestBodyType() *TypeDef { + if o.Request == nil { + return nil + } + if o.Request.Field != nil && o.Request.Field.Type != nil { + for _, field := range o.Request.Field.Type.Fields { + if field.Annotations.Has(AnnotationTypeRequest) { + return field.Type + } + } + } + if o.Request.IsRequestBody && o.Request.RequestBody != nil { + return o.Request.RequestBody.Type + } + return nil +} + +func isSensitiveBodyType(t *TypeDef) bool { + if t.Format == "password" { + return true + } + if t.Extensions == nil { + return false + } + marker := t.Extensions.All["x-speakeasy-param-sensitive"] + return marker == true || marker == "true" +} + +type sensitiveBodyBuilder struct { + nodes []*sensitiveBodyBuildNode + roots []int +} + +type sensitiveBodyBuildNode struct { + sensitive bool + stream bool + fields []SensitiveBodyField + item int + values int + variants []int + // variantTypes are the schemas of variants, so that pruned variants + // still describe the shapes the union accepts. + variantTypes []*TypeDef +} + +func newSensitiveBodyBuilder() *sensitiveBodyBuilder { + return &sensitiveBodyBuilder{nodes: []*sensitiveBodyBuildNode{{}}} +} + +// addRoot walks a body schema along the edges given by TypeDef.Children. +// Custom types are walked once per root, by identity and by registration ID +// (which also matches the truncated copies that break recursive references), +// so that recursive schemas terminate. +func (b *sensitiveBodyBuilder) addRoot(t *TypeDef) { + if !containsSensitiveBodyType(t) { + return + } + walkedTypes := map[*TypeDef]int{} + walkedIDs := map[string]int{} + var walk func(t *TypeDef) int + walk = func(t *TypeDef) int { + if t == nil { + return 0 + } + if id, ok := walkedTypes[t]; ok { + return id + } + typeID := "" + if t.IsCustomType() && t.ContextStack != nil { + typeID = t.GetRegistrationID() + " name:" + t.Name + if id, ok := walkedIDs[typeID]; ok { + return id + } + } + id := len(b.nodes) + node := &sensitiveBodyBuildNode{} + b.nodes = append(b.nodes, node) + walkedTypes[t] = id + if typeID != "" { + walkedIDs[typeID] = id + } + if isSensitiveBodyType(t) { + node.sensitive = true + return id + } + for field, child := range t.Children { + switch { + case field == nil: + switch t.Type { + case DataTypeArray, DataTypeSet: + node.item = walk(child) + case DataTypeMap: + node.values = walk(child) + case DataTypeEventStream, DataTypeJsonL: + node.stream = true + node.item = walk(child) + case DataTypeUnion: + node.variants = append(node.variants, walk(child)) + node.variantTypes = append(node.variantTypes, child) + } + case t.Type != DataTypeClass && t.Type != DataTypeError: + case field.IsAdditionalProperties: + if child != nil { + node.values = walk(child.ItemType) + } + default: + node.fields = append(node.fields, SensitiveBodyField{Name: sensitiveBodyFieldName(field)}) + if field.Const == nil { + node.fields[len(node.fields)-1].Node = walk(child) + } + } + } + return id + } + b.roots = append(b.roots, walk(t)) +} + +func sensitiveBodyFieldName(field *FieldDef) string { + if field.OriginalName != "" { + return field.OriginalName + } + return field.Name +} + +// containsSensitiveBodyType reports whether t or any type below it is marked +// sensitive, so that schemas without sensitive values are not walked further. +func containsSensitiveBodyType(t *TypeDef) bool { + for _, typ := range t.Walk() { + if isSensitiveBodyType(typ) { + return true + } + } + return false +} + +// build prunes the nodes that cannot reach a sensitive value and renumbers +// the rest. +func (b *sensitiveBodyBuilder) build() *SensitiveBodyGraph { + reachable := make([]bool, len(b.nodes)) + for i, node := range b.nodes { + reachable[i] = node.sensitive + } + for changed := true; changed; { + changed = false + for i, node := range b.nodes { + if reachable[i] { + continue + } + children := append([]int{node.item, node.values}, node.variants...) + for _, field := range node.fields { + children = append(children, field.Node) + } + for _, child := range children { + if child != 0 && reachable[child] { + reachable[i] = true + changed = true + break + } + } + } + } + + remap := make([]int, len(b.nodes)) + graph := &SensitiveBodyGraph{Nodes: []*SensitiveBodyNode{{}}} + for i := 1; i < len(b.nodes); i++ { + if reachable[i] { + remap[i] = len(graph.Nodes) + graph.Nodes = append(graph.Nodes, &SensitiveBodyNode{}) + } + } + for _, root := range b.roots { + if root != 0 && reachable[root] { + graph.Roots = append(graph.Roots, remap[root]) + } + } + if len(graph.Roots) == 0 { + return nil + } + + for i, node := range b.nodes { + if i == 0 || !reachable[i] { + continue + } + out := graph.Nodes[remap[i]] + if node.sensitive || node.stream { + out.Sensitive = true + continue + } + out.Values = remap[node.values] + for _, field := range node.fields { + out.Fields = append(out.Fields, SensitiveBodyField{Name: field.Name, Node: remap[field.Node]}) + } + sort.SliceStable(out.Fields, func(a, b int) bool { return out.Fields[a].Name < out.Fields[b].Name }) + out.Item = remap[node.item] + for v, variant := range node.variants { + if remap[variant] != 0 { + out.Variants = append(out.Variants, remap[variant]) + } else { + out.addPlainShapes(node.variantTypes[v], map[*TypeDef]bool{}) + } + } + } + return graph +} + +// addPlainShapes records the JSON types of values that t, a union member +// holding nothing sensitive, accepts. Nested unions are followed once each, +// so that recursive unions terminate. +func (n *SensitiveBodyNode) addPlainShapes(t *TypeDef, visited map[*TypeDef]bool) { + if t == nil || visited[t] { + return + } + visited[t] = true + switch t.Type { + case DataTypeClass, DataTypeError, DataTypeMap: + n.PlainObject = true + case DataTypeArray, DataTypeSet: + n.PlainArray = true + case DataTypeUnion: + for _, member := range t.AssociatedTypes { + n.addPlainShapes(member, visited) + } + case DataTypeEnum: + if t.Enum != nil && t.Enum.Type != nil && t.Enum.Type.Type != DataTypeEnum { + n.addPlainShapes(t.Enum.Type, visited) + } else { + n.PlainString = true + } + case DataTypeAny: + n.PlainObject, n.PlainArray, n.PlainString, n.PlainNumber, n.PlainBoolean = true, true, true, true, true + case DataTypeBoolean: + n.PlainBoolean = true + case DataTypeInteger, DataTypeInt32, DataTypeNumber, DataTypeFloat32: + n.PlainNumber = true + case DataTypeBigInt, DataTypeDecimal: + n.PlainNumber, n.PlainString = true, true + case DataTypeString, DataTypeDate, DataTypeDateTime, DataTypeUUID, DataTypeDuration, DataTypeBytes: + n.PlainString = true + } +} diff --git a/internal/ast/sensitive_body_test.go b/internal/ast/sensitive_body_test.go new file mode 100644 index 000000000..0c9f1ca37 --- /dev/null +++ b/internal/ast/sensitive_body_test.go @@ -0,0 +1,246 @@ +package ast + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func sensitiveString() *TypeDef { + return &TypeDef{Type: DataTypeString, Extensions: &TypeDefExtensions{All: map[string]any{"x-speakeasy-param-sensitive": true}}} +} + +func plainString() *TypeDef { + return &TypeDef{Type: DataTypeString} +} + +func class(name string, fields ...*FieldDef) *TypeDef { + return &TypeDef{Type: DataTypeClass, Name: name, Fields: fields} +} + +func field(name string, typ *TypeDef) *FieldDef { + return &FieldDef{Name: name, OriginalName: name, Type: typ} +} + +func responseOperation(types ...*TypeDef) *Operation { + response := &SubResponse{Code: []string{"200"}} + for _, typ := range types { + response.Content = append(response.Content, &ResponseBodyContent{Content: &FieldDef{Type: typ}}) + } + return &Operation{BaseOperation: BaseOperation{Response: &Response{Responses: SubResponses{response}}}} +} + +// sensitiveChild follows a JSON property from the nodes, as a masker would. +func sensitiveChild(g *SensitiveBodyGraph, nodes []int, name string) []int { + var children []int + for _, id := range sensitiveVariants(g, nodes) { + node := g.Nodes[id] + child := node.Values + for _, f := range node.Fields { + if f.Name == name { + child = f.Node + } + } + if child != 0 { + children = append(children, child) + } + } + return children +} + +func sensitiveItems(g *SensitiveBodyGraph, nodes []int) []int { + var items []int + for _, id := range sensitiveVariants(g, nodes) { + if item := g.Nodes[id].Item; item != 0 { + items = append(items, item) + } + } + return items +} + +func sensitiveVariants(g *SensitiveBodyGraph, nodes []int) []int { + var out []int + seen := map[int]bool{} + var visit func(int) + visit = func(id int) { + if id == 0 || seen[id] { + return + } + seen[id] = true + out = append(out, id) + for _, v := range g.Nodes[id].Variants { + visit(v) + } + } + for _, id := range nodes { + visit(id) + } + return out +} + +func isSensitiveAt(g *SensitiveBodyGraph, nodes []int) bool { + for _, id := range sensitiveVariants(g, nodes) { + if g.Nodes[id].Sensitive { + return true + } + } + return false +} + +func TestSensitiveResponseBodyGraph(t *testing.T) { + t.Parallel() + + tree := class("Tree", field("label", plainString()), field("secret", sensitiveString())) + tree.Fields = append(tree.Fields, field("children", &TypeDef{Type: DataTypeArray, ItemType: tree})) + labels := class("Labels", field("visible", plainString()), &FieldDef{Name: "AdditionalProperties", IsAdditionalProperties: true, Type: &TypeDef{Type: DataTypeMap, ItemType: sensitiveString()}}) + password := &TypeDef{Type: DataTypeString, Format: "password"} + payload := class("Payload", + field("name", plainString()), + field("token", sensitiveString()), + &FieldDef{Name: "Password", OriginalName: "pass_word", Type: password}, + field("nested", class("Nested", field("label", plainString()), field("secret", sensitiveString()))), + field("tokens", &TypeDef{Type: DataTypeArray, ItemType: sensitiveString()}), + field("labels", labels), + field("credential", &TypeDef{Type: DataTypeUnion, Name: "Credential", AssociatedTypes: TypeDefs{ + class("APIKey", field("key", sensitiveString())), + class("Basic", field("user", plainString())), + }}), + field("tree", tree), + field("unrelated", class("Unrelated", field("secret", plainString()))), + ) + errorType := &TypeDef{Type: DataTypeError, Name: "Error", Fields: Fields{field("message", plainString()), field("api_key", sensitiveString())}} + + g := responseOperation(payload, errorType).SensitiveResponseBodyGraph() + require.NotNil(t, g) + require.Len(t, g.Roots, 2) + root := g.Roots + + at := func(nodes []int, path ...string) []int { + for _, step := range path { + if step == "[]" { + nodes = sensitiveItems(g, nodes) + } else { + nodes = sensitiveChild(g, nodes, step) + } + } + return nodes + } + + for _, path := range [][]string{ + {"token"}, + {"pass_word"}, + {"nested", "secret"}, + {"tokens", "[]"}, + {"labels", "undeclared"}, + {"credential", "key"}, + {"tree", "secret"}, + {"tree", "children", "[]", "children", "[]", "secret"}, + {"api_key"}, + } { + assert.True(t, isSensitiveAt(g, at(root, path...)), "%v should be sensitive", path) + } + for _, path := range [][]string{ + {"name"}, + {"Password"}, + {"nested", "label"}, + {"labels", "visible"}, + {"credential", "user"}, + {"tree", "label"}, + {"unrelated", "secret"}, + {"message"}, + } { + assert.False(t, isSensitiveAt(g, at(root, path...)), "%v should not be sensitive", path) + } +} + +func TestSensitiveBodyGraphStreamsAndAbsence(t *testing.T) { + t.Parallel() + + stream := &TypeDef{Type: DataTypeJsonL, ItemType: class("Event", field("token", sensitiveString()))} + g := responseOperation(stream).SensitiveResponseBodyGraph() + require.NotNil(t, g) + assert.True(t, isSensitiveAt(g, g.Roots), "a stream holding sensitive values is sensitive as a whole") + + plain := class("Plain", field("secret", plainString())) + assert.Nil(t, responseOperation(plain).SensitiveResponseBodyGraph()) + assert.Nil(t, responseOperation(&TypeDef{Type: DataTypeEventStream, ItemType: plain}).SensitiveResponseBodyGraph()) + assert.False(t, responseOperation(plain).HasSensitiveBodies()) +} + +func TestSensitiveRequestBodyGraph(t *testing.T) { + t.Parallel() + + body := class("Body", field("name", plainString()), field("secret", sensitiveString())) + direct := &Operation{BaseOperation: BaseOperation{Request: &Request{IsRequestBody: true, RequestBody: &FieldDef{Type: body}}}} + g := direct.SensitiveRequestBodyGraph() + require.NotNil(t, g) + assert.True(t, isSensitiveAt(g, sensitiveChild(g, g.Roots, "secret"))) + assert.True(t, direct.HasSensitiveBodies()) + + wrapped := &Operation{BaseOperation: BaseOperation{Request: &Request{Field: &FieldDef{Type: class("Request", + field("id", plainString()), + &FieldDef{Name: "Body", Type: body, Annotations: Annotations{&RequestAnnotation{}}}, + )}}}} + g = wrapped.SensitiveRequestBodyGraph() + require.NotNil(t, g) + assert.True(t, isSensitiveAt(g, sensitiveChild(g, g.Roots, "secret"))) + + assert.Nil(t, (&Operation{}).SensitiveRequestBodyGraph()) +} + +func TestSensitiveBodyGraphPlainUnionVariants(t *testing.T) { + t.Parallel() + + union := func(name string, variants ...*TypeDef) *TypeDef { + return &TypeDef{Type: DataTypeUnion, Name: name, AssociatedTypes: variants} + } + recursivePlain := union("RecursivePlain", plainString()) + recursivePlain.AssociatedTypes = append(recursivePlain.AssociatedTypes, recursivePlain) + body := class("Body", + field("mixed", union("Mixed", &TypeDef{Type: DataTypeArray, ItemType: sensitiveString()}, class("Label", field("label", plainString())))), + field("text", union("Text", class("APIKey", field("key", sensitiveString())), union("Scalars", plainString(), &TypeDef{Type: DataTypeInteger}))), + field("flag", union("Flag", class("APIKey", field("key", sensitiveString())), &TypeDef{Type: DataTypeBoolean})), + field("count", union("Count", class("APIKey", field("key", sensitiveString())), &TypeDef{Type: DataTypeInt32}, &TypeDef{Type: DataTypeFloat32})), + field("loop", union("Loop", class("APIKey", field("key", sensitiveString())), recursivePlain)), + field("sensitive", union("Sensitive", class("APIKey", field("key", sensitiveString())), sensitiveString())), + ) + g := responseOperation(body).SensitiveResponseBodyGraph() + require.NotNil(t, g) + + mixed := sensitiveChild(g, g.Roots, "mixed") + require.Len(t, mixed, 1) + assert.True(t, g.Nodes[mixed[0]].PlainObject, "a plain object member is recorded") + assert.False(t, g.Nodes[mixed[0]].PlainArray || g.Nodes[mixed[0]].PlainString || g.Nodes[mixed[0]].PlainNumber || g.Nodes[mixed[0]].PlainBoolean) + assert.Len(t, g.Nodes[mixed[0]].Variants, 1) + + declared := make([]string, 0, len(g.Nodes[g.Roots[0]].Fields)) + for _, f := range g.Nodes[g.Roots[0]].Fields { + declared = append(declared, f.Name) + } + assert.Equal(t, []string{"count", "flag", "loop", "mixed", "sensitive", "text"}, declared, "declared properties are listed even when they hold nothing sensitive") + + text := sensitiveChild(g, g.Roots, "text") + require.Len(t, text, 1) + assert.True(t, g.Nodes[text[0]].PlainString && g.Nodes[text[0]].PlainNumber, "members of a plain nested union are recorded") + assert.False(t, g.Nodes[text[0]].PlainObject || g.Nodes[text[0]].PlainArray || g.Nodes[text[0]].PlainBoolean) + + flag := sensitiveChild(g, g.Roots, "flag") + require.Len(t, flag, 1) + assert.True(t, g.Nodes[flag[0]].PlainBoolean) + assert.False(t, g.Nodes[flag[0]].PlainString || g.Nodes[flag[0]].PlainNumber, "a boolean member does not accept text or numbers") + + count := sensitiveChild(g, g.Roots, "count") + require.Len(t, count, 1) + assert.True(t, g.Nodes[count[0]].PlainNumber, "int32 and float32 members are numbers") + assert.False(t, g.Nodes[count[0]].PlainString) + + loop := sensitiveChild(g, g.Roots, "loop") + require.Len(t, loop, 1) + assert.True(t, g.Nodes[loop[0]].PlainString, "a recursive plain union is followed once") + + sensitive := sensitiveChild(g, g.Roots, "sensitive") + require.Len(t, sensitive, 1) + node := g.Nodes[sensitive[0]] + assert.False(t, node.PlainObject || node.PlainArray || node.PlainString || node.PlainNumber || node.PlainBoolean, "a union whose members all hold sensitive values has no plain member") +} diff --git a/templates/perms.go b/templates/perms.go index a58567ebe..cbd4e3ab9 100644 --- a/templates/perms.go +++ b/templates/perms.go @@ -556,6 +556,8 @@ var fileMode = map[string]fs.FileMode{ "templates/go/auxiliary/{{getPollingLocation}}/config.go.stmpl": 0o644, "templates/go/auxiliary/{{getPollingLocation}}/errors.go.stmpl": 0o644, "templates/go/auxiliary/{{getPollingLocation}}/option.go.stmpl": 0o644, + "templates/go/auxiliary/{{getRedactLocation}}": 0o20000000755, + "templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl": 0o644, "templates/go/auxiliary/{{getRetryLocation}}": 0o20000000755, "templates/go/auxiliary/{{getRetryLocation}}/config.go.stmpl": 0o644, "templates/go/auxiliary/{{getTypesLocation}}": 0o20000000755, @@ -617,6 +619,7 @@ var fileMode = map[string]fs.FileMode{ "templates/go/includes/operations.ts": 0o644, "templates/go/includes/options.ts": 0o644, "templates/go/includes/polling.ts": 0o644, + "templates/go/includes/redact.ts": 0o644, "templates/go/includes/responses.ts": 0o644, "templates/go/includes/retractions.ts": 0o644, "templates/go/includes/sanitization.ts": 0o644, @@ -690,6 +693,7 @@ var fileMode = map[string]fs.FileMode{ "templates/go/tests/primary/parameter_additional_test.go.stmpl": 0o644, "templates/go/tests/primary/parameter_encoding_additional_test.go.stmpl": 0o644, "templates/go/tests/primary/polling_additional_test.go.stmpl": 0o644, + "templates/go/tests/primary/redact_additional_test.go.stmpl": 0o644, "templates/go/tests/primary/requestbodies_additional_test.go.stmpl": 0o644, "templates/go/tests/primary/responsebodies_additional_test.go.stmpl": 0o644, "templates/go/tests/primary/retries_test.go.stmpl": 0o644, @@ -2150,6 +2154,7 @@ var fileMode = map[string]fs.FileMode{ "templates/terraform/tests/review/internal": 0o20000000755, "templates/terraform/tests/review/internal/provider": 0o20000000755, "templates/terraform/tests/review/internal/provider/http_client_test.go.stmpl": 0o644, + "templates/terraform/tests/review/internal/provider/sensitive_body_test.go.stmpl": 0o644, "templates/terraform/tests/review/internal/provider/sensitive_values_test.go.stmpl": 0o644, "templates/terraform/tests/review/internal/provider/utils_test.go.stmpl": 0o644, "templates/terraform/tsconfig.json": 0o644, diff --git a/templates/templates/cli/README.md b/templates/templates/cli/README.md index fe8baae85..317c79e27 100644 --- a/templates/templates/cli/README.md +++ b/templates/templates/cli/README.md @@ -1491,6 +1491,7 @@ cli users list --debug - **Headers**: Case-insensitive denylist (`authorization`, `cookie`, `set-cookie`, `x-api-key`, `x-session-token`, `*-secret`, `*-token` suffixes) → `[REDACTED]` - **URL query**: Credential parameters (`api_key`, `token`, `client_secret`, `signature`, etc.) are redacted without reordering the query +- **Body values marked sensitive**: request and response previews first pass through the embedded SDK's `redact` package (`redact.RequestBody` / `redact.ResponseBody`), which masks values marked `x-speakeasy-param-sensitive: true` or `format: password` with `redact.Mask`. The SDK stamps the operation ID on each request context, so the schema graph (built by `Operation.SensitiveRequestBodyGraph` / `SensitiveResponseBodyGraph`) is selected per SDK method, and split request content types each use their own graph. Form fields and multipart parts are matched by name and JSON-encoded fields are masked within; other media types, and bodies that do not match their schema, are masked as a whole. Union variants are not told apart: a field sensitive in one variant is masked in every variant declaring the same name. The marker must sit on the value's own schema: an extension next to a `$ref` is not read. The name-based rules below still apply afterwards - **JSON body**: Separator/case-normalized sensitive keys and canonical base64 strings (encoded length ≥128, excluding all-hex IDs) are redacted; numbers retain their exact lexemes and depth overflow becomes an explicit marker - **Other bodies**: Binary media becomes `` and multipart payloads become a deterministic inventory; human dry-run is not truncated. Debug applies the same redactor, then caps the rendered preview at 4 KiB - **Credentials**: Generated request construction uses request-scoped resolvers. Dry-run reads flags, environment and config, but never probes the OS keychain; whoami also uses the request-scoped resolvers (so `whoami --dry-run` skips the keychain), while configuration retains keychain access diff --git a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl index 67bd3f125..96fcc9464 100644 --- a/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl +++ b/templates/templates/cli/auxiliary/internal/client/diagnostics.go.stmpl @@ -11,6 +11,7 @@ import ( "mime/multipart" "net/http" "net/url" + "reflect" "sort" "strings" @@ -18,6 +19,7 @@ import ( "{{getGolangPackage}}/internal/config" "{{getGolangPackage}}/internal/flagutil" + "{{getGolangPackage}}/internal/sdk/redact" ) // maxBodyPreview is the maximum number of bytes to show in body previews. @@ -256,7 +258,7 @@ func isAllHex(value string) bool { return true } -func decodeJSON(body []byte) (interface{}, bool) { +func parseJSON(body []byte) (interface{}, bool) { dec := json.NewDecoder(bytes.NewReader(body)) dec.UseNumber() var parsed interface{} @@ -266,6 +268,14 @@ func decodeJSON(body []byte) (interface{}, bool) { if err := dec.Decode(&struct{}{}); err != io.EOF { return nil, false } + return parsed, true +} + +func decodeJSON(body []byte) (interface{}, bool) { + parsed, ok := parseJSON(body) + if !ok { + return nil, false + } return redactJSON(parsed, 0), true } @@ -307,6 +317,27 @@ func previewBody(body []byte, contentType string) interface{} { return redactBase64String(string(body)) } +// previewRequestBody masks the values the operation's schema marks sensitive +// before applying the name-based preview rules. +func previewRequestBody(req *http.Request, body []byte) interface{} { + contentType := req.Header.Get("Content-Type") + return previewRedactedBody(redact.RequestBody(req.Context(), contentType, body), contentType) +} + +func previewResponseBody(req *http.Request, resp *http.Response, body []byte) interface{} { + contentType := resp.Header.Get("Content-Type") + return previewRedactedBody(redact.ResponseBody(req.Context(), contentType, body), contentType) +} + +// previewRedactedBody shows a body masked as a whole as the mask itself, +// since it no longer parses as its content type. +func previewRedactedBody(body []byte, contentType string) interface{} { + if string(body) == redact.Mask { + return redact.Mask + } + return previewBody(body, contentType) +} + func previewForm(body []byte) string { raw := string(body) if _, err := url.ParseQuery(raw); err != nil { @@ -325,6 +356,14 @@ func previewForm(body []byte) string { parts[i] = key + "=[REDACTED]" continue } + if parsed, ok := parseJSON([]byte(decodedValue)); ok { + if redacted := redactJSON(parsed, 0); !reflect.DeepEqual(parsed, redacted) { + if encoded, err := encodeJSON(redacted, ""); err == nil { + parts[i] = key + "=" + url.QueryEscape(encoded) + continue + } + } + } redacted := redactBase64String(decodedValue) if redacted != decodedValue { parts[i] = key + "=" + redacted @@ -472,7 +511,7 @@ func (c *DebugClient) Do(req *http.Request) (*http.Response, error) { if readErr != nil { fmt.Fprintf(c.Stderr, "[DEBUG] Request Body Read Error: %v\n", readErr) } else { - body := previewBody(bodyData, req.Header.Get("Content-Type")) + body := previewRequestBody(req, bodyData) fmt.Fprintf(c.Stderr, "[DEBUG] Request Body:\n %s\n", formatBodyPreview(body, maxBodyPreview)) } } @@ -497,7 +536,7 @@ func (c *DebugClient) Do(req *http.Request) (*http.Response, error) { if readErr != nil { fmt.Fprintf(c.Stderr, "[DEBUG] Response Body Read Error: %v\n", readErr) } else if len(bodyData) > 0 { - body := previewBody(bodyData, resp.Header.Get("Content-Type")) + body := previewResponseBody(req, resp, bodyData) fmt.Fprintf(c.Stderr, "[DEBUG] Response Body:\n %s\n", formatBodyPreview(body, maxBodyPreview)) } } @@ -562,7 +601,7 @@ func (c *DryRunClient) Do(req *http.Request) (*http.Response, error) { if err != nil { return nil, fmt.Errorf("read request body for dry-run preview: %w", err) } - body = previewBody(bodyData, req.Header.Get("Content-Type")) + body = previewRequestBody(req, bodyData) } requestURL := redactURL(req.URL.String()) headers := redactHeaders(req.Header) diff --git a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl index 886a5d923..8f2a1960c 100644 --- a/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl +++ b/templates/templates/cli/tests/primary/diagnostics_test.go.stmpl @@ -2,6 +2,7 @@ package tests import ( "encoding/json" + "io" "net/http" "net/http/httptest" "strings" @@ -9,6 +10,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "{{getGolangPackage}}/internal/sdk/redact" ) // newDiagnosticsServer creates a test server that echoes requests and includes @@ -339,6 +342,243 @@ func TestVersionAndWhoamiMachineOutput(t *testing.T) { assert.False(t, json.Valid([]byte(strings.TrimSpace(h.GetStdout()))), "pretty version output turned into JSON") } + +// diagnosticsOutputModes runs a command under each diagnostics mode that +// previews the request body and returns the stream holding that preview. +var diagnosticsOutputModes = []struct { + name string + run func(t *testing.T, args []string) string +}{ + {"dry-run", func(t *testing.T, args []string) string { + h := NewCLITestHarness(t) + require.NoError(t, h.RunWithServer("http://127.0.0.1:1", append([]string{"--dry-run", "--output-format", "pretty"}, args...))) + return h.GetStderr() + }}, + {"dry-run json", func(t *testing.T, args []string) string { + h := NewCLITestHarness(t) + require.NoError(t, h.RunWithServer("http://127.0.0.1:1", append([]string{"--dry-run"}, args...))) + return h.GetStdout() + }}, + {"debug", func(t *testing.T, args []string) string { + s := newDiagnosticsServer() + defer s.Close() + h := NewCLITestHarness(t) + require.NoError(t, h.RunWithServer(s.URL, append([]string{"--debug"}, args...))) + return h.GetStderr() + }}, +} + +func TestDiagnosticsRedactSensitiveBodyFields(t *testing.T) { + cases := []struct { + name string + args []string + secrets []string + visible []string + }{ + { + name: "split json body", + args: []string{"request-bodies", "cli-sensitive-split-body-post-json", "--json-value", "canary-json", "--name", "visible-name", "--form-value", "visible-form-value"}, + secrets: []string{"canary-json"}, + visible: []string{"visible-name", "visible-form-value"}, + }, + { + name: "split multipart body", + args: []string{"request-bodies", "cli-sensitive-split-body-post-multipart", "--multipart-value", "canary-multipart", "--name", "visible-name"}, + secrets: []string{"canary-multipart"}, + visible: []string{"visible-name"}, + }, + { + name: "split form body", + args: []string{"request-bodies", "cli-sensitive-split-body-post-form", "--form-value", "canary-form", "--name", "visible-name"}, + secrets: []string{"canary-form"}, + visible: []string{"visible-name"}, + }, + { + name: "whole json body", + args: []string{"request-bodies", "cli-sensitive-whole-json-post", "--value", "canary-whole-json"}, + secrets: []string{"canary-whole-json"}, + visible: []string{redact.Mask}, + }, + { + name: "whole form body", + args: []string{"request-bodies", "cli-sensitive-whole-form-post", "--value", "canary-whole-form"}, + secrets: []string{"canary-whole-form"}, + visible: []string{redact.Mask}, + }, + { + name: "whole multipart body", + args: []string{"request-bodies", "cli-sensitive-whole-multipart-post", "--value", "canary-whole-multipart"}, + secrets: []string{"canary-whole-multipart"}, + visible: []string{redact.Mask}, + }, + { + name: "whole text body", + args: []string{"request-bodies", "cli-sensitive-whole-text-post", "--request", `"canary-whole-text"`}, + secrets: []string{"canary-whole-text"}, + visible: []string{redact.Mask}, + }, + { + name: "recursive union", + args: []string{"request-bodies", "cli-sensitive-recursive-union-post", "--cli-sensitive-recursive-union", + `{"kind":"branch","value":"canary-root","plain":{"name":"visible-name","value":"visible-root"},` + + `"child":{"kind":"branch","value":"canary-child","plain":{"name":"visible-name","value":"visible-child"},` + + `"child":{"kind":"branch","value":"canary-grandchild"}}}`}, + secrets: []string{"canary-root", "canary-child", "canary-grandchild"}, + visible: []string{"visible-root", "visible-child"}, + }, + { + name: "recursive union-reversed", + args: []string{"request-bodies", "cli-sensitive-recursive-union-reversed-post", "--cli-sensitive-recursive-union-reversed", + `{"kind":"branch","value":"canary-root","plain":{"name":"visible-name","value":"visible-root"},` + + `"child":{"kind":"branch","value":"canary-child","plain":{"name":"visible-name","value":"visible-child"},` + + `"child":{"kind":"branch","value":"canary-grandchild"}}}`}, + secrets: []string{"canary-root", "canary-child", "canary-grandchild"}, + visible: []string{"visible-root", "visible-child"}, + }, + { + name: "recursive schema edges", + args: []string{"request-bodies", "cli-sensitive-body-fields-post", "--body", + `{"name":"visible-name","value":"canary-top","tree":{"value":"canary-root",` + + `"plain":{"name":"visible-name","value":"visible-root"},` + + `"children":[{"value":"canary-array","plain":{"name":"visible-name","value":"visible-array"}}],` + + `"byName":{"first":{"value":"canary-map","child":{"value":"canary-map-child"},` + + `"plain":{"name":"visible-name","value":"visible-map"}}},` + + `"link":{"value":"visible-link","next":{"value":"canary-mutual",` + + `"link":{"value":"visible-inner-link","next":{"value":"canary-inner-mutual"}}}}}}`}, + secrets: []string{"canary-top", "canary-root", "canary-array", "canary-map", "canary-map-child", "canary-mutual", "canary-inner-mutual"}, + visible: []string{"visible-root", "visible-array", "visible-map", "visible-link", "visible-inner-link"}, + }, + { + name: "marked fields at every depth", + args: []string{"request-bodies", "cli-sensitive-body-fields-post", "--body", + `{"name":"visible-name","value":"canary-top","passphrase":"canary-password",` + + `"nested":{"label":"visible-nested","value":"canary-nested"},` + + `"entries":[{"label":"visible-entry","value":"canary-entry"}],` + + `"byName":{"first":{"label":"visible-map","value":"canary-map"}},` + + `"codes":["canary-item"],"codesByName":{"visible-code":"canary-map-value"},` + + `"tree":{"label":"visible-root","value":"canary-root","child":{"label":"visible-child","value":"canary-child",` + + `"child":{"value":"canary-grandchild"},"branch":{"label":"visible-branch","value":"canary-branch",` + + `"next":{"value":"canary-next"}}}}}`}, + secrets: []string{"canary-top", "canary-password", "canary-nested", "canary-entry", "canary-map", + "canary-item", "canary-map-value", "canary-root", "canary-child", "canary-grandchild", + "canary-branch", "canary-next"}, + visible: []string{"visible-name", "visible-nested", "visible-entry", "visible-map", "visible-code", + "visible-root", "visible-child", "visible-branch"}, + }, + { + name: "additional properties beside declared fields", + args: []string{"request-bodies", "cli-sensitive-open-object-post", "--body", + `{"name":"visible-name","value":"visible-value","extra":"canary-extra","entries":{"label":"visible-label"}}`}, + secrets: []string{"canary-extra"}, + visible: []string{"visible-name", "visible-value", "visible-label", "visible-const"}, + }, + { + name: "array variant beside object variant", + args: []string{"request-bodies", "cli-sensitive-open-object-post", "--body", + `{"name":"visible-name","entries":["canary-list"]}`}, + secrets: []string{"canary-list"}, + visible: []string{"visible-name", "visible-const"}, + }, + { + name: "union variant field", + args: []string{"request-bodies", "cli-sensitive-union-body-post", "--scope", "visible-scope", + "--cli-sensitive-secret", `{"kind":"static","id":"visible-id","value":"canary-static"}`}, + secrets: []string{"canary-static"}, + visible: []string{"visible-id"}, + }, + { + name: "expanded union variant flags", + args: []string{"request-bodies", "cli-sensitive-union-body-post", "--scope", "visible-scope", + "--cli-sensitive-secret.basic.id", "visible-id", "--cli-sensitive-secret.basic.username", "visible-user", + "--cli-sensitive-secret.basic.passphrase", "canary-basic"}, + secrets: []string{"canary-basic"}, + visible: []string{"visible-id", "visible-user"}, + }, + { + name: "multipart parts", + args: []string{"request-bodies", "cli-sensitive-multipart-post", "--name", "visible-name", "--value", "canary-top", "--bracketed-payload", "canary-brackets", + "--config-param", `{"label":"visible-label","value":"canary-config","password":"canary-config-password"}`, + "--codes", "canary-code-1", "--codes", "canary-code-2", + "--entries", `[{"label":"visible-entry","value":"canary-entry"}]`, + "--tree", `{"value":"canary-tree","child":{"value":"canary-tree-child","plain":{"name":"visible-name","value":"visible-tree-plain"}}}`}, + secrets: []string{"canary-top", "canary-brackets", "canary-config", "canary-config-password", "canary-code-1", "canary-code-2", "canary-entry", "canary-tree", "canary-tree-child"}, + visible: []string{"visible-name", "visible-label", "visible-entry", "visible-tree-plain"}, + }, + { + name: "form fields", + args: []string{"request-bodies", "cli-sensitive-form-put", "--name", "visible-name", "--value", "canary-top", "--bracketed-payload", "canary-brackets", + "--config-param", `{"label":"visible-label","value":"canary-config","password":"canary-config-password"}`, + "--codes", "canary-code-1", "--codes", "canary-code-2", + "--entries", `[{"label":"visible-entry","value":"canary-entry"}]`, + "--tree", `{"value":"canary-tree","child":{"value":"canary-tree-child","plain":{"name":"visible-name","value":"visible-tree-plain"}}}`}, + secrets: []string{"canary-top", "canary-brackets", "canary-config", "canary-config-password", "canary-code-1", "canary-code-2", "canary-entry", "canary-tree", "canary-tree-child"}, + visible: []string{"visible-name", "visible-label", "visible-entry", "visible-tree-plain"}, + }, + { + name: "form json field without redactions kept as sent", + args: []string{"request-bodies", "cli-sensitive-form-put", "--name", "visible-name", "--value", "canary-top", + "--tree", `{"label":"visible 0 { + {{- if hasAnyRedactGraphs}} + redacted := e.Body + if e.RawResponse != nil && e.RawResponse.Request != nil { + redacted = string(redact.ResponseBody(e.RawResponse.Request.Context(), e.RawResponse.Header.Get("Content-Type"), []byte(e.Body))) + } + body = fmt.Sprintf("\n%s", redacted) + {{- else}} body = fmt.Sprintf("\n%s", e.Body) + {{- end}} } return fmt.Sprintf("%s: Status %d%s", e.Message, e.StatusCode, body) diff --git a/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl new file mode 100644 index 000000000..faca7241b --- /dev/null +++ b/templates/templates/go/auxiliary/{{getRedactLocation}}/redact.go.stmpl @@ -0,0 +1,516 @@ +// Package redact masks values marked sensitive in the API description +// (x-speakeasy-param-sensitive or format: password) in HTTP bodies, so that +// HTTP clients and transports can log requests and responses safely. +// +// The SDK attaches the operation ID to the context of each request whose +// operation has sensitive request or response values. A transport reads it +// back from the request, or from a response's Request, to select the schema +// graph used for masking: +// +// body = redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), body) +// body = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), body) +// +// JSON, application/x-www-form-urlencoded and multipart/form-data bodies are +// masked value by value. Other bodies of an operation with sensitive values are +// masked as a whole. +package redact + +import ( + "bytes" + "context" + "encoding/json" + "io" + "mime" + "mime/multipart" + "net/textproto" + "net/url" + "strings" +) + +// Mask replaces each sensitive value. +const Mask = "(sensitive)" + +const maxDepth = 64 + +type operationKey struct{} + +// WithOperation returns ctx carrying operationID. +func WithOperation(ctx context.Context, operationID string) context.Context { + return context.WithValue(ctx, operationKey{}, operationID) +} + +// Operation returns the operation ID carried by ctx, or "". +func Operation(ctx context.Context) string { + if ctx == nil { + return "" + } + id, _ := ctx.Value(operationKey{}).(string) + return id +} + +// RequestBody masks the sensitive values of the request body of the +// operation carried by ctx, serialized as contentType. Bodies of other +// operations are returned as is. +func RequestBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].request.redact(contentType, body) +} + +// ResponseBody masks the sensitive values of a response body of the +// operation carried by ctx, serialized as contentType, matching every +// response schema of the operation. Bodies of other operations are returned +// as is. +func ResponseBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].response.redact(contentType, body) +} + +// A graph describes where sensitive values sit in a body. Node 0 is a +// sentinel for "nothing sensitive below here"; roots are the body schemas. +type graph struct { + nodes []node + roots []int +} + +type node struct { + sensitive bool + // fields maps JSON property names to their schema; a property that is not + // declared uses values (additional properties or map values). + fields map[string]int + item int + values int + variants []int + // plainObject, plainArray, plainString, plainNumber and plainBoolean + // report union members accepting JSON values of that type and holding + // nothing sensitive. + plainObject bool + plainArray bool + plainString bool + plainNumber bool + plainBoolean bool +} + +type shape int + +const ( + objectShape shape = iota + arrayShape + stringShape + numberShape + booleanShape +) + +// redact returns body unchanged when it has nothing to mask. A body that +// cannot be parsed is masked as a whole. +func (g *graph) redact(contentType string, body []byte) []byte { + if g == nil || len(body) == 0 { + return body + } + if g.sensitive(g.roots) { + return []byte(Mask) + } + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil { + mediaType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) + } + var masked []byte + ok := false + switch { + case mediaType == "application/x-www-form-urlencoded": + masked, _, ok = g.redactForm(body, g.roots) + case mediaType == "multipart/form-data": + masked, _, ok = g.redactMultipart(body, params["boundary"], g.roots) + case mediaType == "" || mediaType == "application/json" || mediaType == "text/json" || strings.HasSuffix(mediaType, "+json"): + masked, _, ok = g.redactJSON(body, g.roots) + } + if !ok { + return []byte(Mask) + } + return masked +} + +func (g *graph) redactJSON(body []byte, roots []int) ([]byte, bool, bool) { + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var value any + if err := dec.Decode(&value); err != nil { + return nil, false, false + } + if _, err := dec.Token(); err != io.EOF { + return nil, false, false + } + // Decoding keeps the last of duplicate keys, so the bytes returned as is + // could hold an earlier, unmasked one. + if hasDuplicateKeys(body) { + return nil, false, false + } + masked, changed := g.mask(value, roots, 0) + if !changed { + return body, false, true + } + var out bytes.Buffer + enc := json.NewEncoder(&out) + enc.SetEscapeHTML(false) + if err := enc.Encode(masked); err != nil { + return nil, false, false + } + return bytes.TrimSuffix(out.Bytes(), []byte("\n")), true, true +} + +// redactForm masks the fields of a form body. A field holding a sensitive +// value or array of values is masked, and a JSON-encoded field is masked +// within. +func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { + pairs := strings.Split(string(body), "&") + changed := false + for i, pair := range pairs { + rawKey, rawValue, _ := strings.Cut(pair, "=") + key, err := url.QueryUnescape(rawKey) + if err != nil || isJSONStructure([]byte(key)) { + return nil, false, false + } + value, err := url.QueryUnescape(rawValue) + if err != nil { + return nil, false, false + } + masked, fieldChanged := g.redactValue([]byte(value), g.field(roots, key)) + if !fieldChanged { + continue + } + changed = true + if string(masked) == Mask { + pairs[i] = rawKey + "=" + Mask + } else { + pairs[i] = rawKey + "=" + url.QueryEscape(string(masked)) + } + } + if !changed { + return body, false, true + } + return []byte(strings.Join(pairs, "&")), true, true +} + +// redactMultipart masks the parts of a multipart body by part name, keeping +// the boundary and the part headers. +func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]byte, bool, bool) { + if boundary == "" { + return nil, false, false + } + reader := multipart.NewReader(bytes.NewReader(body), boundary) + var out bytes.Buffer + writer := multipart.NewWriter(&out) + if err := writer.SetBoundary(boundary); err != nil { + return nil, false, false + } + changed := false + for { + part, err := reader.NextRawPart() + if err == io.EOF { + break + } + if err != nil { + return nil, false, false + } + data, err := io.ReadAll(part) + if err != nil { + return nil, false, false + } + partRoots := g.field(roots, part.FormName()) + masked, partChanged := g.redactValue(data, partRoots) + if !partChanged && len(g.expand(partRoots)) > 0 && isEncodedPart(part.Header) { + masked, partChanged = []byte(Mask), true + } + changed = changed || partChanged + header := part.Header + if partChanged { + // The mask is written as is, not in the part's transfer encoding, + // and with its own length. + header = make(textproto.MIMEHeader, len(part.Header)) + for key, values := range part.Header { + header[key] = values + } + header.Del("Content-Transfer-Encoding") + header.Del("Content-Encoding") + header.Del("Content-Length") + } + partWriter, err := writer.CreatePart(header) + if err != nil { + return nil, false, false + } + if _, err := partWriter.Write(masked); err != nil { + return nil, false, false + } + } + if !changed { + return body, false, true + } + if err := writer.Close(); err != nil { + return nil, false, false + } + return out.Bytes(), true, true +} + +// redactValue masks a form field or multipart part. A value holding a +// sensitive value, or one that cannot be parsed as JSON while holding one, is +// masked as a whole. +func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { + if len(g.expand(roots)) == 0 { + return value, false + } + items := g.items(roots) + if g.sensitive(roots) || g.sensitive(items) && !isJSONStructure(value) { + return []byte(Mask), true + } + // Unencoded text matches a plain string member of the field or of its + // items when sent as repeated fields, unless it starts like a JSON object + // or array, which may carry sensitive values. + if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !isJSONStructure(value) { + return value, false + } + // A scalar may also be one item of an array sent as repeated fields. + if !isJSONStructure(value) { + roots = append(append([]int{}, roots...), items...) + } + if masked, changed, ok := g.redactJSON(value, roots); ok { + return masked, changed + } + return []byte(Mask), true +} + +// isEncodedPart reports whether a multipart part is transfer or content +// encoded, so that its bytes cannot be read as its value. +func isEncodedPart(header textproto.MIMEHeader) bool { + switch strings.ToLower(strings.TrimSpace(header.Get("Content-Transfer-Encoding"))) { + case "", "7bit", "8bit", "binary": + return header.Get("Content-Encoding") != "" + } + return true +} + +// hasDuplicateKeys reports whether a JSON object in body repeats a key. +func hasDuplicateKeys(body []byte) bool { + type object struct { + keys map[string]bool + expectKey bool + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var stack []*object + for { + token, err := dec.Token() + if err != nil { + return false + } + switch t := token.(type) { + case json.Delim: + switch t { + case '{': + stack = append(stack, &object{keys: map[string]bool{}, expectKey: true}) + continue + case '[': + stack = append(stack, nil) + continue + } + stack = stack[:len(stack)-1] + case string: + if n := len(stack); n > 0 && stack[n-1] != nil && stack[n-1].expectKey { + if stack[n-1].keys[t] { + return true + } + stack[n-1].keys[t] = true + stack[n-1].expectKey = false + continue + } + } + if n := len(stack); n > 0 && stack[n-1] != nil { + stack[n-1].expectKey = true + } + } +} + +func isJSONStructure(value []byte) bool { + trimmed := bytes.TrimLeft(value, " \t\r\n") + return len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '[') +} + +func (g *graph) mask(value any, roots []int, depth int) (any, bool) { + roots = g.expand(roots) + if len(roots) == 0 { + return value, false + } + if g.sensitive(roots) { + return Mask, true + } + if depth >= maxDepth { + return Mask, true + } + // A value that does not have the shape of its schema cannot be followed + // to its sensitive values, so it is masked as a whole. + changed := false + switch v := value.(type) { + case nil: + case map[string]any: + if !g.hasProperties(roots) { + return g.mismatch(value, roots, objectShape) + } + for key, child := range v { + if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { + v[key] = masked + changed = true + } + } + case []any: + items := g.items(roots) + if len(items) == 0 { + return g.mismatch(value, roots, arrayShape) + } + for i, child := range v { + if masked, ok := g.mask(child, items, depth+1); ok { + v[i] = masked + changed = true + } + } + case string: + return g.mismatch(value, roots, stringShape) + case json.Number, float64: + return g.mismatch(value, roots, numberShape) + case bool: + return g.mismatch(value, roots, booleanShape) + default: + return Mask, true + } + return value, changed +} + +// mismatch masks a value that does not have the shape of roots, unless a +// union member of its shape holding nothing sensitive may match it. +func (g *graph) mismatch(value any, roots []int, s shape) (any, bool) { + if g.plain(roots, s) { + return value, false + } + return Mask, true +} + +// plain reports whether any of roots, or their union variants, is a union +// with a member of shape s holding nothing sensitive. +func (g *graph) plain(roots []int, s shape) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].plainShape(s) { + return true + } + } + return false +} + +func (n node) plainShape(s shape) bool { + switch s { + case objectShape: + return n.plainObject + case arrayShape: + return n.plainArray + case stringShape: + return n.plainString + case numberShape: + return n.plainNumber + case booleanShape: + return n.plainBoolean + } + return false +} + +// hasProperties reports whether any of roots describes object properties or +// map values. +func (g *graph) hasProperties(roots []int) bool { + for _, id := range g.expand(roots) { + if len(g.nodes[id].fields) > 0 || g.nodes[id].values != 0 { + return true + } + } + return false +} + +// sensitive reports whether any of roots, or their union variants, is +// sensitive as a whole. +func (g *graph) sensitive(roots []int) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].sensitive { + return true + } + } + return false +} + +// child returns the schemas of the property name of roots. +func (g *graph) child(roots []int, name string) []int { + var children []int + for _, id := range g.expand(roots) { + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) + } + } + return children +} + +// field returns the schemas of a form field or multipart part of roots. In +// each schema, a name ending in [] that is not declared as such names an +// array sent as repeated fields. +func (g *graph) field(roots []int, name string) []int { + trimmed, bracketed := strings.CutSuffix(name, "[]") + var children []int + for _, id := range g.expand(roots) { + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared && bracketed { + childID, declared = n.fields[trimmed] + } + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) + } + } + return children +} + +// items returns the array item schemas of roots. +func (g *graph) items(roots []int) []int { + var items []int + for _, id := range g.expand(roots) { + if item := g.nodes[id].item; item != 0 { + items = append(items, item) + } + } + return items +} + +// expand adds the union variants of roots, since the variant a value matches +// is not known. +func (g *graph) expand(roots []int) []int { + var expanded []int + seen := make(map[int]bool) + var visit func(int) + visit = func(id int) { + if id <= 0 || id >= len(g.nodes) || seen[id] { + return + } + seen[id] = true + expanded = append(expanded, id) + for _, variant := range g.nodes[id].variants { + visit(variant) + } + } + for _, id := range roots { + visit(id) + } + return expanded +} + +var operations = map[string]struct{ request, response *graph }{ + {{- range templateRedactGraphs}} + {{.OperationID}}: { {{- if .Request}}request: {{.Request}},{{end}}{{if .Response}} response: {{.Response}},{{end -}} }, + {{- end}} +} diff --git a/templates/templates/go/auxiliary/{{getUtilsLocation}}/utils.go.stmpl b/templates/templates/go/auxiliary/{{getUtilsLocation}}/utils.go.stmpl index 8187439f8..a304fedf0 100644 --- a/templates/templates/go/auxiliary/{{getUtilsLocation}}/utils.go.stmpl +++ b/templates/templates/go/auxiliary/{{getUtilsLocation}}/utils.go.stmpl @@ -20,6 +20,9 @@ import ( "github.com/ericlagergren/decimal" {{end}} "{{getRootModulePath}}/{{getNullableLocation}}" + {{- if hasAnyRedactGraphs}} + "{{getRootModulePath}}/{{getRedactLocation}}" + {{- end}} ) const ( @@ -28,6 +31,14 @@ const ( pathParamTagKey = "pathParam" ) +{{- if hasAnyRedactGraphs}} +// WithOperation marks requests built from ctx as belonging to operationID, so +// that transports can mask the operation's sensitive values when logging. +func WithOperation(ctx context.Context, operationID string) context.Context { + return redact.WithOperation(ctx, operationID) +} +{{end}} + var ( paramRegex = regexp.MustCompile(`({.*?})`) SerializationMethodToContentType = map[string]string{ diff --git a/templates/templates/go/includes/imports.ts b/templates/templates/go/includes/imports.ts index 60a11ddc4..484862d88 100644 --- a/templates/templates/go/includes/imports.ts +++ b/templates/templates/go/includes/imports.ts @@ -56,6 +56,10 @@ function addImport( imp = getRetryLocation(); local = true; break; + case "redact": + imp = getRedactLocation(); + local = true; + break; case "config": imp = `${getInternalPackageName()}/config`; local = true; @@ -208,6 +212,7 @@ function resolveNamespaceAlias(outputLocation: string): string { getUtilsLocation(), getPollingLocation(), getRetryLocation(), + getRedactLocation(), getNullableLocation(), ] .filter(Boolean) diff --git a/templates/templates/go/includes/includes.ts b/templates/templates/go/includes/includes.ts index 26a0c7e09..3e0635a3e 100644 --- a/templates/templates/go/includes/includes.ts +++ b/templates/templates/go/includes/includes.ts @@ -35,3 +35,4 @@ require("includes/polling.ts"); require("includes/options.ts"); require("includes/open-unions.ts"); require("includes/unions.ts"); +require("includes/redact.ts"); diff --git a/templates/templates/go/includes/redact.ts b/templates/templates/go/includes/redact.ts new file mode 100644 index 000000000..9fcad2124 --- /dev/null +++ b/templates/templates/go/includes/redact.ts @@ -0,0 +1,57 @@ +function getRedactLocation() { + return getTypesLocation().replace(/types$/, "redact"); +} +registerTemplateFunc("getRedactLocation", getRedactLocation); + +function renderRedactGraph(graph: SensitiveBodyGraph | null): string { + if (!graph) return ""; + const nodes = graph.Nodes.map((node) => { + const parts: string[] = []; + if (node.Sensitive) parts.push("sensitive: true"); + if (node.Fields?.length) + parts.push( + `fields: map[string]int{ ${node.Fields.map( + (field) => `${JSON.stringify(field.Name)}: ${field.Node}`, + ).join(", ")} }`, + ); + if (node.Item) parts.push(`item: ${node.Item}`); + if (node.Values) parts.push(`values: ${node.Values}`); + if (node.Variants?.length) + parts.push(`variants: []int{ ${node.Variants.join(", ")} }`); + if (node.PlainObject) parts.push("plainObject: true"); + if (node.PlainArray) parts.push("plainArray: true"); + if (node.PlainString) parts.push("plainString: true"); + if (node.PlainNumber) parts.push("plainNumber: true"); + if (node.PlainBoolean) parts.push("plainBoolean: true"); + return `{ ${parts.join(", ")} }`; + }); + return `&graph{ nodes: []node{ ${nodes.join( + ", ", + )} }, roots: []int{ ${graph.Roots.join(", ")} } }`; +} + +function templateRedactGraphs(): { + OperationID: string; + Request: string; + Response: string; +}[] { + return allOperations() + .filter((op) => op.HasSensitiveBodies()) + .map((op) => ({ + OperationID: JSON.stringify(op.ID), + Request: renderRedactGraph(op.SensitiveRequestBodyGraph()), + Response: renderRedactGraph(op.SensitiveResponseBodyGraph()), + })) + .sort((a, b) => a.OperationID.localeCompare(b.OperationID)); +} +registerTemplateFunc("templateRedactGraphs", templateRedactGraphs); + +function hasRedactGraphs(op: Operation): boolean { + return op.HasSensitiveBodies(); +} +registerTemplateFunc("hasRedactGraphs", hasRedactGraphs); + +function hasAnyRedactGraphs(): boolean { + return allOperations().some((op) => op.HasSensitiveBodies()); +} +registerTemplateFunc("hasAnyRedactGraphs", hasAnyRedactGraphs); diff --git a/templates/templates/go/method.go.stmpl b/templates/templates/go/method.go.stmpl index 67b2eb318..40ce30c7b 100644 --- a/templates/templates/go/method.go.stmpl +++ b/templates/templates/go/method.go.stmpl @@ -171,7 +171,8 @@ func (s *{{sanitizeClassName $op.OwningSDK.Type.Name}}) {{$methodName}}(ctx cont {{- end}} } - req, err := http.NewRequestWithContext(ctx, "{{upper $op.Method}}", opURL, {{if and $op.Request $op.Request.RequestBody}}bodyReader{{else}}nil{{end}}) + {{if hasRedactGraphs $op}}{{addImport "utils"}}ctx = utils.WithOperation(ctx, "{{$op.ID}}") + {{end}}req, err := http.NewRequestWithContext(ctx, "{{upper $op.Method}}", opURL, {{if and $op.Request $op.Request.RequestBody}}bodyReader{{else}}nil{{end}}) if err != nil { {{templateErrorReturn $op "fmt.Errorf(\"error creating request: %w\", err)"}} } @@ -780,7 +781,8 @@ func (s *{{sanitizeClassName $op.OwningSDK.Type.Name}}) {{$methodName}}(ctx cont {{- end}} } - req, err := http.NewRequestWithContext(ctx, "{{upper $op.Method}}", opURL, {{if and $op.Request $op.Request.RequestBody}}bodyReader{{else}}nil{{end}}) + {{if hasRedactGraphs $op}}{{addImport "utils"}}ctx = utils.WithOperation(ctx, "{{$op.ID}}") + {{end}}req, err := http.NewRequestWithContext(ctx, "{{upper $op.Method}}", opURL, {{if and $op.Request $op.Request.RequestBody}}bodyReader{{else}}nil{{end}}) if err != nil { {{templateErrorReturn $op "fmt.Errorf(\"error creating request: %w\", err)"}} } diff --git a/templates/templates/go/tests/primary/redact_additional_test.go.stmpl b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl new file mode 100644 index 000000000..4782cf4be --- /dev/null +++ b/templates/templates/go/tests/primary/redact_additional_test.go.stmpl @@ -0,0 +1,325 @@ +package tests + +import ( + "bytes" + "context" + "errors" + "io" + "mime" + "mime/multipart" + "net/http" + "net/url" + "strings" + "testing" + + sdk "{{getRootModulePath}}" + "{{getRootModulePath}}/{{getRedactLocation}}" + "{{getRootModulePath}}/pkg/models/apierrors" + "{{getRootModulePath}}/pkg/models/operations" + "{{getRootModulePath}}/pkg/models/shared" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const sensitiveBodyPayload = `{ + "name": "visible-name", + "password": "secret-password", + "token": "secret-token", + "nested": {"label": "visible-nested", "secret": "secret-nested"}, + "items": [{"label": "visible-item", "secret": "secret-item"}], + "tokens": ["secret-tokens-0", "secret-tokens-1"], + "labels": {"visible": "visible-label", "extra": "secret-label"}, + "credential": {"kind": "password", "password": "secret-credential"}, + "tree": {"label": "visible-root", "secret": "secret-root", "children": [{"label": "visible-child", "secret": "secret-child", "children": []}]}, + "whole": {"value": "secret-whole"}, + "unrelated": {"secret": "visible-unrelated"} +}` + +type redactCaptureClient struct { + status int + operation string + contentType string + body []byte + masked []byte +} + +// Do records the request body as sent and as masked for the operation +// carried by the request context. +func (c *redactCaptureClient) Do(req *http.Request) (*http.Response, error) { + c.operation = redact.Operation(req.Context()) + c.contentType = req.Header.Get("Content-Type") + c.body = nil + if req.Body != nil { + body, err := io.ReadAll(req.Body) + if err != nil { + return nil, err + } + c.body = body + } + c.masked = redact.RequestBody(req.Context(), c.contentType, c.body) + return &http.Response{ + StatusCode: c.status, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(sensitiveBodyPayload)), + Request: req, + }, nil +} + +func assertMasked(t *testing.T, output string, secrets, visible []string) { + t.Helper() + for _, secret := range secrets { + assert.NotContains(t, output, secret) + } + for _, value := range visible { + assert.Contains(t, output, value) + } +} + +func assertSensitiveBodyMasked(t *testing.T, output string) { + t.Helper() + for _, secret := range []string{"secret-password", "secret-token", "secret-nested", "secret-item", "secret-tokens-0", "secret-tokens-1", "secret-label", "secret-credential", "secret-root", "secret-child", "secret-whole"} { + assert.NotContains(t, output, secret) + } + for _, visible := range []string{"visible-name", "visible-nested", "visible-item", "visible-label", "visible-root", "visible-child", "visible-unrelated"} { + assert.Contains(t, output, visible) + } +} + +func TestRedactSensitiveBodies(t *testing.T) { + ctx := redact.WithOperation(context.Background(), "sensitiveBodyRedaction") + + assertSensitiveBodyMasked(t, string(redact.RequestBody(ctx, "application/json", []byte(sensitiveBodyPayload)))) + assertSensitiveBodyMasked(t, string(redact.ResponseBody(ctx, "application/json; charset=utf-8", []byte(sensitiveBodyPayload)))) + + plain := []byte(`{"name": "visible-name"}`) + assert.Equal(t, plain, redact.ResponseBody(ctx, "application/json", plain), "bodies without sensitive values are returned unchanged") + assert.Equal(t, redact.Mask, string(redact.ResponseBody(ctx, "application/json", []byte("secret-text"))), "unparseable bodies are masked as a whole") + assert.Equal(t, redact.Mask, string(redact.ResponseBody(ctx, "text/plain", []byte(`{"name": "visible-name"}`))), "bodies of other media types are masked as a whole") + assert.Equal(t, []byte(sensitiveBodyPayload), redact.ResponseBody(context.Background(), "application/json", []byte(sensitiveBodyPayload)), "requests without an operation are not masked") + + assert.Equal(t, `"`+redact.Mask+`"`, string(redact.ResponseBody(ctx, "application/json", []byte(`"secret-scalar"`))), "values not shaped like their schema are masked") + assert.Equal(t, `{"name":"visible-name","nested":"`+redact.Mask+`"}`, string(redact.ResponseBody(ctx, "application/json", []byte(`{"name": "visible-name", "nested": ["secret-shape"]}`))), "values not shaped like their schema are masked") + assert.Equal(t, `{"name":null}`, string(redact.ResponseBody(ctx, "application/json", []byte(`{"name":null}`))), "null values are kept") + + errorBody := string(redact.ResponseBody(ctx, "application/json", []byte(`{"message": "visible-message", "secret": "secret-error"}`))) + assert.Contains(t, errorBody, "visible-message") + assert.NotContains(t, errorBody, "secret-error", "error response schemas are masked") + + for _, body := range []string{`{"nested": {"secret": "secret-duplicate"}, "nested": null}`, `{"tokens": ["secret-duplicate"], "tokens": null}`} { + assert.Equal(t, redact.Mask, string(redact.ResponseBody(ctx, "application/json", []byte(body))), "bodies repeating a key are masked as a whole") + } + assert.JSONEq(t, `{"name": "visible-name", "nested": {"label": "a", "secret": "`+redact.Mask+`"}, "items": [{"label": "a"}, {"label": "a"}]}`, + string(redact.ResponseBody(ctx, "application/json", []byte(`{"name": "visible-name", "nested": {"label": "a", "secret": "secret-nested"}, "items": [{"label": "a"}, {"label": "a"}]}`))), "keys repeated across objects are not duplicates") + + streamCtx := redact.WithOperation(context.Background(), "sensitiveStreamRedaction") + assert.Equal(t, redact.Mask, string(redact.ResponseBody(streamCtx, "application/jsonl", []byte(`{"label": "visible", "secret": "secret-stream"}`))), "streams holding sensitive values are masked as a whole") +} + +func TestRedactUnionVariantsWithoutSensitiveValues(t *testing.T) { + ctx := redact.WithOperation(context.Background(), "sensitiveBodyRedaction") + redactJSON := func(body string) string { + return string(redact.RequestBody(ctx, "application/json", []byte(body))) + } + + assertMasked(t, redactJSON(`{"name": "visible-name", "tokensOrLabel": ["secret-token-item"]}`), []string{"secret-token-item"}, []string{"visible-name"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "tokensOrLabel": {"label": "visible-label"}}`), nil, []string{"visible-label"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrLabels": {"label": "visible-nested", "secret": "secret-nested"}}`), []string{"secret-nested"}, []string{"visible-nested"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrLabels": ["visible-item"]}`), nil, []string{"visible-item"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrText": {"label": "visible-nested", "secret": "secret-nested"}}`), []string{"secret-nested"}, []string{"visible-nested"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrText": "visible-text"}`), nil, []string{"visible-text"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrText": ["secret-shape"]}`), []string{"secret-shape"}, []string{"visible-name"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrLabels": "secret-shape"}`), []string{"secret-shape"}, []string{"visible-name"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrCount": 4242, "secretOrFlag": true}`), nil, []string{"4242", "true"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrCount": "secret-count", "secretOrFlag": "secret-flag"}`), []string{"secret-count", "secret-flag"}, []string{"visible-name"}) + assertMasked(t, redactJSON(`{"name": "visible-name", "secretOrText": 4242}`), []string{"4242"}, []string{"visible-name"}) + + redactForm := func(field, value string) string { + masked, err := url.ParseQuery(string(redact.RequestBody(ctx, "application/x-www-form-urlencoded", []byte("name=visible-name&"+field+"="+url.QueryEscape(value))))) + require.NoError(t, err) + return masked.Get(field) + } + redactPart := func(field, value string) string { + body := "--b\r\nContent-Disposition: form-data; name=\"" + field + "\"\r\n\r\n" + value + "\r\n--b--\r\n" + part, err := multipart.NewReader(bytes.NewReader(redact.RequestBody(ctx, "multipart/form-data; boundary=b", []byte(body))), "b").NextPart() + require.NoError(t, err) + data, err := io.ReadAll(part) + require.NoError(t, err) + return string(data) + } + for name, redactField := range map[string]func(field, value string) string{"form": redactForm, "multipart": redactPart} { + assert.JSONEq(t, `{"label": "visible-label"}`, redactField("tokensOrLabel", `{"label": "visible-label"}`), "%s: a plain object member stays readable", name) + assert.JSONEq(t, `["`+redact.Mask+`"]`, redactField("tokensOrLabel", `["secret-token-item"]`), "%s: sensitive array items are masked", name) + assert.Equal(t, redact.Mask, redactField("tokensOrLabel", "secret-token-item"), "%s: a repeated sensitive array item is masked", name) + assert.Equal(t, "visible-text", redactField("secretOrText", "visible-text"), "%s: a plain string member stays readable", name) + assert.Equal(t, redact.Mask, redactField("secretOrLabels", `{"secret": "secret-malformed",`), "%s: malformed JSON without a plain string member is masked", name) + assert.Equal(t, "4242", redactField("secretOrText", "4242"), "%s: numeric text of a plain string member stays readable", name) + assert.Equal(t, redact.Mask, redactField("secretOrText", `{"secret": "secret-malformed",`), "%s: malformed JSON is masked beside a plain string member", name) + for _, field := range []string{"secretOrCount", "secretOrFlag"} { + assert.Equal(t, redact.Mask, redactField(field, `{"secret": "secret-malformed",`), "%s: malformed JSON in %s is masked", name, field) + assert.Equal(t, redact.Mask, redactField(field, "secret-text"), "%s: text in %s is masked", name, field) + } + assert.Equal(t, "4242", redactField("secretOrCount", "4242"), "%s: a plain integer member stays readable", name) + assert.Equal(t, "true", redactField("secretOrFlag", "true"), "%s: a plain boolean member stays readable", name) + assert.Equal(t, "visible-name", redactField("name", "visible-name"), "%s: fields without sensitive values stay readable", name) + assert.Equal(t, "visible-item-name", redactField("secretsOrNames", "visible-item-name"), "%s: a repeated item matching a plain string member stays readable", name) + assert.Equal(t, "4242", redactField("secretsOrCounts", "4242"), "%s: a repeated item matching a plain integer member stays readable", name) + assert.Equal(t, "true", redactField("secretsOrFlags", "true"), "%s: a repeated item matching a plain boolean member stays readable", name) + assert.Equal(t, redact.Mask, redactField("secretsOrCounts", "secret-text"), "%s: a repeated text item without a plain string member is masked", name) + assert.NotContains(t, redactField("secretsOrNames", `{"label": "visible-label", "secret": "secret-item"}`), "secret-item", "%s: a repeated object item is masked", name) + } +} + +func TestRedactOperationOnRequestContext(t *testing.T) { + client := &redactCaptureClient{status: http.StatusOK} + s := sdk.New(sdk.WithServerURL("http://localhost"), sdk.WithClient(client)) + + res, err := s.SensitiveBodies.Redaction(context.Background(), shared.SensitiveBodyPayload{Name: "visible-name"}) + require.NoError(t, err) + assert.Equal(t, "sensitiveBodyRedaction", client.operation) + require.NotNil(t, res.SensitiveBodyPayload) + assert.Equal(t, "secret-password", *res.SensitiveBodyPayload.Password, "decoded values are not masked") + + client.status = http.StatusInternalServerError + _, err = s.SensitiveBodies.Redaction(context.Background(), shared.SensitiveBodyPayload{Name: "visible-name"}) + var apiErr *apierrors.APIError + require.True(t, errors.As(err, &apiErr)) + assert.Contains(t, apiErr.Body, "secret-password", "the error body is not masked") + assertSensitiveBodyMasked(t, apiErr.Error()) +} + +func TestRedactFormRequestBody(t *testing.T) { + client := &redactCaptureClient{status: http.StatusOK} + s := sdk.New(sdk.WithServerURL("http://localhost"), sdk.WithClient(client)) + + _, err := s.SensitiveBodies.FormRedaction(context.Background(), shared.SensitiveBodyForm{ + Name: "visible-name", + Token: sdk.String("secret-token"), + Tokens: []string{"secret-tokens-0", "secret-tokens-1"}, + Nested: &shared.SensitiveBodyFormNested{VisibleProperty: sdk.String("visible-nested"), SecretProperty: sdk.String("secret-nested")}, + Metadata: &shared.SensitiveBodyNested{Label: sdk.String("visible-metadata"), Secret: sdk.String("secret-metadata")}, + }) + require.NoError(t, err) + assert.Equal(t, "sensitiveFormRedaction", client.operation) + assert.Contains(t, string(client.body), "secret-token", "the body sent is not masked") + + masked := string(client.masked) + assertMasked(t, masked, + []string{"secret-token", "secret-tokens-0", "secret-tokens-1", "secret-nested", "secret-metadata"}, + []string{"name=visible-name", "visible-nested", "visible-metadata"}) + values, err := url.ParseQuery(masked) + require.NoError(t, err, "the masked body is a form") + assert.Equal(t, []string{redact.Mask, redact.Mask}, values["tokens"]) + assert.JSONEq(t, `{"label": "visible-metadata", "secret": "`+redact.Mask+`"}`, values.Get("metadata")) + + plainCtx := redact.WithOperation(context.Background(), "sensitiveFormRedaction") + plain := []byte("name=visible-name&metadata=%7B%22label%22%3A%22visible%22%7D") + assert.Equal(t, plain, redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", plain), "forms without sensitive values are returned unchanged") + assert.Equal(t, "name=visible-name&metadata="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("name=visible-name&metadata=secret-text"))), "unparseable JSON-encoded fields are masked") + assert.Equal(t, "name=visible-name&tokens[]="+redact.Mask+"&tokens[]="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("name=visible-name&tokens[]=secret-tokens-0&tokens[]=secret-tokens-1"))), "arrays sent with a [] suffix are matched by their property name") + assert.Equal(t, "tokens="+redact.Mask+"&tokens="+redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte("tokens=null&tokens=secret-tokens-0"))), "every repeated sensitive item is masked") + for _, body := range []string{`{"token": "secret-json", "name": "visible-name"}`, " \n\t{\"token\": \"secret-json\"}", `[{"token": "secret-json"}]`} { + assert.Equal(t, redact.Mask, string(redact.RequestBody(plainCtx, "application/x-www-form-urlencoded", []byte(body))), "a JSON body labelled as a form is masked as a whole: %q", body) + } +} + +func TestRedactBracketedFieldNames(t *testing.T) { + client := &redactCaptureClient{status: http.StatusOK} + s := sdk.New(sdk.WithServerURL("http://localhost"), sdk.WithClient(client)) + + _, _ = s.SensitiveBodies.BracketedFormRedaction(context.Background(), shared.SensitiveBodyBracketedForm{ + Name: "visible-name", + BracketedPayload: sdk.String("secret-brackets"), + Token: sdk.String("secret-token"), + BracketedToken: sdk.String("visible-bracketed"), + }) + assert.Equal(t, "sensitiveBracketedFormRedaction", client.operation) + values, err := url.ParseQuery(string(client.masked)) + require.NoError(t, err, "the masked body is a form") + assert.Equal(t, []string{redact.Mask}, values["payload[]"], "a property whose name ends in brackets is matched by its exact name") + assert.Equal(t, []string{redact.Mask}, values["token"]) + assert.Equal(t, []string{"visible-bracketed"}, values["token[]"], "a plain bracketed property is not mistaken for a sensitive one") + assert.Equal(t, []string{"visible-name"}, values["name"]) + + ctx := redact.WithOperation(context.Background(), "sensitiveBracketedFormRedaction") + assert.Equal(t, "name=visible-name&tokens[]="+redact.Mask+"&tokens[]="+redact.Mask, string(redact.RequestBody(ctx, "application/x-www-form-urlencoded", []byte("name=visible-name&tokens[]=secret-tokens-0&tokens[]=secret-tokens-1"))), "arrays sent with a [] suffix are matched by their property name") + assert.Equal(t, "tokens[]="+redact.Mask, string(redact.ResponseBody(ctx, "application/x-www-form-urlencoded", []byte("tokens[]=secret-tokens-0"))), "a bracketed name declared by one response schema still matches a sensitive array of another") + + _, _ = s.SensitiveBodies.BracketedMultipartRedaction(context.Background(), shared.SensitiveBodyBracketedMultipart{ + Name: "visible-name", + BracketedPayload: sdk.String("secret-brackets"), + }) + assert.Equal(t, "sensitiveBracketedMultipartRedaction", client.operation) + assertMasked(t, string(client.masked), []string{"secret-brackets"}, []string{"visible-name", `name="payload[]"`}) +} + +func TestRedactMultipartRequestBody(t *testing.T) { + client := &redactCaptureClient{status: http.StatusOK} + s := sdk.New(sdk.WithServerURL("http://localhost"), sdk.WithClient(client)) + + _, err := s.SensitiveBodies.MultipartRedaction(context.Background(), shared.SensitiveBodyMultipart{ + Name: "visible-name", + Token: sdk.String("secret-token"), + Tokens: []string{"secret-tokens-0", "secret-tokens-1"}, + Metadata: &shared.SensitiveBodyNested{Label: sdk.String("visible-metadata"), Secret: sdk.String("secret-metadata")}, + File: &shared.File{FileName: "visible.txt", Content: []byte("visible-file-content")}, + }) + require.NoError(t, err) + assert.Equal(t, "sensitiveMultipartRedaction", client.operation) + assert.Contains(t, string(client.body), "secret-token", "the body sent is not masked") + + assertMasked(t, string(client.masked), + []string{"secret-token", "secret-tokens-0", "secret-tokens-1", "secret-metadata"}, + []string{"visible-name", "visible-metadata", "visible.txt", "visible-file-content"}) + + _, params, err := mime.ParseMediaType(client.contentType) + require.NoError(t, err) + reader := multipart.NewReader(bytes.NewReader(client.masked), params["boundary"]) + parts := map[string][]string{} + for { + part, err := reader.NextPart() + if err == io.EOF { + break + } + require.NoError(t, err, "the masked body is multipart with the same boundary") + data, err := io.ReadAll(part) + require.NoError(t, err) + parts[part.FormName()] = append(parts[part.FormName()], string(data)) + } + assert.Equal(t, []string{"visible-name"}, parts["name"]) + assert.Equal(t, []string{redact.Mask}, parts["token"]) + assert.Equal(t, []string{redact.Mask, redact.Mask}, parts["tokens"]) + assert.Equal(t, []string{"visible-file-content"}, parts["file"]) + + ctx := redact.WithOperation(context.Background(), "sensitiveMultipartRedaction") + assert.Equal(t, redact.Mask, string(redact.RequestBody(ctx, "multipart/form-data", client.body)), "multipart bodies without a boundary are masked as a whole") + assert.Equal(t, redact.Mask, string(redact.RequestBody(ctx, "multipart/mixed; boundary="+params["boundary"], client.body)), "other multipart bodies are masked as a whole") + + encoded := "--b\r\nContent-Disposition: form-data; name=\"token\"\r\nContent-Transfer-Encoding: base64\r\nContent-Length: 20\r\n\r\nc2VjcmV0LWVuY29kZWQ=\r\n--b--\r\n" + masked := string(redact.RequestBody(ctx, "multipart/form-data; boundary=b", []byte(encoded))) + assert.NotContains(t, masked, "c2VjcmV0LWVuY29kZWQ=") + assert.NotContains(t, masked, "Content-Transfer-Encoding", "masked parts are not transfer encoded") + assert.NotContains(t, masked, "Content-Length", "masked parts do not keep the original length") + assert.Contains(t, masked, redact.Mask) + + payloadCtx := redact.WithOperation(context.Background(), "sensitiveBodyRedaction") + encodedText := "--b\r\nContent-Disposition: form-data; name=\"secretOrText\"\r\nContent-Transfer-Encoding: base64\r\n\r\neyJzZWNyZXQiOiJzZWNyZXQtZW5jb2RlZCJ9\r\n--b--\r\n" + masked = string(redact.RequestBody(payloadCtx, "multipart/form-data; boundary=b", []byte(encodedText))) + assert.NotContains(t, masked, "eyJzZWNyZXQiOiJzZWNyZXQtZW5jb2RlZCJ9", "encoded parts of a field with sensitive values are masked") + plainEncoded := "--b\r\nContent-Disposition: form-data; name=\"name\"\r\nContent-Transfer-Encoding: base64\r\n\r\ndmlzaWJsZS1uYW1l\r\n--b--\r\n" + assert.Equal(t, plainEncoded, string(redact.RequestBody(payloadCtx, "multipart/form-data; boundary=b", []byte(plainEncoded))), "encoded parts without sensitive values are kept") +} + +func TestRedactSplitRequestBodies(t *testing.T) { + client := &redactCaptureClient{status: http.StatusOK} + s := sdk.New(sdk.WithServerURL("http://localhost"), sdk.WithClient(client)) + + _, err := s.SensitiveBodies.SplitRedactionJSON(context.Background(), operations.SensitiveSplitRedactionJSONRequestBody{Name: "visible-name", JSONSecret: "secret-json"}) + require.NoError(t, err) + assert.Equal(t, "sensitiveSplitRedaction_json", client.operation) + assertMasked(t, string(client.masked), []string{"secret-json"}, []string{"visible-name"}) + + _, err = s.SensitiveBodies.SplitRedactionForm(context.Background(), operations.SensitiveSplitRedactionFormRequestBody{Name: "visible-name", FormSecret: "secret-form"}) + require.NoError(t, err) + assert.Equal(t, "sensitiveSplitRedaction_form", client.operation) + assertMasked(t, string(client.masked), []string{"secret-form"}, []string{"name=visible-name"}) +} diff --git a/templates/templates/terraform/auxiliary/internal/provider/utils.go.stmpl b/templates/templates/terraform/auxiliary/internal/provider/utils.go.stmpl index 5de82f435..09bd99bec 100644 --- a/templates/templates/terraform/auxiliary/internal/provider/utils.go.stmpl +++ b/templates/templates/terraform/auxiliary/internal/provider/utils.go.stmpl @@ -20,6 +20,7 @@ import ( "github.com/hashicorp/terraform-plugin-log/tflog" tfReflect "{{getRootPackage}}/internal/provider/reflect" + "{{getSDKPackage}}/redact" ) func debugResponse(response *http.Response) string { @@ -31,14 +32,32 @@ func debugResponse(response *http.Response) string { return err.Error() } } - dumpRes, err := httputil.DumpResponse(response, true) + // The body is read directly rather than from a full dump, which would + // re-encode a chunked body and so prevent it from being parsed. + dumpRes, err := httputil.DumpResponse(response, false) if err != nil { - dumpRes, err = httputil.DumpResponse(response, false) - if err != nil { - return err.Error() + return err.Error() + } + ctx := response.Request.Context() + if response.Body != nil { + if body, err := io.ReadAll(response.Body); err == nil { + response.Body = io.NopCloser(bytes.NewReader(body)) + dumpRes = append(dumpRes, redact.ResponseBody(ctx, response.Header.Get("Content-Type"), body)...) } } - return redactSensitiveValues(response.Request.Context(), fmt.Sprintf("**Request**:\n%s\n**Response**:\n%s", string(dumpReq), string(dumpRes))) + dumpReq = redactDumpBody(dumpReq, func(body []byte) []byte { + return redact.RequestBody(ctx, response.Request.Header.Get("Content-Type"), body) + }) + return redactSensitiveValues(ctx, fmt.Sprintf("**Request**:\n%s\n**Response**:\n%s", string(dumpReq), string(dumpRes))) +} + +// redactDumpBody applies redactBody to the body of an HTTP message dump. +func redactDumpBody(dump []byte, redactBody func([]byte) []byte) []byte { + head, body, found := bytes.Cut(dump, []byte("\r\n\r\n")) + if !found { + return dump + } + return bytes.Join([][]byte{head, redactBody(body)}, []byte("\r\n\r\n")) } func merge(ctx context.Context, req resource.UpdateRequest, resp *resource.UpdateResponse, target interface{}) { @@ -207,7 +226,7 @@ func decomposeRequestForLogging(req *http.Request) (map[string]interface{}, erro } // Read the rest of the body content - fields[FieldHttpRequestBody] = bodyFromRestOfRequestReader(reqReader) + fields[FieldHttpRequestBody] = string(redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), []byte(bodyFromRestOfRequestReader(reqReader)))) redactSensitiveFields(req, fields) return fields, nil } @@ -282,6 +301,9 @@ func decomposeResponseForLogging(res *http.Response) (map[string]interface{}, er // http.Client res.Body = io.NopCloser(bytes.NewBuffer(resBody)) + if res.Request != nil { + resBody = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), resBody) + } fields[FieldHttpResponseBody] = string(resBody) redactSensitiveFields(res.Request, fields) diff --git a/templates/templates/terraform/tests/review/internal/provider/sensitive_body_test.go.stmpl b/templates/templates/terraform/tests/review/internal/provider/sensitive_body_test.go.stmpl new file mode 100644 index 000000000..3c5ce1710 --- /dev/null +++ b/templates/templates/terraform/tests/review/internal/provider/sensitive_body_test.go.stmpl @@ -0,0 +1,71 @@ +package provider + +import ( + "bytes" + stderrors "errors" + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/hashicorp/terraform-plugin-log/tflogtest" + + "{{getSDKPackage}}" + "{{getSDKPackage}}/models/errors" + "{{getSDKPackage}}/models/operations" +) + +// A computed sensitive value is unknown before the response arrives, so only +// the response schema can identify it in the transport log and diagnostics. +func TestSensitiveResponseValuesRedactedFromLogsAndErrors(t *testing.T) { + status := http.StatusOK + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"id":"visible-id","string_request":"visible-request","string_response":"computed-secret","string_request_and_response":"both-secret"}`)) + })) + defer server.Close() + + var logs bytes.Buffer + ctx := tflogtest.RootLogger(context.Background(), &logs) + client := sdk.New( + sdk.WithServerURL(server.URL), + sdk.WithClient(&http.Client{Transport: NewProviderHTTPTransport(ProviderHTTPTransportOpts{Transport: http.DefaultTransport})}), + ) + + res, err := client.GetXParamSensitive(ctx, operations.GetXParamSensitiveRequest{ID: "visible-id"}) + if err != nil { + t.Fatal(err) + } + if got := res.XParamSensitiveResponse.GetStringResponse(); got == nil || *got != "computed-secret" { + t.Errorf("SDK decoded string_response as %v, want the original value", got) + } + + status = http.StatusBadRequest + _, err = client.GetXParamSensitive(ctx, operations.GetXParamSensitiveRequest{ID: "visible-id"}) + var apiErr *errors.SDKError + if !stderrors.As(err, &apiErr) { + t.Fatalf("expected an SDK API error, got %v", err) + } + if !strings.Contains(apiErr.Body, "computed-secret") { + t.Errorf("SDKError.Body should keep the original body: %s", apiErr.Body) + } + + for name, output := range map[string]string{ + "transport log": logs.String(), + "error message": err.Error(), + "debug response": debugResponse(apiErr.RawResponse), + } { + for _, secret := range []string{"computed-secret", "both-secret"} { + if strings.Contains(output, secret) { + t.Errorf("%s leaked %q: %s", name, secret, output) + } + } + for _, visible := range []string{"visible-id", "visible-request"} { + if !strings.Contains(output, visible) { + t.Errorf("%s dropped non-sensitive %q: %s", name, visible, output) + } + } + } +} diff --git a/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml new file mode 100644 index 000000000..645b47ba7 --- /dev/null +++ b/tests/specs/fragments/primary/cli-sensitive-body-fields.yaml @@ -0,0 +1,575 @@ +openapi: 3.1.0 +security: + - apiKeyAuth: [] + - oauth2: [] + - clientCredentials: [read, write] + - customSchemeAppId: [] + - basicHttp: [] + - accessToken: [] + - {} +paths: + /anything/cli/sensitiveBodyFields: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveBodyFieldsPost + description: >- + Request body whose secrets have generic names, marked sensitive with + x-speakeasy-param-sensitive or format: password at the top level, in + a nested object, in array items and map values (as object fields or as + the items and values themselves), and at every depth of nested + recursive schemas. Diagnostics output must redact each of them while + keeping their unmarked siblings readable. + tags: + - requestBodies + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CliSensitiveBodyFields" + responses: + "200": + description: OK + /anything/cli/sensitiveUnionBody/{scope}: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveUnionBodyPost + description: >- + Discriminated-union request body next to a path parameter, whose + variants mark a generically named field sensitive. Diagnostics output + must redact it whichever variant is sent. + tags: + - requestBodies + parameters: + - name: scope + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CliSensitiveSecret" + responses: + "200": + description: OK + /anything/cli/sensitiveEncodedFields: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveMultipartPost + description: >- + Multipart request body with a sensitive top-level part, a sensitive + array sent as repeated parts, a JSON part that nests a sensitive + field, and an array of objects with a sensitive field. Diagnostics + output must redact all of them. + tags: + - requestBodies + requestBody: + required: true + content: + multipart/form-data: + schema: + $ref: "#/components/schemas/CliSensitiveEncodedFields" + responses: + "200": + description: OK + put: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveFormPut + description: >- + Form-encoded request body with a sensitive top-level field, a sensitive + array sent as repeated fields, a JSON-encoded field that nests a + sensitive field, and an array of objects with a sensitive field. + Diagnostics output must redact all of them. + tags: + - requestBodies + requestBody: + required: true + content: + application/x-www-form-urlencoded: + schema: + $ref: "#/components/schemas/CliSensitiveEncodedFields" + encoding: + config: + contentType: application/json + responses: + "200": + description: OK + /anything/cli/plainBodyFields: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliPlainBodyFieldsPost + description: >- + Request body with a field that shares the name of a sensitive field in + another schema but is not marked sensitive. Diagnostics output must + keep it readable. + tags: + - requestBodies + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CliPlainBodyFields" + responses: + "200": + description: OK + /anything/cli/sensitiveSplitBody: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveSplitBodyPost + description: Distinct request content types each contain a sensitive value. + tags: [requestBodies] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [jsonValue] + properties: + jsonValue: + type: string + x-speakeasy-param-sensitive: true + formValue: + description: Not marked sensitive in this content type, unlike its form counterpart. + type: string + name: + type: string + multipart/form-data: + schema: + type: object + required: [multipartValue] + properties: + multipartValue: + type: string + x-speakeasy-param-sensitive: true + name: + type: string + application/x-www-form-urlencoded: + schema: + type: object + required: [formValue] + properties: + formValue: + type: string + x-speakeasy-param-sensitive: true + name: + type: string + responses: + "200": + description: OK + /anything/cli/sensitiveWholeJson: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveWholeJsonPost + description: The entire request body is marked sensitive on its schema. + tags: [requestBodies] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CliSensitiveWholeBody" + responses: + "200": + description: OK + /anything/cli/sensitiveWholeForm: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveWholeFormPost + description: The entire request body is marked sensitive on its schema. + tags: [requestBodies] + requestBody: + required: true + content: + application/x-www-form-urlencoded: + schema: + $ref: "#/components/schemas/CliSensitiveWholeBody" + responses: + "200": + description: OK + /anything/cli/sensitiveWholeMultipart: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveWholeMultipartPost + description: The entire request body is marked sensitive on its schema. + tags: [requestBodies] + requestBody: + required: true + content: + multipart/form-data: + schema: + $ref: "#/components/schemas/CliSensitiveWholeBody" + responses: + "200": + description: OK + /anything/cli/sensitiveWholeText: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveWholeTextPost + description: The entire request body is marked sensitive on its schema. + tags: [requestBodies] + requestBody: + required: true + content: + text/plain: + schema: + $ref: "#/components/schemas/CliSensitiveWholeValue" + responses: + "200": + description: OK + /anything/cli/sensitiveRecursiveUnion: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveRecursiveUnionPost + description: Union variants share a sensitive field and one variant is recursive. + tags: [requestBodies] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CliSensitiveRecursiveUnion" + responses: + "200": + description: OK + /anything/cli/sensitiveRecursiveUnionReversed: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveRecursiveUnionReversedPost + description: Union variants share a sensitive field and one variant is recursive. + tags: [requestBodies] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CliSensitiveRecursiveUnionReversed" + responses: + "200": + description: OK + /anything/cli/sensitiveOpenObject: + post: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveOpenObjectPost + description: >- + Request body whose additional properties are marked sensitive next to + declared fields, one of them a union of a sensitive array and a plain + object. Diagnostics output must redact the additional properties and + the array items while keeping the declared fields, the constant and + the plain object readable. + tags: [requestBodies] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CliSensitiveOpenObject" + responses: + "200": + description: OK + /anything/cli/sensitiveResponse: + get: + x-speakeasy-exclude-test-sdk: [csharp, go, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + operationId: cliSensitiveResponseGet + description: >- + Response body with a field marked sensitive beside an unmarked one. + Debug output must redact the marked value in the logged response while + keeping its sibling readable. + tags: + - responseBodies + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/CliSensitiveResponse" +components: + schemas: + CliSensitiveBodyFields: + type: object + required: + - name + - value + properties: + name: + type: string + value: + type: string + x-speakeasy-param-sensitive: true + passphrase: + type: string + format: password + nested: + type: object + properties: + label: + type: string + value: + type: string + x-speakeasy-param-sensitive: true + entries: + type: array + items: + type: object + properties: + label: + type: string + value: + type: string + format: password + byName: + type: object + additionalProperties: + type: object + properties: + label: + type: string + value: + type: string + x-speakeasy-param-sensitive: true + codes: + type: array + items: + type: string + format: password + codesByName: + type: object + additionalProperties: + type: string + x-speakeasy-param-sensitive: true + tree: + $ref: "#/components/schemas/CliSensitiveNode" + CliSensitiveNode: + type: object + properties: + label: + type: string + value: + type: string + x-speakeasy-param-sensitive: true + child: + $ref: "#/components/schemas/CliSensitiveNode" + branch: + $ref: "#/components/schemas/CliSensitiveBranch" + plain: + $ref: "#/components/schemas/CliPlainBodyFields" + children: + type: array + items: + $ref: "#/components/schemas/CliSensitiveNode" + byName: + type: object + additionalProperties: + $ref: "#/components/schemas/CliSensitiveNode" + link: + $ref: "#/components/schemas/CliSensitiveLink" + CliSensitiveBranch: + type: object + properties: + label: + type: string + value: + type: string + format: password + next: + $ref: "#/components/schemas/CliSensitiveBranch" + CliSensitiveEncodedFields: + type: object + required: + - name + - value + properties: + payload[]: + description: Property whose wire name ends in brackets; it is not an array. + type: string + x-speakeasy-name-override: bracketedPayload + x-speakeasy-param-sensitive: true + name: + type: string + value: + type: string + x-speakeasy-param-sensitive: true + config: + type: object + properties: + label: + type: string + value: + type: string + format: password + password: + description: Not marked sensitive; its name alone requires redaction. + type: string + codes: + type: array + items: + type: string + format: password + entries: + type: array + items: + type: object + properties: + label: + type: string + value: + type: string + x-speakeasy-param-sensitive: true + tree: + $ref: "#/components/schemas/CliSensitiveNode" + CliSensitiveSecret: + oneOf: + - $ref: "#/components/schemas/CliSensitiveStaticSecret" + - $ref: "#/components/schemas/CliSensitiveBasicSecret" + discriminator: + propertyName: kind + mapping: + static: "#/components/schemas/CliSensitiveStaticSecret" + basic: "#/components/schemas/CliSensitiveBasicSecret" + CliSensitiveStaticSecret: + type: object + required: + - kind + - id + - value + properties: + kind: + type: string + const: static + id: + type: string + value: + type: string + writeOnly: true + x-speakeasy-param-sensitive: true + CliSensitiveBasicSecret: + type: object + required: + - kind + - id + - username + - passphrase + properties: + kind: + type: string + const: basic + id: + type: string + username: + type: string + passphrase: + type: string + writeOnly: true + format: password + CliPlainBodyFields: + type: object + required: + - name + - value + properties: + name: + type: string + value: + type: string + CliSensitiveWholeBody: + type: object + x-speakeasy-param-sensitive: true + required: [value] + properties: + value: + type: string + CliSensitiveWholeValue: + type: string + format: password + CliSensitiveLink: + type: object + properties: + value: + type: string + next: + $ref: "#/components/schemas/CliSensitiveNode" + CliSensitiveRecursiveLeaf: + type: object + required: [kind, value] + properties: + kind: + type: string + const: leaf + value: + type: string + x-speakeasy-param-sensitive: true + CliSensitiveRecursiveBranch: + type: object + required: [kind, value] + properties: + kind: + type: string + const: branch + value: + type: string + x-speakeasy-param-sensitive: true + child: + $ref: "#/components/schemas/CliSensitiveRecursiveBranch" + plain: + $ref: "#/components/schemas/CliPlainBodyFields" + CliSensitiveRecursiveUnion: + oneOf: + - $ref: "#/components/schemas/CliSensitiveRecursiveLeaf" + - $ref: "#/components/schemas/CliSensitiveRecursiveBranch" + discriminator: + propertyName: kind + mapping: + leaf: "#/components/schemas/CliSensitiveRecursiveLeaf" + branch: "#/components/schemas/CliSensitiveRecursiveBranch" + CliSensitiveRecursiveUnionReversed: + oneOf: + - $ref: "#/components/schemas/CliSensitiveRecursiveBranch" + - $ref: "#/components/schemas/CliSensitiveRecursiveLeaf" + discriminator: + propertyName: kind + mapping: + leaf: "#/components/schemas/CliSensitiveRecursiveLeaf" + branch: "#/components/schemas/CliSensitiveRecursiveBranch" + CliSensitiveOpenObject: + type: object + required: [kind, name] + properties: + kind: + type: string + const: visible-const + name: + type: string + value: + type: string + entries: + $ref: "#/components/schemas/CliSensitiveListOrLabel" + additionalProperties: + type: string + x-speakeasy-param-sensitive: true + CliSensitiveListOrLabel: + oneOf: + - type: array + items: + type: string + format: password + - type: object + required: [label] + properties: + label: + type: string + CliSensitiveResponse: + type: object + required: + - name + - value + properties: + name: + type: string + value: + type: string + x-speakeasy-param-sensitive: true diff --git a/tests/specs/fragments/primary/sensitive-body-redaction.yaml b/tests/specs/fragments/primary/sensitive-body-redaction.yaml new file mode 100644 index 000000000..a6b48dfce --- /dev/null +++ b/tests/specs/fragments/primary/sensitive-body-redaction.yaml @@ -0,0 +1,450 @@ +openapi: 3.1.0 +security: + - apiKeyAuth: [] + - oauth2: [] + - clientCredentials: [read, write] + - customSchemeAppId: [] + - basicHttp: [] + - accessToken: [] + - {} +paths: + /anything/sensitiveBodyRedaction: + post: + operationId: sensitiveBodyRedaction + x-speakeasy-group: sensitiveBodies + x-speakeasy-name-override: redaction + description: >- + Exercises locating values marked sensitive (x-speakeasy-param-sensitive + or format password) in request and response bodies by schema path, so + that they can be masked when bodies are logged. Sensitive values appear + at the top level, in nested objects, in array items, as array items, as + additionalProperties values beside a declared plain property, in union + variants, through a recursive schema, and as a whole object. An + unmarked property sharing a name with a sensitive one stays readable. + The error response schema also holds a sensitive value. + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/SensitiveBodyPayload" + responses: + "200": + description: The payload, including newly returned sensitive values. + content: + application/json: + schema: + $ref: "#/components/schemas/SensitiveBodyPayload" + "400": + description: An error whose body holds a sensitive value. + content: + application/json: + schema: + $ref: "#/components/schemas/SensitiveBodyError" + /anything/sensitiveStreamRedaction: + get: + operationId: sensitiveStreamRedaction + x-speakeasy-group: sensitiveBodies + x-speakeasy-name-override: streamRedaction + description: >- + Exercises a streamed JSON Lines response whose items hold a value marked + sensitive. A stream body is not parsed item by item, so it is masked as + a whole when logged. + responses: + "200": + description: A stream of items holding sensitive values. + content: + application/jsonl: + schema: + $ref: "#/components/schemas/SensitiveBodyNested" + /anything/sensitiveFormRedaction: + post: + operationId: sensitiveFormRedaction + x-speakeasy-group: sensitiveBodies + x-speakeasy-name-override: formRedaction + description: >- + Exercises locating values marked sensitive in a form-urlencoded request + body by field name: a sensitive field, an array of sensitive values + sent as repeated fields, an object-valued field holding a sensitive + property, and a JSON-encoded field holding a sensitive property. Plain + fields stay readable. + requestBody: + required: true + content: + application/x-www-form-urlencoded: + schema: + $ref: "#/components/schemas/SensitiveBodyForm" + encoding: + metadata: + contentType: application/json + responses: + "200": + description: OK + /anything/sensitiveMultipartRedaction: + post: + operationId: sensitiveMultipartRedaction + x-speakeasy-group: sensitiveBodies + x-speakeasy-name-override: multipartRedaction + description: >- + Exercises locating values marked sensitive in a multipart request body + by part name: a sensitive part, an array of sensitive values sent as + repeated parts, and a JSON-encoded part holding a sensitive property. + Plain parts and file parts stay readable. + requestBody: + required: true + content: + multipart/form-data: + schema: + $ref: "#/components/schemas/SensitiveBodyMultipart" + encoding: + metadata: + contentType: application/json + responses: + "200": + description: OK + /anything/sensitiveSplitRedaction: + post: + operationId: sensitiveSplitRedaction + x-speakeasy-group: sensitiveBodies + x-speakeasy-name-override: splitRedaction + description: >- + Exercises a request body with distinct schemas per content type, each + holding a different sensitive property, so that the body sent in each + content type is masked with its own schema. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [name, jsonSecret] + properties: + name: + type: string + jsonSecret: + type: string + x-speakeasy-param-sensitive: true + application/x-www-form-urlencoded: + schema: + type: object + required: [name, formSecret] + properties: + name: + type: string + formSecret: + type: string + x-speakeasy-param-sensitive: true + responses: + "200": + description: OK + /anything/sensitiveBracketedFormRedaction: + post: + operationId: sensitiveBracketedFormRedaction + x-speakeasy-group: sensitiveBodies + x-speakeasy-name-override: bracketedFormRedaction + x-speakeasy-exclude-test-sdk: [cli, csharp, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + description: >- + Exercises form field names ending in brackets: a sensitive property + whose wire name ends in brackets, a plain bracketed property beside a + sensitive one of the same base name, and an array of sensitive values + sent as repeated fields with a bracket suffix. A response schema + declaring a plain bracketed property must not hide the sensitive array + of another response schema. + requestBody: + required: true + content: + application/x-www-form-urlencoded: + schema: + $ref: "#/components/schemas/SensitiveBodyBracketedForm" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/SensitiveBodyBracketedTokens" + "400": + description: Bad request + content: + application/json: + schema: + $ref: "#/components/schemas/SensitiveBodyBracketedError" + /anything/sensitiveBracketedMultipartRedaction: + post: + operationId: sensitiveBracketedMultipartRedaction + x-speakeasy-group: sensitiveBodies + x-speakeasy-name-override: bracketedMultipartRedaction + x-speakeasy-exclude-test-sdk: [cli, csharp, javav2, mcp-typescript, php, postman, pythonv2, ruby, terraform, typescriptv2, unity] + description: >- + Exercises a multipart part whose name ends in brackets and holds a + sensitive value, beside a plain part. + requestBody: + required: true + content: + multipart/form-data: + schema: + $ref: "#/components/schemas/SensitiveBodyBracketedMultipart" + responses: + "200": + description: OK +components: + schemas: + SensitiveBodyPayload: + type: object + required: + - name + properties: + name: + type: string + password: + type: string + format: password + token: + type: string + x-speakeasy-param-sensitive: true + nested: + $ref: "#/components/schemas/SensitiveBodyNested" + items: + type: array + items: + $ref: "#/components/schemas/SensitiveBodyNested" + tokens: + type: array + items: + type: string + x-speakeasy-param-sensitive: true + labels: + $ref: "#/components/schemas/SensitiveBodyLabels" + credential: + oneOf: + - $ref: "#/components/schemas/SensitiveBodyAPIKeyCredential" + - $ref: "#/components/schemas/SensitiveBodyPasswordCredential" + tree: + $ref: "#/components/schemas/SensitiveBodyTree" + whole: + type: object + x-speakeasy-param-sensitive: true + properties: + value: + type: string + unrelated: + $ref: "#/components/schemas/SensitiveBodyUnrelated" + tokensOrLabel: + description: Union of an array of sensitive values and an object without sensitive values; an object value stays readable. + oneOf: + - type: array + items: + type: string + x-speakeasy-param-sensitive: true + - $ref: "#/components/schemas/SensitiveBodyPlainLabel" + secretOrLabels: + description: Union of an object with a sensitive value and an array without sensitive values; an array value stays readable. + oneOf: + - $ref: "#/components/schemas/SensitiveBodyNested" + - type: array + items: + type: string + secretOrText: + description: Union of an object with a sensitive value and a string without sensitive values; a string value stays readable. + oneOf: + - $ref: "#/components/schemas/SensitiveBodyNested" + - type: string + secretOrCount: + description: Union of an object with a sensitive value and an integer; an integer value stays readable, text does not. + oneOf: + - $ref: "#/components/schemas/SensitiveBodyNested" + - type: integer + secretOrFlag: + description: Union of an object with a sensitive value and a boolean; a boolean value stays readable, text does not. + oneOf: + - $ref: "#/components/schemas/SensitiveBodyNested" + - type: boolean + secretsOrNames: + description: Array of a union of an object with a sensitive value and a string; string items sent as repeated fields stay readable. + type: array + items: + oneOf: + - $ref: "#/components/schemas/SensitiveBodyNested" + - type: string + secretsOrCounts: + description: Array of a union of an object with a sensitive value and an integer; integer items sent as repeated fields stay readable. + type: array + items: + oneOf: + - $ref: "#/components/schemas/SensitiveBodyNested" + - type: integer + secretsOrFlags: + description: Array of a union of an object with a sensitive value and a boolean; boolean items sent as repeated fields stay readable. + type: array + items: + oneOf: + - $ref: "#/components/schemas/SensitiveBodyNested" + - type: boolean + SensitiveBodyPlainLabel: + type: object + properties: + label: + type: string + SensitiveBodyError: + type: object + properties: + message: + type: string + secret: + type: string + x-speakeasy-param-sensitive: true + SensitiveBodyNested: + type: object + properties: + label: + type: string + secret: + type: string + x-speakeasy-param-sensitive: true + SensitiveBodyLabels: + type: object + properties: + visible: + type: string + additionalProperties: + type: string + x-speakeasy-param-sensitive: true + SensitiveBodyAPIKeyCredential: + type: object + required: + - kind + - key + properties: + kind: + type: string + const: api_key + key: + type: string + x-speakeasy-param-sensitive: true + SensitiveBodyPasswordCredential: + type: object + required: + - kind + - password + properties: + kind: + type: string + const: password + password: + type: string + format: password + SensitiveBodyTree: + type: object + properties: + label: + type: string + secret: + type: string + x-speakeasy-param-sensitive: true + children: + type: array + items: + $ref: "#/components/schemas/SensitiveBodyTree" + SensitiveBodyUnrelated: + type: object + properties: + secret: + type: string + SensitiveBodyForm: + type: object + required: + - name + properties: + name: + type: string + token: + type: string + x-speakeasy-param-sensitive: true + tokens: + type: array + items: + type: string + x-speakeasy-param-sensitive: true + nested: + $ref: "#/components/schemas/SensitiveBodyFormNested" + metadata: + $ref: "#/components/schemas/SensitiveBodyNested" + SensitiveBodyFormNested: + type: object + properties: + visibleProperty: + type: string + secretProperty: + type: string + x-speakeasy-param-sensitive: true + SensitiveBodyMultipart: + type: object + required: + - name + properties: + name: + type: string + token: + type: string + x-speakeasy-param-sensitive: true + tokens: + type: array + items: + type: string + x-speakeasy-param-sensitive: true + metadata: + $ref: "#/components/schemas/SensitiveBodyNested" + file: + type: string + format: binary + SensitiveBodyBracketedForm: + type: object + required: + - name + properties: + name: + type: string + payload[]: + description: Property whose wire name ends in brackets; it is not an array. + type: string + x-speakeasy-name-override: bracketedPayload + x-speakeasy-param-sensitive: true + token: + type: string + x-speakeasy-param-sensitive: true + token[]: + description: Property whose wire name ends in brackets, holding nothing sensitive, beside a sensitive token property. + type: string + x-speakeasy-name-override: bracketedToken + tokens: + type: array + items: + type: string + x-speakeasy-param-sensitive: true + SensitiveBodyBracketedTokens: + type: object + properties: + tokens: + type: array + items: + type: string + x-speakeasy-param-sensitive: true + SensitiveBodyBracketedError: + type: object + properties: + tokens[]: + description: Property whose wire name ends in brackets, holding nothing sensitive, while another response schema has a sensitive tokens array. + type: string + x-speakeasy-name-override: bracketedTokens + SensitiveBodyBracketedMultipart: + type: object + required: + - name + properties: + name: + type: string + payload[]: + description: Property whose wire name ends in brackets; it is not an array. + type: string + x-speakeasy-name-override: bracketedPayload + x-speakeasy-param-sensitive: true diff --git a/zSDKs/sdk-cli/.speakeasy/gen.lock b/zSDKs/sdk-cli/.speakeasy/gen.lock index d35205043..28af1a5e0 100644 --- a/zSDKs/sdk-cli/.speakeasy/gen.lock +++ b/zSDKs/sdk-cli/.speakeasy/gen.lock @@ -222,7 +222,7 @@ trackedFiles: internal/client/client.go: last_write_checksum: sha1:bca25435230238b7194798699e137ef94d983844 internal/client/diagnostics.go: - last_write_checksum: sha1:36245f95381109b1b8e1b0a041acd7b89737cd2e + last_write_checksum: sha1:3f9ea9a986c0a6bf7a487bb53a0eabc36bdaefc1 internal/clierrors/clierrors.go: last_write_checksum: sha1:e17fb3e33e439c6997834182011540427d760857 internal/config/config.go: @@ -565,6 +565,8 @@ trackedFiles: last_write_checksum: sha1:b828c94c45116e632925027e4e2c211e971e04e2 internal/sdk/optionalnullable/optionalnullable_test.go: last_write_checksum: sha1:d3991667754ed67db803aeab4a6239f6f9a1be37 + internal/sdk/redact/redact.go: + last_write_checksum: sha1:c034b68429958375c94f272c0ec5523e5a5b009f internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: diff --git a/zSDKs/sdk-cli/internal/client/diagnostics.go b/zSDKs/sdk-cli/internal/client/diagnostics.go index 1c44a0921..c92049cd7 100644 --- a/zSDKs/sdk-cli/internal/client/diagnostics.go +++ b/zSDKs/sdk-cli/internal/client/diagnostics.go @@ -12,6 +12,7 @@ import ( "mime/multipart" "net/http" "net/url" + "reflect" "sort" "strings" @@ -19,6 +20,7 @@ import ( "openapi/internal/config" "openapi/internal/flagutil" + "openapi/internal/sdk/redact" ) // maxBodyPreview is the maximum number of bytes to show in body previews. @@ -255,7 +257,7 @@ func isAllHex(value string) bool { return true } -func decodeJSON(body []byte) (interface{}, bool) { +func parseJSON(body []byte) (interface{}, bool) { dec := json.NewDecoder(bytes.NewReader(body)) dec.UseNumber() var parsed interface{} @@ -265,6 +267,14 @@ func decodeJSON(body []byte) (interface{}, bool) { if err := dec.Decode(&struct{}{}); err != io.EOF { return nil, false } + return parsed, true +} + +func decodeJSON(body []byte) (interface{}, bool) { + parsed, ok := parseJSON(body) + if !ok { + return nil, false + } return redactJSON(parsed, 0), true } @@ -306,6 +316,27 @@ func previewBody(body []byte, contentType string) interface{} { return redactBase64String(string(body)) } +// previewRequestBody masks the values the operation's schema marks sensitive +// before applying the name-based preview rules. +func previewRequestBody(req *http.Request, body []byte) interface{} { + contentType := req.Header.Get("Content-Type") + return previewRedactedBody(redact.RequestBody(req.Context(), contentType, body), contentType) +} + +func previewResponseBody(req *http.Request, resp *http.Response, body []byte) interface{} { + contentType := resp.Header.Get("Content-Type") + return previewRedactedBody(redact.ResponseBody(req.Context(), contentType, body), contentType) +} + +// previewRedactedBody shows a body masked as a whole as the mask itself, +// since it no longer parses as its content type. +func previewRedactedBody(body []byte, contentType string) interface{} { + if string(body) == redact.Mask { + return redact.Mask + } + return previewBody(body, contentType) +} + func previewForm(body []byte) string { raw := string(body) if _, err := url.ParseQuery(raw); err != nil { @@ -324,6 +355,14 @@ func previewForm(body []byte) string { parts[i] = key + "=[REDACTED]" continue } + if parsed, ok := parseJSON([]byte(decodedValue)); ok { + if redacted := redactJSON(parsed, 0); !reflect.DeepEqual(parsed, redacted) { + if encoded, err := encodeJSON(redacted, ""); err == nil { + parts[i] = key + "=" + url.QueryEscape(encoded) + continue + } + } + } redacted := redactBase64String(decodedValue) if redacted != decodedValue { parts[i] = key + "=" + redacted @@ -471,7 +510,7 @@ func (c *DebugClient) Do(req *http.Request) (*http.Response, error) { if readErr != nil { fmt.Fprintf(c.Stderr, "[DEBUG] Request Body Read Error: %v\n", readErr) } else { - body := previewBody(bodyData, req.Header.Get("Content-Type")) + body := previewRequestBody(req, bodyData) fmt.Fprintf(c.Stderr, "[DEBUG] Request Body:\n %s\n", formatBodyPreview(body, maxBodyPreview)) } } @@ -496,7 +535,7 @@ func (c *DebugClient) Do(req *http.Request) (*http.Response, error) { if readErr != nil { fmt.Fprintf(c.Stderr, "[DEBUG] Response Body Read Error: %v\n", readErr) } else if len(bodyData) > 0 { - body := previewBody(bodyData, resp.Header.Get("Content-Type")) + body := previewResponseBody(req, resp, bodyData) fmt.Fprintf(c.Stderr, "[DEBUG] Response Body:\n %s\n", formatBodyPreview(body, maxBodyPreview)) } } @@ -561,7 +600,7 @@ func (c *DryRunClient) Do(req *http.Request) (*http.Response, error) { if err != nil { return nil, fmt.Errorf("read request body for dry-run preview: %w", err) } - body = previewBody(bodyData, req.Header.Get("Content-Type")) + body = previewRequestBody(req, bodyData) } requestURL := redactURL(req.URL.String()) headers := redactHeaders(req.Header) diff --git a/zSDKs/sdk-cli/internal/sdk/redact/redact.go b/zSDKs/sdk-cli/internal/sdk/redact/redact.go new file mode 100644 index 000000000..4d890914d --- /dev/null +++ b/zSDKs/sdk-cli/internal/sdk/redact/redact.go @@ -0,0 +1,514 @@ +// Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + +// Package redact masks values marked sensitive in the API description +// (x-speakeasy-param-sensitive or format: password) in HTTP bodies, so that +// HTTP clients and transports can log requests and responses safely. +// +// The SDK attaches the operation ID to the context of each request whose +// operation has sensitive request or response values. A transport reads it +// back from the request, or from a response's Request, to select the schema +// graph used for masking: +// +// body = redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), body) +// body = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), body) +// +// JSON, application/x-www-form-urlencoded and multipart/form-data bodies are +// masked value by value. Other bodies of an operation with sensitive values are +// masked as a whole. +package redact + +import ( + "bytes" + "context" + "encoding/json" + "io" + "mime" + "mime/multipart" + "net/textproto" + "net/url" + "strings" +) + +// Mask replaces each sensitive value. +const Mask = "(sensitive)" + +const maxDepth = 64 + +type operationKey struct{} + +// WithOperation returns ctx carrying operationID. +func WithOperation(ctx context.Context, operationID string) context.Context { + return context.WithValue(ctx, operationKey{}, operationID) +} + +// Operation returns the operation ID carried by ctx, or "". +func Operation(ctx context.Context) string { + if ctx == nil { + return "" + } + id, _ := ctx.Value(operationKey{}).(string) + return id +} + +// RequestBody masks the sensitive values of the request body of the +// operation carried by ctx, serialized as contentType. Bodies of other +// operations are returned as is. +func RequestBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].request.redact(contentType, body) +} + +// ResponseBody masks the sensitive values of a response body of the +// operation carried by ctx, serialized as contentType, matching every +// response schema of the operation. Bodies of other operations are returned +// as is. +func ResponseBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].response.redact(contentType, body) +} + +// A graph describes where sensitive values sit in a body. Node 0 is a +// sentinel for "nothing sensitive below here"; roots are the body schemas. +type graph struct { + nodes []node + roots []int +} + +type node struct { + sensitive bool + // fields maps JSON property names to their schema; a property that is not + // declared uses values (additional properties or map values). + fields map[string]int + item int + values int + variants []int + // plainObject, plainArray, plainString, plainNumber and plainBoolean + // report union members accepting JSON values of that type and holding + // nothing sensitive. + plainObject bool + plainArray bool + plainString bool + plainNumber bool + plainBoolean bool +} + +type shape int + +const ( + objectShape shape = iota + arrayShape + stringShape + numberShape + booleanShape +) + +// redact returns body unchanged when it has nothing to mask. A body that +// cannot be parsed is masked as a whole. +func (g *graph) redact(contentType string, body []byte) []byte { + if g == nil || len(body) == 0 { + return body + } + if g.sensitive(g.roots) { + return []byte(Mask) + } + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil { + mediaType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) + } + var masked []byte + ok := false + switch { + case mediaType == "application/x-www-form-urlencoded": + masked, _, ok = g.redactForm(body, g.roots) + case mediaType == "multipart/form-data": + masked, _, ok = g.redactMultipart(body, params["boundary"], g.roots) + case mediaType == "" || mediaType == "application/json" || mediaType == "text/json" || strings.HasSuffix(mediaType, "+json"): + masked, _, ok = g.redactJSON(body, g.roots) + } + if !ok { + return []byte(Mask) + } + return masked +} + +func (g *graph) redactJSON(body []byte, roots []int) ([]byte, bool, bool) { + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var value any + if err := dec.Decode(&value); err != nil { + return nil, false, false + } + if _, err := dec.Token(); err != io.EOF { + return nil, false, false + } + // Decoding keeps the last of duplicate keys, so the bytes returned as is + // could hold an earlier, unmasked one. + if hasDuplicateKeys(body) { + return nil, false, false + } + masked, changed := g.mask(value, roots, 0) + if !changed { + return body, false, true + } + var out bytes.Buffer + enc := json.NewEncoder(&out) + enc.SetEscapeHTML(false) + if err := enc.Encode(masked); err != nil { + return nil, false, false + } + return bytes.TrimSuffix(out.Bytes(), []byte("\n")), true, true +} + +// redactForm masks the fields of a form body. A field holding a sensitive +// value or array of values is masked, and a JSON-encoded field is masked +// within. +func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { + pairs := strings.Split(string(body), "&") + changed := false + for i, pair := range pairs { + rawKey, rawValue, _ := strings.Cut(pair, "=") + key, err := url.QueryUnescape(rawKey) + if err != nil || isJSONStructure([]byte(key)) { + return nil, false, false + } + value, err := url.QueryUnescape(rawValue) + if err != nil { + return nil, false, false + } + masked, fieldChanged := g.redactValue([]byte(value), g.field(roots, key)) + if !fieldChanged { + continue + } + changed = true + if string(masked) == Mask { + pairs[i] = rawKey + "=" + Mask + } else { + pairs[i] = rawKey + "=" + url.QueryEscape(string(masked)) + } + } + if !changed { + return body, false, true + } + return []byte(strings.Join(pairs, "&")), true, true +} + +// redactMultipart masks the parts of a multipart body by part name, keeping +// the boundary and the part headers. +func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]byte, bool, bool) { + if boundary == "" { + return nil, false, false + } + reader := multipart.NewReader(bytes.NewReader(body), boundary) + var out bytes.Buffer + writer := multipart.NewWriter(&out) + if err := writer.SetBoundary(boundary); err != nil { + return nil, false, false + } + changed := false + for { + part, err := reader.NextRawPart() + if err == io.EOF { + break + } + if err != nil { + return nil, false, false + } + data, err := io.ReadAll(part) + if err != nil { + return nil, false, false + } + partRoots := g.field(roots, part.FormName()) + masked, partChanged := g.redactValue(data, partRoots) + if !partChanged && len(g.expand(partRoots)) > 0 && isEncodedPart(part.Header) { + masked, partChanged = []byte(Mask), true + } + changed = changed || partChanged + header := part.Header + if partChanged { + // The mask is written as is, not in the part's transfer encoding, + // and with its own length. + header = make(textproto.MIMEHeader, len(part.Header)) + for key, values := range part.Header { + header[key] = values + } + header.Del("Content-Transfer-Encoding") + header.Del("Content-Encoding") + header.Del("Content-Length") + } + partWriter, err := writer.CreatePart(header) + if err != nil { + return nil, false, false + } + if _, err := partWriter.Write(masked); err != nil { + return nil, false, false + } + } + if !changed { + return body, false, true + } + if err := writer.Close(); err != nil { + return nil, false, false + } + return out.Bytes(), true, true +} + +// redactValue masks a form field or multipart part. A value holding a +// sensitive value, or one that cannot be parsed as JSON while holding one, is +// masked as a whole. +func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { + if len(g.expand(roots)) == 0 { + return value, false + } + items := g.items(roots) + if g.sensitive(roots) || g.sensitive(items) && !isJSONStructure(value) { + return []byte(Mask), true + } + // Unencoded text matches a plain string member of the field or of its + // items when sent as repeated fields, unless it starts like a JSON object + // or array, which may carry sensitive values. + if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !isJSONStructure(value) { + return value, false + } + // A scalar may also be one item of an array sent as repeated fields. + if !isJSONStructure(value) { + roots = append(append([]int{}, roots...), items...) + } + if masked, changed, ok := g.redactJSON(value, roots); ok { + return masked, changed + } + return []byte(Mask), true +} + +// isEncodedPart reports whether a multipart part is transfer or content +// encoded, so that its bytes cannot be read as its value. +func isEncodedPart(header textproto.MIMEHeader) bool { + switch strings.ToLower(strings.TrimSpace(header.Get("Content-Transfer-Encoding"))) { + case "", "7bit", "8bit", "binary": + return header.Get("Content-Encoding") != "" + } + return true +} + +// hasDuplicateKeys reports whether a JSON object in body repeats a key. +func hasDuplicateKeys(body []byte) bool { + type object struct { + keys map[string]bool + expectKey bool + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var stack []*object + for { + token, err := dec.Token() + if err != nil { + return false + } + switch t := token.(type) { + case json.Delim: + switch t { + case '{': + stack = append(stack, &object{keys: map[string]bool{}, expectKey: true}) + continue + case '[': + stack = append(stack, nil) + continue + } + stack = stack[:len(stack)-1] + case string: + if n := len(stack); n > 0 && stack[n-1] != nil && stack[n-1].expectKey { + if stack[n-1].keys[t] { + return true + } + stack[n-1].keys[t] = true + stack[n-1].expectKey = false + continue + } + } + if n := len(stack); n > 0 && stack[n-1] != nil { + stack[n-1].expectKey = true + } + } +} + +func isJSONStructure(value []byte) bool { + trimmed := bytes.TrimLeft(value, " \t\r\n") + return len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '[') +} + +func (g *graph) mask(value any, roots []int, depth int) (any, bool) { + roots = g.expand(roots) + if len(roots) == 0 { + return value, false + } + if g.sensitive(roots) { + return Mask, true + } + if depth >= maxDepth { + return Mask, true + } + // A value that does not have the shape of its schema cannot be followed + // to its sensitive values, so it is masked as a whole. + changed := false + switch v := value.(type) { + case nil: + case map[string]any: + if !g.hasProperties(roots) { + return g.mismatch(value, roots, objectShape) + } + for key, child := range v { + if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { + v[key] = masked + changed = true + } + } + case []any: + items := g.items(roots) + if len(items) == 0 { + return g.mismatch(value, roots, arrayShape) + } + for i, child := range v { + if masked, ok := g.mask(child, items, depth+1); ok { + v[i] = masked + changed = true + } + } + case string: + return g.mismatch(value, roots, stringShape) + case json.Number, float64: + return g.mismatch(value, roots, numberShape) + case bool: + return g.mismatch(value, roots, booleanShape) + default: + return Mask, true + } + return value, changed +} + +// mismatch masks a value that does not have the shape of roots, unless a +// union member of its shape holding nothing sensitive may match it. +func (g *graph) mismatch(value any, roots []int, s shape) (any, bool) { + if g.plain(roots, s) { + return value, false + } + return Mask, true +} + +// plain reports whether any of roots, or their union variants, is a union +// with a member of shape s holding nothing sensitive. +func (g *graph) plain(roots []int, s shape) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].plainShape(s) { + return true + } + } + return false +} + +func (n node) plainShape(s shape) bool { + switch s { + case objectShape: + return n.plainObject + case arrayShape: + return n.plainArray + case stringShape: + return n.plainString + case numberShape: + return n.plainNumber + case booleanShape: + return n.plainBoolean + } + return false +} + +// hasProperties reports whether any of roots describes object properties or +// map values. +func (g *graph) hasProperties(roots []int) bool { + for _, id := range g.expand(roots) { + if len(g.nodes[id].fields) > 0 || g.nodes[id].values != 0 { + return true + } + } + return false +} + +// sensitive reports whether any of roots, or their union variants, is +// sensitive as a whole. +func (g *graph) sensitive(roots []int) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].sensitive { + return true + } + } + return false +} + +// child returns the schemas of the property name of roots. +func (g *graph) child(roots []int, name string) []int { + var children []int + for _, id := range g.expand(roots) { + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) + } + } + return children +} + +// field returns the schemas of a form field or multipart part of roots. In +// each schema, a name ending in [] that is not declared as such names an +// array sent as repeated fields. +func (g *graph) field(roots []int, name string) []int { + trimmed, bracketed := strings.CutSuffix(name, "[]") + var children []int + for _, id := range g.expand(roots) { + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared && bracketed { + childID, declared = n.fields[trimmed] + } + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) + } + } + return children +} + +// items returns the array item schemas of roots. +func (g *graph) items(roots []int) []int { + var items []int + for _, id := range g.expand(roots) { + if item := g.nodes[id].item; item != 0 { + items = append(items, item) + } + } + return items +} + +// expand adds the union variants of roots, since the variant a value matches +// is not known. +func (g *graph) expand(roots []int) []int { + var expanded []int + seen := make(map[int]bool) + var visit func(int) + visit = func(id int) { + if id <= 0 || id >= len(g.nodes) || seen[id] { + return + } + seen[id] = true + expanded = append(expanded, id) + for _, variant := range g.nodes[id].variants { + visit(variant) + } + } + for _, id := range roots { + visit(id) + } + return expanded +} + +var operations = map[string]struct{ request, response *graph }{} diff --git a/zSDKs/sdk-go/.speakeasy/gen.lock b/zSDKs/sdk-go/.speakeasy/gen.lock index ad322fc15..568101f9a 100644 --- a/zSDKs/sdk-go/.speakeasy/gen.lock +++ b/zSDKs/sdk-go/.speakeasy/gen.lock @@ -782,6 +782,8 @@ trackedFiles: last_write_checksum: sha1:f33d36ba2e0ca6811572cd08ce53fe5f3c5664aa recursiveformfield.go: last_write_checksum: sha1:2e06532b8c5445500bd2ae8ae8586d24e911a3e2 + redact/redact.go: + last_write_checksum: sha1:c034b68429958375c94f272c0ec5523e5a5b009f renderassetop.go: last_write_checksum: sha1:a788e389967e88581337759b330af30c4e562bf8 requesttimeouterror.go: diff --git a/zSDKs/sdk-go/redact/redact.go b/zSDKs/sdk-go/redact/redact.go new file mode 100644 index 000000000..4d890914d --- /dev/null +++ b/zSDKs/sdk-go/redact/redact.go @@ -0,0 +1,514 @@ +// Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + +// Package redact masks values marked sensitive in the API description +// (x-speakeasy-param-sensitive or format: password) in HTTP bodies, so that +// HTTP clients and transports can log requests and responses safely. +// +// The SDK attaches the operation ID to the context of each request whose +// operation has sensitive request or response values. A transport reads it +// back from the request, or from a response's Request, to select the schema +// graph used for masking: +// +// body = redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), body) +// body = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), body) +// +// JSON, application/x-www-form-urlencoded and multipart/form-data bodies are +// masked value by value. Other bodies of an operation with sensitive values are +// masked as a whole. +package redact + +import ( + "bytes" + "context" + "encoding/json" + "io" + "mime" + "mime/multipart" + "net/textproto" + "net/url" + "strings" +) + +// Mask replaces each sensitive value. +const Mask = "(sensitive)" + +const maxDepth = 64 + +type operationKey struct{} + +// WithOperation returns ctx carrying operationID. +func WithOperation(ctx context.Context, operationID string) context.Context { + return context.WithValue(ctx, operationKey{}, operationID) +} + +// Operation returns the operation ID carried by ctx, or "". +func Operation(ctx context.Context) string { + if ctx == nil { + return "" + } + id, _ := ctx.Value(operationKey{}).(string) + return id +} + +// RequestBody masks the sensitive values of the request body of the +// operation carried by ctx, serialized as contentType. Bodies of other +// operations are returned as is. +func RequestBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].request.redact(contentType, body) +} + +// ResponseBody masks the sensitive values of a response body of the +// operation carried by ctx, serialized as contentType, matching every +// response schema of the operation. Bodies of other operations are returned +// as is. +func ResponseBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].response.redact(contentType, body) +} + +// A graph describes where sensitive values sit in a body. Node 0 is a +// sentinel for "nothing sensitive below here"; roots are the body schemas. +type graph struct { + nodes []node + roots []int +} + +type node struct { + sensitive bool + // fields maps JSON property names to their schema; a property that is not + // declared uses values (additional properties or map values). + fields map[string]int + item int + values int + variants []int + // plainObject, plainArray, plainString, plainNumber and plainBoolean + // report union members accepting JSON values of that type and holding + // nothing sensitive. + plainObject bool + plainArray bool + plainString bool + plainNumber bool + plainBoolean bool +} + +type shape int + +const ( + objectShape shape = iota + arrayShape + stringShape + numberShape + booleanShape +) + +// redact returns body unchanged when it has nothing to mask. A body that +// cannot be parsed is masked as a whole. +func (g *graph) redact(contentType string, body []byte) []byte { + if g == nil || len(body) == 0 { + return body + } + if g.sensitive(g.roots) { + return []byte(Mask) + } + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil { + mediaType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) + } + var masked []byte + ok := false + switch { + case mediaType == "application/x-www-form-urlencoded": + masked, _, ok = g.redactForm(body, g.roots) + case mediaType == "multipart/form-data": + masked, _, ok = g.redactMultipart(body, params["boundary"], g.roots) + case mediaType == "" || mediaType == "application/json" || mediaType == "text/json" || strings.HasSuffix(mediaType, "+json"): + masked, _, ok = g.redactJSON(body, g.roots) + } + if !ok { + return []byte(Mask) + } + return masked +} + +func (g *graph) redactJSON(body []byte, roots []int) ([]byte, bool, bool) { + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var value any + if err := dec.Decode(&value); err != nil { + return nil, false, false + } + if _, err := dec.Token(); err != io.EOF { + return nil, false, false + } + // Decoding keeps the last of duplicate keys, so the bytes returned as is + // could hold an earlier, unmasked one. + if hasDuplicateKeys(body) { + return nil, false, false + } + masked, changed := g.mask(value, roots, 0) + if !changed { + return body, false, true + } + var out bytes.Buffer + enc := json.NewEncoder(&out) + enc.SetEscapeHTML(false) + if err := enc.Encode(masked); err != nil { + return nil, false, false + } + return bytes.TrimSuffix(out.Bytes(), []byte("\n")), true, true +} + +// redactForm masks the fields of a form body. A field holding a sensitive +// value or array of values is masked, and a JSON-encoded field is masked +// within. +func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { + pairs := strings.Split(string(body), "&") + changed := false + for i, pair := range pairs { + rawKey, rawValue, _ := strings.Cut(pair, "=") + key, err := url.QueryUnescape(rawKey) + if err != nil || isJSONStructure([]byte(key)) { + return nil, false, false + } + value, err := url.QueryUnescape(rawValue) + if err != nil { + return nil, false, false + } + masked, fieldChanged := g.redactValue([]byte(value), g.field(roots, key)) + if !fieldChanged { + continue + } + changed = true + if string(masked) == Mask { + pairs[i] = rawKey + "=" + Mask + } else { + pairs[i] = rawKey + "=" + url.QueryEscape(string(masked)) + } + } + if !changed { + return body, false, true + } + return []byte(strings.Join(pairs, "&")), true, true +} + +// redactMultipart masks the parts of a multipart body by part name, keeping +// the boundary and the part headers. +func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]byte, bool, bool) { + if boundary == "" { + return nil, false, false + } + reader := multipart.NewReader(bytes.NewReader(body), boundary) + var out bytes.Buffer + writer := multipart.NewWriter(&out) + if err := writer.SetBoundary(boundary); err != nil { + return nil, false, false + } + changed := false + for { + part, err := reader.NextRawPart() + if err == io.EOF { + break + } + if err != nil { + return nil, false, false + } + data, err := io.ReadAll(part) + if err != nil { + return nil, false, false + } + partRoots := g.field(roots, part.FormName()) + masked, partChanged := g.redactValue(data, partRoots) + if !partChanged && len(g.expand(partRoots)) > 0 && isEncodedPart(part.Header) { + masked, partChanged = []byte(Mask), true + } + changed = changed || partChanged + header := part.Header + if partChanged { + // The mask is written as is, not in the part's transfer encoding, + // and with its own length. + header = make(textproto.MIMEHeader, len(part.Header)) + for key, values := range part.Header { + header[key] = values + } + header.Del("Content-Transfer-Encoding") + header.Del("Content-Encoding") + header.Del("Content-Length") + } + partWriter, err := writer.CreatePart(header) + if err != nil { + return nil, false, false + } + if _, err := partWriter.Write(masked); err != nil { + return nil, false, false + } + } + if !changed { + return body, false, true + } + if err := writer.Close(); err != nil { + return nil, false, false + } + return out.Bytes(), true, true +} + +// redactValue masks a form field or multipart part. A value holding a +// sensitive value, or one that cannot be parsed as JSON while holding one, is +// masked as a whole. +func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { + if len(g.expand(roots)) == 0 { + return value, false + } + items := g.items(roots) + if g.sensitive(roots) || g.sensitive(items) && !isJSONStructure(value) { + return []byte(Mask), true + } + // Unencoded text matches a plain string member of the field or of its + // items when sent as repeated fields, unless it starts like a JSON object + // or array, which may carry sensitive values. + if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !isJSONStructure(value) { + return value, false + } + // A scalar may also be one item of an array sent as repeated fields. + if !isJSONStructure(value) { + roots = append(append([]int{}, roots...), items...) + } + if masked, changed, ok := g.redactJSON(value, roots); ok { + return masked, changed + } + return []byte(Mask), true +} + +// isEncodedPart reports whether a multipart part is transfer or content +// encoded, so that its bytes cannot be read as its value. +func isEncodedPart(header textproto.MIMEHeader) bool { + switch strings.ToLower(strings.TrimSpace(header.Get("Content-Transfer-Encoding"))) { + case "", "7bit", "8bit", "binary": + return header.Get("Content-Encoding") != "" + } + return true +} + +// hasDuplicateKeys reports whether a JSON object in body repeats a key. +func hasDuplicateKeys(body []byte) bool { + type object struct { + keys map[string]bool + expectKey bool + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var stack []*object + for { + token, err := dec.Token() + if err != nil { + return false + } + switch t := token.(type) { + case json.Delim: + switch t { + case '{': + stack = append(stack, &object{keys: map[string]bool{}, expectKey: true}) + continue + case '[': + stack = append(stack, nil) + continue + } + stack = stack[:len(stack)-1] + case string: + if n := len(stack); n > 0 && stack[n-1] != nil && stack[n-1].expectKey { + if stack[n-1].keys[t] { + return true + } + stack[n-1].keys[t] = true + stack[n-1].expectKey = false + continue + } + } + if n := len(stack); n > 0 && stack[n-1] != nil { + stack[n-1].expectKey = true + } + } +} + +func isJSONStructure(value []byte) bool { + trimmed := bytes.TrimLeft(value, " \t\r\n") + return len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '[') +} + +func (g *graph) mask(value any, roots []int, depth int) (any, bool) { + roots = g.expand(roots) + if len(roots) == 0 { + return value, false + } + if g.sensitive(roots) { + return Mask, true + } + if depth >= maxDepth { + return Mask, true + } + // A value that does not have the shape of its schema cannot be followed + // to its sensitive values, so it is masked as a whole. + changed := false + switch v := value.(type) { + case nil: + case map[string]any: + if !g.hasProperties(roots) { + return g.mismatch(value, roots, objectShape) + } + for key, child := range v { + if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { + v[key] = masked + changed = true + } + } + case []any: + items := g.items(roots) + if len(items) == 0 { + return g.mismatch(value, roots, arrayShape) + } + for i, child := range v { + if masked, ok := g.mask(child, items, depth+1); ok { + v[i] = masked + changed = true + } + } + case string: + return g.mismatch(value, roots, stringShape) + case json.Number, float64: + return g.mismatch(value, roots, numberShape) + case bool: + return g.mismatch(value, roots, booleanShape) + default: + return Mask, true + } + return value, changed +} + +// mismatch masks a value that does not have the shape of roots, unless a +// union member of its shape holding nothing sensitive may match it. +func (g *graph) mismatch(value any, roots []int, s shape) (any, bool) { + if g.plain(roots, s) { + return value, false + } + return Mask, true +} + +// plain reports whether any of roots, or their union variants, is a union +// with a member of shape s holding nothing sensitive. +func (g *graph) plain(roots []int, s shape) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].plainShape(s) { + return true + } + } + return false +} + +func (n node) plainShape(s shape) bool { + switch s { + case objectShape: + return n.plainObject + case arrayShape: + return n.plainArray + case stringShape: + return n.plainString + case numberShape: + return n.plainNumber + case booleanShape: + return n.plainBoolean + } + return false +} + +// hasProperties reports whether any of roots describes object properties or +// map values. +func (g *graph) hasProperties(roots []int) bool { + for _, id := range g.expand(roots) { + if len(g.nodes[id].fields) > 0 || g.nodes[id].values != 0 { + return true + } + } + return false +} + +// sensitive reports whether any of roots, or their union variants, is +// sensitive as a whole. +func (g *graph) sensitive(roots []int) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].sensitive { + return true + } + } + return false +} + +// child returns the schemas of the property name of roots. +func (g *graph) child(roots []int, name string) []int { + var children []int + for _, id := range g.expand(roots) { + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) + } + } + return children +} + +// field returns the schemas of a form field or multipart part of roots. In +// each schema, a name ending in [] that is not declared as such names an +// array sent as repeated fields. +func (g *graph) field(roots []int, name string) []int { + trimmed, bracketed := strings.CutSuffix(name, "[]") + var children []int + for _, id := range g.expand(roots) { + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared && bracketed { + childID, declared = n.fields[trimmed] + } + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) + } + } + return children +} + +// items returns the array item schemas of roots. +func (g *graph) items(roots []int) []int { + var items []int + for _, id := range g.expand(roots) { + if item := g.nodes[id].item; item != 0 { + items = append(items, item) + } + } + return items +} + +// expand adds the union variants of roots, since the variant a value matches +// is not known. +func (g *graph) expand(roots []int) []int { + var expanded []int + seen := make(map[int]bool) + var visit func(int) + visit = func(id int) { + if id <= 0 || id >= len(g.nodes) || seen[id] { + return + } + seen[id] = true + expanded = append(expanded, id) + for _, variant := range g.nodes[id].variants { + visit(variant) + } + } + for _, id := range roots { + visit(id) + } + return expanded +} + +var operations = map[string]struct{ request, response *graph }{} diff --git a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock index 618b8b520..885da2a81 100644 --- a/zSDKs/terraform-provider-testing/.speakeasy/gen.lock +++ b/zSDKs/terraform-provider-testing/.speakeasy/gen.lock @@ -1183,6 +1183,8 @@ trackedFiles: last_write_checksum: sha1:90486b28f731a812a5a29ea27474c3d1c250b05c internal/provider/rootunionwriteonly_resource_sdk.go: last_write_checksum: sha1:3877a6339d3c954216c275907f26f5f8f606a7ac + internal/provider/sensitive_body_test.go: + last_write_checksum: sha1:77d5081775c1662a561e65eae2cf506d23a67597 internal/provider/sensitive_values.go: last_write_checksum: sha1:81d8e01d37718f9af451660d4ebc5f6126569c60 internal/provider/sensitive_values_test.go: @@ -2076,7 +2078,7 @@ trackedFiles: internal/provider/unsoundreadonlysingleop_resource_sdk.go: last_write_checksum: sha1:9ebe7ce0a238c2f58ec4fab7e587fc8d91f1b5bd internal/provider/utils.go: - last_write_checksum: sha1:b7eb39b516cfeced2367ff75019fca2b7f91d6c3 + last_write_checksum: sha1:fe2b0224198eb372507274fbe4a5f9ca93087c5e internal/provider/utils_test.go: last_write_checksum: sha1:40da24e32eac29a8aac14e772e5fce44044f2ae0 internal/provider/xadditionalpropertiesname_data_source.go: @@ -2422,11 +2424,11 @@ trackedFiles: internal/sdk/internal/utils/union_test.go: last_write_checksum: sha1:c55e5e44c26dfae67b83359851996122a3dfc9cf internal/sdk/internal/utils/utils.go: - last_write_checksum: sha1:2b7f8f2bd131693f165c80adb2e47f404819d65d + last_write_checksum: sha1:76e9fbc958c040371954894ddb2d99a63f0331d1 internal/sdk/models/errors/responsevalidationerror.go: last_write_checksum: sha1:aa7f69462e3a38d05371225adeb8c87bb433e6c4 internal/sdk/models/errors/sdkerror.go: - last_write_checksum: sha1:5d65fce5049df7c5113e11c9ccb1e4c46b91901c + last_write_checksum: sha1:4e7181799f7433d1b24dd35116a9fe39510c0c9b internal/sdk/models/operations/closexentityoperationclosemultipleop1.go: last_write_checksum: sha1:7664dbc7a028e9f840a833fd5a24ade28c6bded3 internal/sdk/models/operations/closexentityoperationclosemultipleop2.go: @@ -3571,10 +3573,12 @@ trackedFiles: last_write_checksum: sha1:9695f32fc3b427c945b3aa34c3ada4dab0ba9ac1 internal/sdk/polling/option.go: last_write_checksum: sha1:69d4557bf43a1a97b2d3735c442d2d25be53e56f + internal/sdk/redact/redact.go: + last_write_checksum: sha1:90d5f1a7d4f0176deedf3d6a905eb39968a4ec38 internal/sdk/retry/config.go: last_write_checksum: sha1:102d1953fbd7e9f312c4442c71ccca2eaaeaa27d internal/sdk/sdk.go: - last_write_checksum: sha1:224db2187527a941b29d524d57531f33cfc70f48 + last_write_checksum: sha1:a2997f8af41cead3493768284c9cc8c950f8f57c internal/sdk/types/bigint.go: last_write_checksum: sha1:49b004005d0461fb04b846eca062b070b0360b31 internal/sdk/types/date.go: diff --git a/zSDKs/terraform-provider-testing/internal/provider/sensitive_body_test.go b/zSDKs/terraform-provider-testing/internal/provider/sensitive_body_test.go new file mode 100644 index 000000000..1c9a49722 --- /dev/null +++ b/zSDKs/terraform-provider-testing/internal/provider/sensitive_body_test.go @@ -0,0 +1,73 @@ +// Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + +package provider + +import ( + "bytes" + "context" + stderrors "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/hashicorp/terraform-plugin-log/tflogtest" + + "github.com/hashicorp/terraform-provider-testing/internal/sdk" + "github.com/hashicorp/terraform-provider-testing/internal/sdk/models/errors" + "github.com/hashicorp/terraform-provider-testing/internal/sdk/models/operations" +) + +// A computed sensitive value is unknown before the response arrives, so only +// the response schema can identify it in the transport log and diagnostics. +func TestSensitiveResponseValuesRedactedFromLogsAndErrors(t *testing.T) { + status := http.StatusOK + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"id":"visible-id","string_request":"visible-request","string_response":"computed-secret","string_request_and_response":"both-secret"}`)) + })) + defer server.Close() + + var logs bytes.Buffer + ctx := tflogtest.RootLogger(context.Background(), &logs) + client := sdk.New( + sdk.WithServerURL(server.URL), + sdk.WithClient(&http.Client{Transport: NewProviderHTTPTransport(ProviderHTTPTransportOpts{Transport: http.DefaultTransport})}), + ) + + res, err := client.GetXParamSensitive(ctx, operations.GetXParamSensitiveRequest{ID: "visible-id"}) + if err != nil { + t.Fatal(err) + } + if got := res.XParamSensitiveResponse.GetStringResponse(); got == nil || *got != "computed-secret" { + t.Errorf("SDK decoded string_response as %v, want the original value", got) + } + + status = http.StatusBadRequest + _, err = client.GetXParamSensitive(ctx, operations.GetXParamSensitiveRequest{ID: "visible-id"}) + var apiErr *errors.SDKError + if !stderrors.As(err, &apiErr) { + t.Fatalf("expected an SDK API error, got %v", err) + } + if !strings.Contains(apiErr.Body, "computed-secret") { + t.Errorf("SDKError.Body should keep the original body: %s", apiErr.Body) + } + + for name, output := range map[string]string{ + "transport log": logs.String(), + "error message": err.Error(), + "debug response": debugResponse(apiErr.RawResponse), + } { + for _, secret := range []string{"computed-secret", "both-secret"} { + if strings.Contains(output, secret) { + t.Errorf("%s leaked %q: %s", name, secret, output) + } + } + for _, visible := range []string{"visible-id", "visible-request"} { + if !strings.Contains(output, visible) { + t.Errorf("%s dropped non-sensitive %q: %s", name, visible, output) + } + } + } +} diff --git a/zSDKs/terraform-provider-testing/internal/provider/utils.go b/zSDKs/terraform-provider-testing/internal/provider/utils.go index 9defadc19..b1d7c03c2 100644 --- a/zSDKs/terraform-provider-testing/internal/provider/utils.go +++ b/zSDKs/terraform-provider-testing/internal/provider/utils.go @@ -22,6 +22,7 @@ import ( "github.com/hashicorp/terraform-plugin-log/tflog" tfReflect "github.com/hashicorp/terraform-provider-testing/internal/provider/reflect" + "github.com/hashicorp/terraform-provider-testing/internal/sdk/redact" ) func debugResponse(response *http.Response) string { @@ -44,14 +45,32 @@ func debugResponse(response *http.Response) string { return err.Error() } } - dumpRes, err := httputil.DumpResponse(response, true) + // The body is read directly rather than from a full dump, which would + // re-encode a chunked body and so prevent it from being parsed. + dumpRes, err := httputil.DumpResponse(response, false) if err != nil { - dumpRes, err = httputil.DumpResponse(response, false) - if err != nil { - return err.Error() + return err.Error() + } + ctx := response.Request.Context() + if response.Body != nil { + if body, err := io.ReadAll(response.Body); err == nil { + response.Body = io.NopCloser(bytes.NewReader(body)) + dumpRes = append(dumpRes, redact.ResponseBody(ctx, response.Header.Get("Content-Type"), body)...) } } - return redactSensitiveValues(response.Request.Context(), fmt.Sprintf("**Request**:\n%s\n**Response**:\n%s", string(dumpReq), string(dumpRes))) + dumpReq = redactDumpBody(dumpReq, func(body []byte) []byte { + return redact.RequestBody(ctx, response.Request.Header.Get("Content-Type"), body) + }) + return redactSensitiveValues(ctx, fmt.Sprintf("**Request**:\n%s\n**Response**:\n%s", string(dumpReq), string(dumpRes))) +} + +// redactDumpBody applies redactBody to the body of an HTTP message dump. +func redactDumpBody(dump []byte, redactBody func([]byte) []byte) []byte { + head, body, found := bytes.Cut(dump, []byte("\r\n\r\n")) + if !found { + return dump + } + return bytes.Join([][]byte{head, redactBody(body)}, []byte("\r\n\r\n")) } func merge(ctx context.Context, req resource.UpdateRequest, resp *resource.UpdateResponse, target interface{}) { @@ -220,7 +239,7 @@ func decomposeRequestForLogging(req *http.Request) (map[string]interface{}, erro } // Read the rest of the body content - fields[FieldHttpRequestBody] = bodyFromRestOfRequestReader(reqReader) + fields[FieldHttpRequestBody] = string(redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), []byte(bodyFromRestOfRequestReader(reqReader)))) redactSensitiveFields(req, fields) return fields, nil } @@ -306,6 +325,9 @@ func decomposeResponseForLogging(res *http.Response) (map[string]interface{}, er // http.Client res.Body = io.NopCloser(bytes.NewBuffer(resBody)) + if res.Request != nil { + resBody = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), resBody) + } fields[FieldHttpResponseBody] = string(resBody) redactSensitiveFields(res.Request, fields) diff --git a/zSDKs/terraform-provider-testing/internal/sdk/internal/utils/utils.go b/zSDKs/terraform-provider-testing/internal/sdk/internal/utils/utils.go index f239efa3c..0ffc2fe6d 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/internal/utils/utils.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/internal/utils/utils.go @@ -17,6 +17,7 @@ import ( "time" "github.com/hashicorp/terraform-provider-testing/internal/sdk/optionalnullable" + "github.com/hashicorp/terraform-provider-testing/internal/sdk/redact" ) const ( @@ -25,6 +26,12 @@ const ( pathParamTagKey = "pathParam" ) +// WithOperation marks requests built from ctx as belonging to operationID, so +// that transports can mask the operation's sensitive values when logging. +func WithOperation(ctx context.Context, operationID string) context.Context { + return redact.WithOperation(ctx, operationID) +} + var ( paramRegex = regexp.MustCompile(`({.*?})`) SerializationMethodToContentType = map[string]string{ diff --git a/zSDKs/terraform-provider-testing/internal/sdk/models/errors/sdkerror.go b/zSDKs/terraform-provider-testing/internal/sdk/models/errors/sdkerror.go index c633d568d..0601b95a5 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/models/errors/sdkerror.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/models/errors/sdkerror.go @@ -5,6 +5,8 @@ package errors import ( "fmt" "net/http" + + "github.com/hashicorp/terraform-provider-testing/internal/sdk/redact" ) type SDKError struct { @@ -28,7 +30,11 @@ func NewSDKError(message string, statusCode int, body string, httpRes *http.Resp func (e *SDKError) Error() string { body := "" if len(e.Body) > 0 { - body = fmt.Sprintf("\n%s", e.Body) + redacted := e.Body + if e.RawResponse != nil && e.RawResponse.Request != nil { + redacted = string(redact.ResponseBody(e.RawResponse.Request.Context(), e.RawResponse.Header.Get("Content-Type"), []byte(e.Body))) + } + body = fmt.Sprintf("\n%s", redacted) } return fmt.Sprintf("%s: Status %d%s", e.Message, e.StatusCode, body) diff --git a/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go new file mode 100644 index 000000000..4da057659 --- /dev/null +++ b/zSDKs/terraform-provider-testing/internal/sdk/redact/redact.go @@ -0,0 +1,518 @@ +// Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + +// Package redact masks values marked sensitive in the API description +// (x-speakeasy-param-sensitive or format: password) in HTTP bodies, so that +// HTTP clients and transports can log requests and responses safely. +// +// The SDK attaches the operation ID to the context of each request whose +// operation has sensitive request or response values. A transport reads it +// back from the request, or from a response's Request, to select the schema +// graph used for masking: +// +// body = redact.RequestBody(req.Context(), req.Header.Get("Content-Type"), body) +// body = redact.ResponseBody(res.Request.Context(), res.Header.Get("Content-Type"), body) +// +// JSON, application/x-www-form-urlencoded and multipart/form-data bodies are +// masked value by value. Other bodies of an operation with sensitive values are +// masked as a whole. +package redact + +import ( + "bytes" + "context" + "encoding/json" + "io" + "mime" + "mime/multipart" + "net/textproto" + "net/url" + "strings" +) + +// Mask replaces each sensitive value. +const Mask = "(sensitive)" + +const maxDepth = 64 + +type operationKey struct{} + +// WithOperation returns ctx carrying operationID. +func WithOperation(ctx context.Context, operationID string) context.Context { + return context.WithValue(ctx, operationKey{}, operationID) +} + +// Operation returns the operation ID carried by ctx, or "". +func Operation(ctx context.Context) string { + if ctx == nil { + return "" + } + id, _ := ctx.Value(operationKey{}).(string) + return id +} + +// RequestBody masks the sensitive values of the request body of the +// operation carried by ctx, serialized as contentType. Bodies of other +// operations are returned as is. +func RequestBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].request.redact(contentType, body) +} + +// ResponseBody masks the sensitive values of a response body of the +// operation carried by ctx, serialized as contentType, matching every +// response schema of the operation. Bodies of other operations are returned +// as is. +func ResponseBody(ctx context.Context, contentType string, body []byte) []byte { + return operations[Operation(ctx)].response.redact(contentType, body) +} + +// A graph describes where sensitive values sit in a body. Node 0 is a +// sentinel for "nothing sensitive below here"; roots are the body schemas. +type graph struct { + nodes []node + roots []int +} + +type node struct { + sensitive bool + // fields maps JSON property names to their schema; a property that is not + // declared uses values (additional properties or map values). + fields map[string]int + item int + values int + variants []int + // plainObject, plainArray, plainString, plainNumber and plainBoolean + // report union members accepting JSON values of that type and holding + // nothing sensitive. + plainObject bool + plainArray bool + plainString bool + plainNumber bool + plainBoolean bool +} + +type shape int + +const ( + objectShape shape = iota + arrayShape + stringShape + numberShape + booleanShape +) + +// redact returns body unchanged when it has nothing to mask. A body that +// cannot be parsed is masked as a whole. +func (g *graph) redact(contentType string, body []byte) []byte { + if g == nil || len(body) == 0 { + return body + } + if g.sensitive(g.roots) { + return []byte(Mask) + } + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil { + mediaType = strings.ToLower(strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])) + } + var masked []byte + ok := false + switch { + case mediaType == "application/x-www-form-urlencoded": + masked, _, ok = g.redactForm(body, g.roots) + case mediaType == "multipart/form-data": + masked, _, ok = g.redactMultipart(body, params["boundary"], g.roots) + case mediaType == "" || mediaType == "application/json" || mediaType == "text/json" || strings.HasSuffix(mediaType, "+json"): + masked, _, ok = g.redactJSON(body, g.roots) + } + if !ok { + return []byte(Mask) + } + return masked +} + +func (g *graph) redactJSON(body []byte, roots []int) ([]byte, bool, bool) { + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var value any + if err := dec.Decode(&value); err != nil { + return nil, false, false + } + if _, err := dec.Token(); err != io.EOF { + return nil, false, false + } + // Decoding keeps the last of duplicate keys, so the bytes returned as is + // could hold an earlier, unmasked one. + if hasDuplicateKeys(body) { + return nil, false, false + } + masked, changed := g.mask(value, roots, 0) + if !changed { + return body, false, true + } + var out bytes.Buffer + enc := json.NewEncoder(&out) + enc.SetEscapeHTML(false) + if err := enc.Encode(masked); err != nil { + return nil, false, false + } + return bytes.TrimSuffix(out.Bytes(), []byte("\n")), true, true +} + +// redactForm masks the fields of a form body. A field holding a sensitive +// value or array of values is masked, and a JSON-encoded field is masked +// within. +func (g *graph) redactForm(body []byte, roots []int) ([]byte, bool, bool) { + pairs := strings.Split(string(body), "&") + changed := false + for i, pair := range pairs { + rawKey, rawValue, _ := strings.Cut(pair, "=") + key, err := url.QueryUnescape(rawKey) + if err != nil || isJSONStructure([]byte(key)) { + return nil, false, false + } + value, err := url.QueryUnescape(rawValue) + if err != nil { + return nil, false, false + } + masked, fieldChanged := g.redactValue([]byte(value), g.field(roots, key)) + if !fieldChanged { + continue + } + changed = true + if string(masked) == Mask { + pairs[i] = rawKey + "=" + Mask + } else { + pairs[i] = rawKey + "=" + url.QueryEscape(string(masked)) + } + } + if !changed { + return body, false, true + } + return []byte(strings.Join(pairs, "&")), true, true +} + +// redactMultipart masks the parts of a multipart body by part name, keeping +// the boundary and the part headers. +func (g *graph) redactMultipart(body []byte, boundary string, roots []int) ([]byte, bool, bool) { + if boundary == "" { + return nil, false, false + } + reader := multipart.NewReader(bytes.NewReader(body), boundary) + var out bytes.Buffer + writer := multipart.NewWriter(&out) + if err := writer.SetBoundary(boundary); err != nil { + return nil, false, false + } + changed := false + for { + part, err := reader.NextRawPart() + if err == io.EOF { + break + } + if err != nil { + return nil, false, false + } + data, err := io.ReadAll(part) + if err != nil { + return nil, false, false + } + partRoots := g.field(roots, part.FormName()) + masked, partChanged := g.redactValue(data, partRoots) + if !partChanged && len(g.expand(partRoots)) > 0 && isEncodedPart(part.Header) { + masked, partChanged = []byte(Mask), true + } + changed = changed || partChanged + header := part.Header + if partChanged { + // The mask is written as is, not in the part's transfer encoding, + // and with its own length. + header = make(textproto.MIMEHeader, len(part.Header)) + for key, values := range part.Header { + header[key] = values + } + header.Del("Content-Transfer-Encoding") + header.Del("Content-Encoding") + header.Del("Content-Length") + } + partWriter, err := writer.CreatePart(header) + if err != nil { + return nil, false, false + } + if _, err := partWriter.Write(masked); err != nil { + return nil, false, false + } + } + if !changed { + return body, false, true + } + if err := writer.Close(); err != nil { + return nil, false, false + } + return out.Bytes(), true, true +} + +// redactValue masks a form field or multipart part. A value holding a +// sensitive value, or one that cannot be parsed as JSON while holding one, is +// masked as a whole. +func (g *graph) redactValue(value []byte, roots []int) ([]byte, bool) { + if len(g.expand(roots)) == 0 { + return value, false + } + items := g.items(roots) + if g.sensitive(roots) || g.sensitive(items) && !isJSONStructure(value) { + return []byte(Mask), true + } + // Unencoded text matches a plain string member of the field or of its + // items when sent as repeated fields, unless it starts like a JSON object + // or array, which may carry sensitive values. + if (g.plain(roots, stringShape) || g.plain(items, stringShape)) && !isJSONStructure(value) { + return value, false + } + // A scalar may also be one item of an array sent as repeated fields. + if !isJSONStructure(value) { + roots = append(append([]int{}, roots...), items...) + } + if masked, changed, ok := g.redactJSON(value, roots); ok { + return masked, changed + } + return []byte(Mask), true +} + +// isEncodedPart reports whether a multipart part is transfer or content +// encoded, so that its bytes cannot be read as its value. +func isEncodedPart(header textproto.MIMEHeader) bool { + switch strings.ToLower(strings.TrimSpace(header.Get("Content-Transfer-Encoding"))) { + case "", "7bit", "8bit", "binary": + return header.Get("Content-Encoding") != "" + } + return true +} + +// hasDuplicateKeys reports whether a JSON object in body repeats a key. +func hasDuplicateKeys(body []byte) bool { + type object struct { + keys map[string]bool + expectKey bool + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var stack []*object + for { + token, err := dec.Token() + if err != nil { + return false + } + switch t := token.(type) { + case json.Delim: + switch t { + case '{': + stack = append(stack, &object{keys: map[string]bool{}, expectKey: true}) + continue + case '[': + stack = append(stack, nil) + continue + } + stack = stack[:len(stack)-1] + case string: + if n := len(stack); n > 0 && stack[n-1] != nil && stack[n-1].expectKey { + if stack[n-1].keys[t] { + return true + } + stack[n-1].keys[t] = true + stack[n-1].expectKey = false + continue + } + } + if n := len(stack); n > 0 && stack[n-1] != nil { + stack[n-1].expectKey = true + } + } +} + +func isJSONStructure(value []byte) bool { + trimmed := bytes.TrimLeft(value, " \t\r\n") + return len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '[') +} + +func (g *graph) mask(value any, roots []int, depth int) (any, bool) { + roots = g.expand(roots) + if len(roots) == 0 { + return value, false + } + if g.sensitive(roots) { + return Mask, true + } + if depth >= maxDepth { + return Mask, true + } + // A value that does not have the shape of its schema cannot be followed + // to its sensitive values, so it is masked as a whole. + changed := false + switch v := value.(type) { + case nil: + case map[string]any: + if !g.hasProperties(roots) { + return g.mismatch(value, roots, objectShape) + } + for key, child := range v { + if masked, ok := g.mask(child, g.child(roots, key), depth+1); ok { + v[key] = masked + changed = true + } + } + case []any: + items := g.items(roots) + if len(items) == 0 { + return g.mismatch(value, roots, arrayShape) + } + for i, child := range v { + if masked, ok := g.mask(child, items, depth+1); ok { + v[i] = masked + changed = true + } + } + case string: + return g.mismatch(value, roots, stringShape) + case json.Number, float64: + return g.mismatch(value, roots, numberShape) + case bool: + return g.mismatch(value, roots, booleanShape) + default: + return Mask, true + } + return value, changed +} + +// mismatch masks a value that does not have the shape of roots, unless a +// union member of its shape holding nothing sensitive may match it. +func (g *graph) mismatch(value any, roots []int, s shape) (any, bool) { + if g.plain(roots, s) { + return value, false + } + return Mask, true +} + +// plain reports whether any of roots, or their union variants, is a union +// with a member of shape s holding nothing sensitive. +func (g *graph) plain(roots []int, s shape) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].plainShape(s) { + return true + } + } + return false +} + +func (n node) plainShape(s shape) bool { + switch s { + case objectShape: + return n.plainObject + case arrayShape: + return n.plainArray + case stringShape: + return n.plainString + case numberShape: + return n.plainNumber + case booleanShape: + return n.plainBoolean + } + return false +} + +// hasProperties reports whether any of roots describes object properties or +// map values. +func (g *graph) hasProperties(roots []int) bool { + for _, id := range g.expand(roots) { + if len(g.nodes[id].fields) > 0 || g.nodes[id].values != 0 { + return true + } + } + return false +} + +// sensitive reports whether any of roots, or their union variants, is +// sensitive as a whole. +func (g *graph) sensitive(roots []int) bool { + for _, id := range g.expand(roots) { + if g.nodes[id].sensitive { + return true + } + } + return false +} + +// child returns the schemas of the property name of roots. +func (g *graph) child(roots []int, name string) []int { + var children []int + for _, id := range g.expand(roots) { + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) + } + } + return children +} + +// field returns the schemas of a form field or multipart part of roots. In +// each schema, a name ending in [] that is not declared as such names an +// array sent as repeated fields. +func (g *graph) field(roots []int, name string) []int { + trimmed, bracketed := strings.CutSuffix(name, "[]") + var children []int + for _, id := range g.expand(roots) { + n := g.nodes[id] + childID, declared := n.fields[name] + if !declared && bracketed { + childID, declared = n.fields[trimmed] + } + if !declared { + childID = n.values + } + if childID != 0 { + children = append(children, childID) + } + } + return children +} + +// items returns the array item schemas of roots. +func (g *graph) items(roots []int) []int { + var items []int + for _, id := range g.expand(roots) { + if item := g.nodes[id].item; item != 0 { + items = append(items, item) + } + } + return items +} + +// expand adds the union variants of roots, since the variant a value matches +// is not known. +func (g *graph) expand(roots []int) []int { + var expanded []int + seen := make(map[int]bool) + var visit func(int) + visit = func(id int) { + if id <= 0 || id >= len(g.nodes) || seen[id] { + return + } + seen[id] = true + expanded = append(expanded, id) + for _, variant := range g.nodes[id].variants { + visit(variant) + } + } + for _, id := range roots { + visit(id) + } + return expanded +} + +var operations = map[string]struct{ request, response *graph }{ + "create-x-param-sensitive": {request: &graph{nodes: []node{{}, {fields: map[string]int{"string_request": 3, "string_request_and_response": 2, "string_response": 0}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}, response: &graph{nodes: []node{{}, {fields: map[string]int{"id": 0, "string_request": 0, "string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, + "get-x-param-sensitive": {response: &graph{nodes: []node{{}, {fields: map[string]int{"id": 0, "string_request": 0, "string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, + "update-x-param-sensitive": {request: &graph{nodes: []node{{}, {fields: map[string]int{"string_request": 3, "string_request_and_response": 2, "string_response": 0}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}, response: &graph{nodes: []node{{}, {fields: map[string]int{"id": 0, "string_request": 0, "string_request_and_response": 2, "string_response": 3}}, {sensitive: true}, {sensitive: true}}, roots: []int{1}}}, +} diff --git a/zSDKs/terraform-provider-testing/internal/sdk/sdk.go b/zSDKs/terraform-provider-testing/internal/sdk/sdk.go index 4dc8e6c7f..461079173 100644 --- a/zSDKs/terraform-provider-testing/internal/sdk/sdk.go +++ b/zSDKs/terraform-provider-testing/internal/sdk/sdk.go @@ -32302,6 +32302,7 @@ func (s *SDK) CreateXParamSensitive(ctx context.Context, request shared.XParamSe defer cancel() } + ctx = utils.WithOperation(ctx, "create-x-param-sensitive") req, err := http.NewRequestWithContext(ctx, "POST", opURL, bodyReader) if err != nil { return nil, fmt.Errorf("error creating request: %w", err) @@ -32546,6 +32547,7 @@ func (s *SDK) GetXParamSensitive(ctx context.Context, request operations.GetXPar defer cancel() } + ctx = utils.WithOperation(ctx, "get-x-param-sensitive") req, err := http.NewRequestWithContext(ctx, "GET", opURL, nil) if err != nil { return nil, fmt.Errorf("error creating request: %w", err) @@ -32681,6 +32683,7 @@ func (s *SDK) UpdateXParamSensitive(ctx context.Context, request operations.Upda defer cancel() } + ctx = utils.WithOperation(ctx, "update-x-param-sensitive") req, err := http.NewRequestWithContext(ctx, "PATCH", opURL, bodyReader) if err != nil { return nil, fmt.Errorf("error creating request: %w", err)