diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d1663cc --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,36 @@ +--- +name: CI + +"on": + push: + pull_request: + +permissions: + contents: read + +jobs: + test: + name: Build and test + runs-on: ubuntu-latest + env: + GOFLAGS: -mod=vendor + steps: + - name: Check out repository + # v7.0.1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + + - name: Set up Go + # v7.0.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e + with: + go-version-file: go.mod + cache: true + + - name: Build + run: go build ./... + + - name: Vet + run: go vet ./... + + - name: Test + run: go test ./... diff --git a/README.md b/README.md index 5fba9ce..71acfc6 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,35 @@ In practice that limits the transpiler to basic Butane features such as users, g - End to end integration with Flatcar and ClusterAPI. - Documentation covering supported fields and known limitations. +### Usage + +Build `bt`, then pass a cloud-config file or pipe one on standard input: + +```sh +go build -o bt ./cmd/bt +bt cloud-config.yaml > butane.yaml +# or: cat cloud-config.yaml | bt -o butane.yaml +``` + +`bt` currently converts only the Cluster API worker `users` subset. A config +is either converted completely or rejected; unsupported fields are never +silently discarded. + +| cloud-config field | Butane field | +| ----------------------- | --------------------- | +| `name` | `name` | +| `passwd` | `password_hash` | +| `gecos` | `gecos` | +| `homedir` | `home_dir` | +| `shell` | `shell` | +| `ssh_authorized_keys` | `ssh_authorized_keys` | + +Password hashes remain locked, matching cloud-init's default +`lock_passwd: true`. Supplementary and primary groups, sudo configuration, +password unlocking, and every non-`users` stanza are not yet supported. +Jinja templates are not evaluated and are rejected; render them before passing +the resulting cloud-config to `bt`. + ### A Flatcar Container Linux project Flatcar Container Linux is a fully open source, minimal-footprint, secure by default and always up-to-date Linux distribution for running containers at scale. diff --git a/cmd/bt/main.go b/cmd/bt/main.go new file mode 100644 index 0000000..415c9f8 --- /dev/null +++ b/cmd/bt/main.go @@ -0,0 +1,105 @@ +package main + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/flatcar/Butane-init" + "github.com/spf13/cobra" +) + +var errUsage = errors.New("invalid command usage") + +func main() { + cmd := newCommand(os.Stdin, os.Stdout, os.Stderr) + if err := cmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + if errors.Is(err, errUsage) { + os.Exit(2) + } + os.Exit(1) + } +} + +func newCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { + var outputPath string + cmd := &cobra.Command{ + Use: "bt [input-file]", + Short: "Transpile cloud-config YAML to Flatcar Butane YAML", + SilenceErrors: true, + SilenceUsage: true, + Args: func(_ *cobra.Command, args []string) error { + if len(args) > 1 { + return fmt.Errorf("%w: expected at most one input file", errUsage) + } + return nil + }, + RunE: func(_ *cobra.Command, args []string) error { + input, err := readInput(stdin, args) + if err != nil { + return err + } + output, err := transpile.Transpile(input) + if err != nil { + return err + } + if outputPath == "" || outputPath == "-" { + _, err = stdout.Write(output) + return err + } + return writeAtomically(outputPath, output) + }, + } + cmd.SetIn(stdin) + cmd.SetOut(stdout) + cmd.SetErr(stderr) + cmd.SetFlagErrorFunc(func(_ *cobra.Command, err error) error { + return fmt.Errorf("%w: %v", errUsage, err) + }) + cmd.Flags().StringVarP(&outputPath, "output", "o", "", "write output to a file instead of stdout") + return cmd +} + +func readInput(stdin io.Reader, args []string) ([]byte, error) { + if len(args) == 0 || args[0] == "-" { + input, err := io.ReadAll(stdin) + if err != nil { + return nil, fmt.Errorf("read stdin: %w", err) + } + return input, nil + } + input, err := os.ReadFile(args[0]) + if err != nil { + return nil, fmt.Errorf("read %q: %w", args[0], err) + } + return input, nil +} + +func writeAtomically(path string, contents []byte) error { + dir := filepath.Dir(path) + temp, err := os.CreateTemp(dir, ".bt-*") + if err != nil { + return fmt.Errorf("create temporary output: %w", err) + } + tempPath := temp.Name() + defer os.Remove(tempPath) + + if err := temp.Chmod(0o600); err != nil { + temp.Close() + return fmt.Errorf("set output permissions: %w", err) + } + if _, err := temp.Write(contents); err != nil { + temp.Close() + return fmt.Errorf("write output: %w", err) + } + if err := temp.Close(); err != nil { + return fmt.Errorf("close output: %w", err) + } + if err := os.Rename(tempPath, path); err != nil { + return fmt.Errorf("replace %q: %w", path, err) + } + return nil +} diff --git a/cmd/bt/main_test.go b/cmd/bt/main_test.go new file mode 100644 index 0000000..fd9a7ac --- /dev/null +++ b/cmd/bt/main_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "testing" +) + +func TestCommandTranspilesStdinToStdout(t *testing.T) { + stdin := bytes.NewBufferString("#cloud-config\nusers: []\n") + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + cmd := newCommand(stdin, stdout, stderr) + cmd.SetArgs(nil) + + if err := cmd.Execute(); err != nil { + t.Fatalf("Execute() error = %v; stderr = %q", err, stderr.String()) + } + + want := "version: 1.1.0\nvariant: flatcar\n" + if got := stdout.String(); got != want { + t.Fatalf("stdout mismatch\nwant:\n%s\ngot:\n%s", want, got) + } + if got := stderr.String(); got != "" { + t.Fatalf("stderr = %q, want empty", got) + } +} + +func TestWriteAtomicallyCreatesPrivateOutput(t *testing.T) { + path := filepath.Join(t.TempDir(), "butane.yaml") + if err := writeAtomically(path, []byte("contents")); err != nil { + t.Fatalf("writeAtomically() error = %v", err) + } + + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat output: %v", err) + } + if got, want := info.Mode().Perm(), os.FileMode(0o600); got != want { + t.Fatalf("output permissions = %o, want %o", got, want) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f7fdf83 --- /dev/null +++ b/go.mod @@ -0,0 +1,27 @@ +module github.com/flatcar/Butane-init + +go 1.27.0 + +require ( + github.com/coreos/ignition/v2 v2.27.0 + github.com/goccy/go-yaml v1.19.2 + github.com/spf13/cobra v1.10.2 +) + +require ( + github.com/aws/aws-sdk-go-v2 v1.43.4 // indirect + github.com/clarketm/json v1.17.1 // indirect + github.com/coreos/go-json v0.0.0-20230131223807-18775e0fb4fb // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.7.0 // indirect + github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/vincent-petithory/dataurl v1.0.0 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..db7cfe3 --- /dev/null +++ b/go.sum @@ -0,0 +1,48 @@ +github.com/aws/aws-sdk-go-v2 v1.43.4 h1:b9FTvbRwy+JCsfp2Wp6wV/KbOx3Aj7nkoFb2cRX0IhE= +github.com/aws/aws-sdk-go-v2 v1.43.4/go.mod h1:70vwSy16txshwG+g55WkpgPKDIByzHI8ccBsOteo3bQ= +github.com/clarketm/json v1.17.1 h1:U1IxjqJkJ7bRK4L6dyphmoO840P6bdhPdbbLySourqI= +github.com/clarketm/json v1.17.1/go.mod h1:ynr2LRfb0fQU34l07csRNBTcivjySLLiY1YzQqKVfdo= +github.com/coreos/go-json v0.0.0-20230131223807-18775e0fb4fb h1:rmqyI19j3Z/74bIRhuC59RB442rXUazKNueVpfJPxg4= +github.com/coreos/go-json v0.0.0-20230131223807-18775e0fb4fb/go.mod h1:rcFZM3uxVvdyNmsAV2jopgPD1cs5SPWJWU5dOz2LUnw= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= +github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= +github.com/coreos/ignition/v2 v2.27.0 h1:kEWbmy7HnDeOAOxQOrZFPcyvL2GPCOZSW6Tu+f7IhTA= +github.com/coreos/ignition/v2 v2.27.0/go.mod h1:/TFCOigkyeJze8G74Dev5MXB9Zcz4I+GLgHrBr/cTrA= +github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687 h1:uSmlDgJGbUB0bwQBcZomBTottKwEDF5fF8UjSwKSzWM= +github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687/go.mod h1:Salmysdw7DAVuobBW/LwsKKgpyCPHUhjyJoMJD+ZJiI= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= +github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/testcases/README.md b/testcases/README.md new file mode 100644 index 0000000..e30ed8e --- /dev/null +++ b/testcases/README.md @@ -0,0 +1,24 @@ +# Cluster API cloud-config fixtures + +These fixtures are adapted from Cluster API v1.11.1 at commit +`dc0fb87601071371044f60c59096ecad1b84397f`: + +- The + [cloud-init users template](https://github.com/kubernetes-sigs/cluster-api/blob/dc0fb87601071371044f60c59096ecad1b84397f/bootstrap/kubeadm/internal/cloudinit/users.go#L20-L57) + defines the emitted field names and value shapes. +- The + [populated user test](https://github.com/kubernetes-sigs/cluster-api/blob/dc0fb87601071371044f60c59096ecad1b84397f/bootstrap/kubeadm/internal/ignition/clc/clc_test.go#L85-L99) + supplies representative values. +- The + [cloud-init users and groups documentation](https://docs.cloud-init.io/en/24.2/reference/modules.html#users-and-groups) + defines the input semantics. + +Secrets and SSH keys are nonfunctional test values. The fixtures are deliberately +small excerpts of Cluster API output so each one isolates a transpilation +boundary. + +| Fixture | Purpose | Current result | +| --- | --- | --- | +| `cluster-api-supported-user.yaml` | Every currently supported user field | Success | +| `cluster-api-groups.yaml` | Cluster API's comma-separated groups and primary group | Rejected until group support lands | +| `cluster-api-deferred-fields.yaml` | Remaining rendered account-policy fields | Rejected as unsupported | diff --git a/testcases/cluster-api-deferred-fields.yaml b/testcases/cluster-api-deferred-fields.yaml new file mode 100644 index 0000000..c4e2a4f --- /dev/null +++ b/testcases/cluster-api-deferred-fields.yaml @@ -0,0 +1,7 @@ +#cloud-config +# Cluster API renders these account-policy fields as scalar values. +users: + - name: fixture-user + inactive: true + lock_passwd: false + sudo: "ALL=(ALL) NOPASSWD:ALL" diff --git a/testcases/cluster-api-groups.yaml b/testcases/cluster-api-groups.yaml new file mode 100644 index 0000000..649015c --- /dev/null +++ b/testcases/cluster-api-groups.yaml @@ -0,0 +1,6 @@ +#cloud-config +# Cluster API renders additional groups as one comma-separated scalar. +users: + - name: foo + groups: "foo, bar" + primary_group: foo diff --git a/testcases/cluster-api-supported-user.yaml b/testcases/cluster-api-supported-user.yaml new file mode 100644 index 0000000..daede5a --- /dev/null +++ b/testcases/cluster-api-supported-user.yaml @@ -0,0 +1,10 @@ +#cloud-config +# Adapted from the Cluster API v1.11.1 populated-user test. +users: + - name: foo + passwd: "$6$REDACTED_TEST_HASH" + gecos: Foo B. Bar + homedir: /home/foo + shell: /bin/false + ssh_authorized_keys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIREDACTED fixture@example diff --git a/transpile.go b/transpile.go new file mode 100644 index 0000000..ee8f831 --- /dev/null +++ b/transpile.go @@ -0,0 +1,148 @@ +// Package transpile converts the supported cloud-config subset to Butane. +package transpile + +import ( + "fmt" + "sort" + "strings" + + butane "github.com/coreos/ignition/v2/butane/config" + butanecommon "github.com/coreos/ignition/v2/butane/config/common" + schema "github.com/coreos/ignition/v2/butane/config/flatcar/v1_1" + "github.com/goccy/go-yaml" + "github.com/goccy/go-yaml/ast" + "github.com/goccy/go-yaml/lexer" + "github.com/goccy/go-yaml/parser" + "github.com/goccy/go-yaml/token" +) + +const ( + variant = "flatcar" + version = "1.1.0" +) + +// ValidationProblem describes one invalid or unsupported cloud-config value. +type ValidationProblem struct { + Path string + Message string + Line int + Column int +} + +// ValidationErrors contains all problems found while validating a cloud-config. +type ValidationErrors []ValidationProblem + +func (e ValidationErrors) Error() string { + messages := make([]string, len(e)) + for i, problem := range e { + if problem.Line > 0 { + messages[i] = fmt.Sprintf("[%d:%d] %s: %s", problem.Line, problem.Column, problem.Path, problem.Message) + } else { + messages[i] = fmt.Sprintf("%s: %s", problem.Path, problem.Message) + } + } + return strings.Join(messages, "\n") +} + +// Transpile converts one cloud-config document into a Flatcar Butane config. +func Transpile(input []byte) ([]byte, error) { + if err := validateCloudConfigHeader(input); err != nil { + return nil, fmt.Errorf("validating cloud config header: %w", err) + } + + file, err := parser.ParseBytes(input, 0) + if err != nil { + return nil, fmt.Errorf("parse cloud-config: %s", yaml.FormatError(err, false, true)) + } + if len(file.Docs) > 1 { + return nil, fmt.Errorf("cloud-config must contain exactly one YAML document") + } + if err := rejectYAMLReferences(input); err != nil { + return nil, err + } + + var document map[string]any + if len(file.Docs) == 0 { + document = map[string]any{} + } else if err := yaml.Unmarshal(input, &document); err != nil { + return nil, fmt.Errorf("decode cloud-config: %s", yaml.FormatError(err, false, true)) + } + + var problems ValidationErrors + for _, key := range sortedKeys(document) { + if key != "users" { + problems = append(problems, problem(file, key, "unsupported field")) + } + } + + users, userProblems := parseUsers(document, file) + problems = append(problems, userProblems...) + if len(problems) > 0 { + return nil, problems + } + + config := schema.Config{Variant: variant, Version: version} + if len(users) > 0 { + config.Passwd.Users = users + } + out, err := yaml.MarshalWithOptions(config, yaml.OmitZero(), yaml.Indent(2), yaml.IndentSequence(true)) + if err != nil { + return nil, fmt.Errorf("encode Butane config: %w", err) + } + if _, _, err := butane.TranslateBytes(out, butanecommon.TranslateBytesOptions{}); err != nil { + return nil, fmt.Errorf("validate generated Butane config: %w", err) + } + return out, nil +} + +func validateCloudConfigHeader(input []byte) error { + firstLine, _, _ := strings.Cut(string(input), "\n") + switch strings.TrimSpace(firstLine) { + case "## template: jinja": + return fmt.Errorf("Jinja templates are unsupported; render the template before transpiling") + case "#cloud-config": + return nil + default: + return fmt.Errorf("missing #cloud-config header") + } +} + +func rejectYAMLReferences(input []byte) error { + for _, tok := range lexer.Tokenize(string(input)) { + if tok.Type == token.AnchorType || tok.Type == token.AliasType || tok.Type == token.MergeKeyType { + return fmt.Errorf("[%d:%d] YAML aliases and anchors are unsupported", tok.Position.Line, tok.Position.Column) + } + } + return nil +} + +func sortedKeys(values map[string]any) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func problem(file *ast.File, path, message string) ValidationProblem { + problem := ValidationProblem{Path: path, Message: message} + locationPath := path + for locationPath != "" { + yamlPath, err := yaml.PathString("$." + locationPath) + if err == nil { + if node, err := yamlPath.FilterFile(file); err == nil && node.GetToken() != nil { + pos := node.GetToken().Position + problem.Line = pos.Line + problem.Column = pos.Column + return problem + } + } + separator := strings.LastIndex(locationPath, ".") + if separator == -1 { + break + } + locationPath = locationPath[:separator] + } + return problem +} diff --git a/transpile_test.go b/transpile_test.go new file mode 100644 index 0000000..07c869b --- /dev/null +++ b/transpile_test.go @@ -0,0 +1,116 @@ +package transpile_test + +import ( + "errors" + "os" + "reflect" + "strings" + "testing" + + "github.com/flatcar/Butane-init" +) + +func TestTranspileRejectsInvalidDocument(t *testing.T) { + tests := map[string]struct { + input string + wantErr string + }{ + "missing header": { + input: "users: []\n", + wantErr: "missing #cloud-config header", + }, + "arbitrary comment before header": { + input: "# generated config\n#cloud-config\nusers: []\n", + wantErr: "missing #cloud-config header", + }, + "blank line before header": { + input: "\n#cloud-config\nusers: []\n", + wantErr: "missing #cloud-config header", + }, + "jinja template": { + input: "## template: jinja\n#cloud-config\nusers:\n - name: alice\n gecos: \"{{ ds.meta_data.name }}\"\n", + wantErr: "Jinja templates are unsupported", + }, + "unsupported top-level field": { + input: "#cloud-config\nruncmd: []\n", + wantErr: "runcmd", + }, + "yaml alias": { + input: "#cloud-config\nusers:\n - &user\n name: alice\n - *user\n", + wantErr: "YAML aliases and anchors are unsupported", + }, + "multiple documents": { + input: "#cloud-config\nusers: []\n---\nusers: []\n", + wantErr: "exactly one YAML document", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := transpile.Transpile([]byte(tt.input)) + if err == nil { + t.Fatalf("Transpile() error = nil, want an error containing %q", tt.wantErr) + } + if got != nil { + t.Fatalf("Transpile() output = %q, want nil on error", got) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("Transpile() error = %q, want it to contain %q", err, tt.wantErr) + } + }) + } +} + +func TestTranspileReturnsStructuredValidationErrors(t *testing.T) { + input := "#cloud-config\nruncmd: []\nusers:\n - shell: \"\"\n" + + got, err := transpile.Transpile([]byte(input)) + if err == nil { + t.Fatal("Transpile() error = nil, want validation errors") + } + if got != nil { + t.Fatalf("Transpile() output = %q, want nil on error", got) + } + + var problems transpile.ValidationErrors + if !errors.As(err, &problems) { + t.Fatalf("Transpile() error type = %T, want transpile.ValidationErrors", err) + } + want := transpile.ValidationErrors{ + {Path: "runcmd", Message: "unsupported field", Line: 2, Column: 9}, + {Path: "users[0].name", Message: "is required", Line: 4, Column: 10}, + {Path: "users[0].shell", Message: "must not be empty", Line: 4, Column: 12}, + } + if !reflect.DeepEqual(problems, want) { + t.Fatalf("Transpile() validation errors = %#v, want %#v", problems, want) + } + + wantMessage := "[2:9] runcmd: unsupported field\n" + + "[4:10] users[0].name: is required\n" + + "[4:12] users[0].shell: must not be empty" + if err.Error() != wantMessage { + t.Fatalf("Transpile() error = %q, want %q", err, wantMessage) + } +} + +func TestTranspileOmitsZeroValuedSchemaFields(t *testing.T) { + input := "#cloud-config\nusers:\n - name: alice\n" + want := "version: 1.1.0\nvariant: flatcar\npasswd:\n users:\n - name: alice\n" + + got, err := transpile.Transpile([]byte(input)) + if err != nil { + t.Fatalf("Transpile() error = %v", err) + } + if string(got) != want { + t.Fatalf("Transpile() output mismatch\nwant:\n%s\ngot:\n%s", want, got) + } +} + +func readFixture(t *testing.T, name string) string { + t.Helper() + contents, err := os.ReadFile("testcases/" + name) + if err != nil { + t.Fatalf("read fixture %q: %v", name, err) + } + return string(contents) +} diff --git a/users.go b/users.go new file mode 100644 index 0000000..92319b5 --- /dev/null +++ b/users.go @@ -0,0 +1,139 @@ +package transpile + +import ( + "fmt" + "strings" + + base "github.com/coreos/ignition/v2/butane/base/v0_5" + "github.com/goccy/go-yaml/ast" +) + +func parseUsers(document map[string]any, file *ast.File) ([]base.PasswdUser, ValidationErrors) { + var problems ValidationErrors + rawUsers, exists := document["users"] + if !exists { + return nil, problems + } + items, ok := rawUsers.([]any) + if !ok { + problems = append(problems, problem(file, "users", "must be a list")) + return nil, problems + } + + users := make([]base.PasswdUser, 0, len(items)) + seenUsers := map[string]struct{}{} + for i, item := range items { + path := fmt.Sprintf("users[%d]", i) + fields, ok := item.(map[string]any) + if !ok { + problems = append(problems, problem(file, path, "must be an object")) + continue + } + + user, userProblems := parseUser(fields, path, file) + problems = append(problems, userProblems...) + if user.Name != "" { + if _, duplicate := seenUsers[user.Name]; duplicate { + problems = append(problems, problem(file, path+".name", fmt.Sprintf("duplicate user %q", user.Name))) + } else { + seenUsers[user.Name] = struct{}{} + } + } + users = append(users, user) + } + return users, problems +} + +func parseUser(fields map[string]any, path string, file *ast.File) (base.PasswdUser, ValidationErrors) { + allowed := map[string]bool{ + "name": true, "passwd": true, "gecos": true, "homedir": true, + "shell": true, "ssh_authorized_keys": true, + } + var problems ValidationErrors + for _, key := range sortedKeys(fields) { + if !allowed[key] { + problems = append(problems, problem(file, path+"."+key, "unsupported field")) + } + } + + var user base.PasswdUser + user.Name = requiredString(fields, "name", path, file, &problems) + user.Gecos = optionalString(fields, "gecos", path, file, &problems) + user.HomeDir = optionalString(fields, "homedir", path, file, &problems) + user.Shell = optionalString(fields, "shell", path, file, &problems) + if passwd := optionalString(fields, "passwd", path, file, &problems); passwd != nil { + locked := lockPassword(*passwd) + user.PasswordHash = &locked + } + user.SSHAuthorizedKeys = sshKeys(fields, path, file, &problems) + return user, problems +} + +func requiredString(fields map[string]any, key, path string, file *ast.File, problems *ValidationErrors) string { + value := optionalString(fields, key, path, file, problems) + if value == nil { + if _, exists := fields[key]; !exists { + *problems = append(*problems, problem(file, path+"."+key, "is required")) + } + return "" + } + return *value +} + +func optionalString(fields map[string]any, key, path string, file *ast.File, problems *ValidationErrors) *string { + raw, exists := fields[key] + if !exists { + return nil + } + value, ok := raw.(string) + if !ok { + *problems = append(*problems, problem(file, path+"."+key, "must be a string")) + return nil + } + if value == "" { + *problems = append(*problems, problem(file, path+"."+key, "must not be empty")) + return nil + } + return &value +} + +func sshKeys(fields map[string]any, path string, file *ast.File, problems *ValidationErrors) []base.SSHAuthorizedKey { + raw, exists := fields["ssh_authorized_keys"] + if !exists { + return nil + } + items, ok := raw.([]any) + if !ok { + *problems = append(*problems, problem(file, path+".ssh_authorized_keys", "must be a list")) + return nil + } + if len(items) == 0 { + *problems = append(*problems, problem(file, path+".ssh_authorized_keys", "must not be empty")) + return nil + } + + keys := make([]base.SSHAuthorizedKey, 0, len(items)) + seen := map[string]struct{}{} + for i, rawKey := range items { + keyPath := fmt.Sprintf("%s.ssh_authorized_keys[%d]", path, i) + key, ok := rawKey.(string) + if !ok || key == "" { + *problems = append(*problems, problem(file, keyPath, "must be a non-empty string")) + continue + } + if _, duplicate := seen[key]; duplicate { + *problems = append(*problems, problem(file, keyPath, "duplicate SSH authorized key")) + continue + } + seen[key] = struct{}{} + keys = append(keys, base.SSHAuthorizedKey(key)) + } + return keys +} + +func lockPassword(hash string) string { + if strings.HasPrefix(hash, "!") || strings.HasPrefix(hash, "*") { + return hash + } + return "!" + hash +} diff --git a/users_test.go b/users_test.go new file mode 100644 index 0000000..5463c5e --- /dev/null +++ b/users_test.go @@ -0,0 +1,76 @@ +package transpile_test + +import ( + "strings" + "testing" + + "github.com/flatcar/Butane-init" +) + +func TestTranspileClusterAPIWorkerUser(t *testing.T) { + input := readFixture(t, "cluster-api-supported-user.yaml") + + want := `version: 1.1.0 +variant: flatcar +passwd: + users: + - gecos: Foo B. Bar + home_dir: /home/foo + name: foo + password_hash: "!$6$REDACTED_TEST_HASH" + ssh_authorized_keys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIREDACTED fixture@example + shell: /bin/false +` + + got, err := transpile.Transpile([]byte(input)) + if err != nil { + t.Fatalf("Transpile() error = %v", err) + } + if string(got) != want { + t.Fatalf("Transpile() output mismatch\nwant:\n%s\ngot:\n%s", want, got) + } +} + +func TestTranspileRejectsInvalidUsers(t *testing.T) { + tests := map[string]struct { + input string + wantErr string + }{ + "Cluster API groups fixture": { + input: readFixture(t, "cluster-api-groups.yaml"), + wantErr: "groups", + }, + "Cluster API deferred fields fixture": { + input: readFixture(t, "cluster-api-deferred-fields.yaml"), + wantErr: "inactive", + }, + "empty optional field": { + input: "#cloud-config\nusers:\n - name: alice\n shell: \"\"\n", + wantErr: "users[0].shell", + }, + "duplicate username": { + input: "#cloud-config\nusers:\n - name: alice\n - name: alice\n", + wantErr: "duplicate user", + }, + "missing username has source location": { + input: "#cloud-config\nusers:\n - shell: /bin/bash\n", + wantErr: "[3:10] users[0].name: is required", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := transpile.Transpile([]byte(tt.input)) + if err == nil { + t.Fatalf("Transpile() error = nil, want an error containing %q", tt.wantErr) + } + if got != nil { + t.Fatalf("Transpile() output = %q, want nil on error", got) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("Transpile() error = %q, want it to contain %q", err, tt.wantErr) + } + }) + } +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/LICENSE.txt b/vendor/github.com/aws/aws-sdk-go-v2/LICENSE.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/aws/aws-sdk-go-v2/NOTICE.txt b/vendor/github.com/aws/aws-sdk-go-v2/NOTICE.txt new file mode 100644 index 0000000..899129e --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/NOTICE.txt @@ -0,0 +1,3 @@ +AWS SDK for Go +Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +Copyright 2014-2015 Stripe, Inc. diff --git a/vendor/github.com/aws/aws-sdk-go-v2/aws/arn/arn.go b/vendor/github.com/aws/aws-sdk-go-v2/aws/arn/arn.go new file mode 100644 index 0000000..fe63fed --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/aws/arn/arn.go @@ -0,0 +1,92 @@ +// Package arn provides a parser for interacting with Amazon Resource Names. +package arn + +import ( + "errors" + "strings" +) + +const ( + arnDelimiter = ":" + arnSections = 6 + arnPrefix = "arn:" + + // zero-indexed + sectionPartition = 1 + sectionService = 2 + sectionRegion = 3 + sectionAccountID = 4 + sectionResource = 5 + + // errors + invalidPrefix = "arn: invalid prefix" + invalidSections = "arn: not enough sections" +) + +// ARN captures the individual fields of an Amazon Resource Name. +// See http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html for more information. +type ARN struct { + // The partition that the resource is in. For standard AWS regions, the partition is "aws". If you have resources in + // other partitions, the partition is "aws-partitionname". For example, the partition for resources in the China + // (Beijing) region is "aws-cn". + Partition string + + // The service namespace that identifies the AWS product (for example, Amazon S3, IAM, or Amazon RDS). For a list of + // namespaces, see + // http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#genref-aws-service-namespaces. + Service string + + // The region the resource resides in. Note that the ARNs for some resources do not require a region, so this + // component might be omitted. + Region string + + // The ID of the AWS account that owns the resource, without the hyphens. For example, 123456789012. Note that the + // ARNs for some resources don't require an account number, so this component might be omitted. + AccountID string + + // The content of this part of the ARN varies by service. It often includes an indicator of the type of resource — + // for example, an IAM user or Amazon RDS database - followed by a slash (/) or a colon (:), followed by the + // resource name itself. Some services allows paths for resource names, as described in + // http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arns-paths. + Resource string +} + +// Parse parses an ARN into its constituent parts. +// +// Some example ARNs: +// arn:aws:elasticbeanstalk:us-east-1:123456789012:environment/My App/MyEnvironment +// arn:aws:iam::123456789012:user/David +// arn:aws:rds:eu-west-1:123456789012:db:mysql-db +// arn:aws:s3:::my_corporate_bucket/exampleobject.png +func Parse(arn string) (ARN, error) { + if !strings.HasPrefix(arn, arnPrefix) { + return ARN{}, errors.New(invalidPrefix) + } + sections := strings.SplitN(arn, arnDelimiter, arnSections) + if len(sections) != arnSections { + return ARN{}, errors.New(invalidSections) + } + return ARN{ + Partition: sections[sectionPartition], + Service: sections[sectionService], + Region: sections[sectionRegion], + AccountID: sections[sectionAccountID], + Resource: sections[sectionResource], + }, nil +} + +// IsARN returns whether the given string is an arn +// by looking for whether the string starts with arn: +func IsARN(arn string) bool { + return strings.HasPrefix(arn, arnPrefix) && strings.Count(arn, ":") >= arnSections-1 +} + +// String returns the canonical representation of the ARN +func (arn ARN) String() string { + return arnPrefix + + arn.Partition + arnDelimiter + + arn.Service + arnDelimiter + + arn.Region + arnDelimiter + + arn.AccountID + arnDelimiter + + arn.Resource +} diff --git a/vendor/github.com/clarketm/json/LICENSE b/vendor/github.com/clarketm/json/LICENSE new file mode 100644 index 0000000..6a66aea --- /dev/null +++ b/vendor/github.com/clarketm/json/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/clarketm/json/README.md b/vendor/github.com/clarketm/json/README.md new file mode 100644 index 0000000..e5c3524 --- /dev/null +++ b/vendor/github.com/clarketm/json/README.md @@ -0,0 +1,19 @@ +# [json](https://godoc.org/github.com/clarketm/json) +> Mirrors [golang/go](https://github.com/golang/go) [![Golang version](https://img.shields.io/badge/go-1.12.7-green)](https://github.com/golang/go/releases/tag/go1.12.7) + +Drop-in replacement for Golang [`encoding/json`](https://golang.org/pkg/encoding/json/) with additional features. + +## Installation +```shell +$ go get -u github.com/clarketm/json +``` + +## Usage +Same usage as Golang [`encoding/json`](https://golang.org/pkg/encoding/json/). + +## Features +- Support zero values of structs with `omitempty`: [golang/go#11939](https://github.com/golang/go/issues/11939). +> If `omitempty` is applied to a struct and all the children of the struct are *empty*, then on marshalling it will be **omitted** from the encoded json. + +## License +Refer to the [Golang](https://github.com/golang/go/blob/master/LICENSE) license. See [LICENSE](LICENSE) for more information. diff --git a/vendor/github.com/clarketm/json/decode.go b/vendor/github.com/clarketm/json/decode.go new file mode 100644 index 0000000..a9917e7 --- /dev/null +++ b/vendor/github.com/clarketm/json/decode.go @@ -0,0 +1,1310 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Represents JSON data structure using native Go types: booleans, floats, +// strings, arrays, and maps. + +package json + +import ( + "encoding" + "encoding/base64" + "fmt" + "reflect" + "strconv" + "strings" + "unicode" + "unicode/utf16" + "unicode/utf8" +) + +// Unmarshal parses the JSON-encoded data and stores the result +// in the value pointed to by v. If v is nil or not a pointer, +// Unmarshal returns an InvalidUnmarshalError. +// +// Unmarshal uses the inverse of the encodings that +// Marshal uses, allocating maps, slices, and pointers as necessary, +// with the following additional rules: +// +// To unmarshal JSON into a pointer, Unmarshal first handles the case of +// the JSON being the JSON literal null. In that case, Unmarshal sets +// the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into +// the value pointed at by the pointer. If the pointer is nil, Unmarshal +// allocates a new value for it to point to. +// +// To unmarshal JSON into a value implementing the Unmarshaler interface, +// Unmarshal calls that value's UnmarshalJSON method, including +// when the input is a JSON null. +// Otherwise, if the value implements encoding.TextUnmarshaler +// and the input is a JSON quoted string, Unmarshal calls that value's +// UnmarshalText method with the unquoted form of the string. +// +// To unmarshal JSON into a struct, Unmarshal matches incoming object +// keys to the keys used by Marshal (either the struct field name or its tag), +// preferring an exact match but also accepting a case-insensitive match. By +// default, object keys which don't have a corresponding struct field are +// ignored (see Decoder.DisallowUnknownFields for an alternative). +// +// To unmarshal JSON into an interface value, +// Unmarshal stores one of these in the interface value: +// +// bool, for JSON booleans +// float64, for JSON numbers +// string, for JSON strings +// []interface{}, for JSON arrays +// map[string]interface{}, for JSON objects +// nil for JSON null +// +// To unmarshal a JSON array into a slice, Unmarshal resets the slice length +// to zero and then appends each element to the slice. +// As a special case, to unmarshal an empty JSON array into a slice, +// Unmarshal replaces the slice with a new empty slice. +// +// To unmarshal a JSON array into a Go array, Unmarshal decodes +// JSON array elements into corresponding Go array elements. +// If the Go array is smaller than the JSON array, +// the additional JSON array elements are discarded. +// If the JSON array is smaller than the Go array, +// the additional Go array elements are set to zero values. +// +// To unmarshal a JSON object into a map, Unmarshal first establishes a map to +// use. If the map is nil, Unmarshal allocates a new map. Otherwise Unmarshal +// reuses the existing map, keeping existing entries. Unmarshal then stores +// key-value pairs from the JSON object into the map. The map's key type must +// either be any string type, an integer, implement json.Unmarshaler, or +// implement encoding.TextUnmarshaler. +// +// If a JSON value is not appropriate for a given target type, +// or if a JSON number overflows the target type, Unmarshal +// skips that field and completes the unmarshaling as best it can. +// If no more serious errors are encountered, Unmarshal returns +// an UnmarshalTypeError describing the earliest such error. In any +// case, it's not guaranteed that all the remaining fields following +// the problematic one will be unmarshaled into the target object. +// +// The JSON null value unmarshals into an interface, map, pointer, or slice +// by setting that Go value to nil. Because null is often used in JSON to mean +// ``not present,'' unmarshaling a JSON null into any other Go type has no effect +// on the value and produces no error. +// +// When unmarshaling quoted strings, invalid UTF-8 or +// invalid UTF-16 surrogate pairs are not treated as an error. +// Instead, they are replaced by the Unicode replacement +// character U+FFFD. +// +func Unmarshal(data []byte, v interface{}) error { + // Check for well-formedness. + // Avoids filling out half a data structure + // before discovering a JSON syntax error. + var d decodeState + err := checkValid(data, &d.scan) + if err != nil { + return err + } + + d.init(data) + return d.unmarshal(v) +} + +// Unmarshaler is the interface implemented by types +// that can unmarshal a JSON description of themselves. +// The input can be assumed to be a valid encoding of +// a JSON value. UnmarshalJSON must copy the JSON data +// if it wishes to retain the data after returning. +// +// By convention, to approximate the behavior of Unmarshal itself, +// Unmarshalers implement UnmarshalJSON([]byte("null")) as a no-op. +type Unmarshaler interface { + UnmarshalJSON([]byte) error +} + +// An UnmarshalTypeError describes a JSON value that was +// not appropriate for a value of a specific Go type. +type UnmarshalTypeError struct { + Value string // description of JSON value - "bool", "array", "number -5" + Type reflect.Type // type of Go value it could not be assigned to + Offset int64 // error occurred after reading Offset bytes + Struct string // name of the struct type containing the field + Field string // the full path from root node to the field +} + +func (e *UnmarshalTypeError) Error() string { + if e.Struct != "" || e.Field != "" { + return "json: cannot unmarshal " + e.Value + " into Go struct field " + e.Struct + "." + e.Field + " of type " + e.Type.String() + } + return "json: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() +} + +// An UnmarshalFieldError describes a JSON object key that +// led to an unexported (and therefore unwritable) struct field. +// +// Deprecated: No longer used; kept for compatibility. +type UnmarshalFieldError struct { + Key string + Type reflect.Type + Field reflect.StructField +} + +func (e *UnmarshalFieldError) Error() string { + return "json: cannot unmarshal object key " + strconv.Quote(e.Key) + " into unexported field " + e.Field.Name + " of type " + e.Type.String() +} + +// An InvalidUnmarshalError describes an invalid argument passed to Unmarshal. +// (The argument to Unmarshal must be a non-nil pointer.) +type InvalidUnmarshalError struct { + Type reflect.Type +} + +func (e *InvalidUnmarshalError) Error() string { + if e.Type == nil { + return "json: Unmarshal(nil)" + } + + if e.Type.Kind() != reflect.Ptr { + return "json: Unmarshal(non-pointer " + e.Type.String() + ")" + } + return "json: Unmarshal(nil " + e.Type.String() + ")" +} + +func (d *decodeState) unmarshal(v interface{}) error { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr || rv.IsNil() { + return &InvalidUnmarshalError{reflect.TypeOf(v)} + } + + d.scan.reset() + d.scanWhile(scanSkipSpace) + // We decode rv not rv.Elem because the Unmarshaler interface + // test must be applied at the top level of the value. + err := d.value(rv) + if err != nil { + return d.addErrorContext(err) + } + return d.savedError +} + +// A Number represents a JSON number literal. +type Number string + +// String returns the literal text of the number. +func (n Number) String() string { return string(n) } + +// Float64 returns the number as a float64. +func (n Number) Float64() (float64, error) { + return strconv.ParseFloat(string(n), 64) +} + +// Int64 returns the number as an int64. +func (n Number) Int64() (int64, error) { + return strconv.ParseInt(string(n), 10, 64) +} + +// An errorContext provides context for type errors during decoding. +type errorContext struct { + Struct reflect.Type + FieldStack []string +} + +// decodeState represents the state while decoding a JSON value. +type decodeState struct { + data []byte + off int // next read offset in data + opcode int // last read result + scan scanner + errorContext *errorContext + savedError error + useNumber bool + disallowUnknownFields bool +} + +// readIndex returns the position of the last byte read. +func (d *decodeState) readIndex() int { + return d.off - 1 +} + +// phasePanicMsg is used as a panic message when we end up with something that +// shouldn't happen. It can indicate a bug in the JSON decoder, or that +// something is editing the data slice while the decoder executes. +const phasePanicMsg = "JSON decoder out of sync - data changing underfoot?" + +func (d *decodeState) init(data []byte) *decodeState { + d.data = data + d.off = 0 + d.savedError = nil + if d.errorContext != nil { + d.errorContext.Struct = nil + // Reuse the allocated space for the FieldStack slice. + d.errorContext.FieldStack = d.errorContext.FieldStack[:0] + } + return d +} + +// saveError saves the first err it is called with, +// for reporting at the end of the unmarshal. +func (d *decodeState) saveError(err error) { + if d.savedError == nil { + d.savedError = d.addErrorContext(err) + } +} + +// addErrorContext returns a new error enhanced with information from d.errorContext +func (d *decodeState) addErrorContext(err error) error { + if d.errorContext != nil && (d.errorContext.Struct != nil || len(d.errorContext.FieldStack) > 0) { + switch err := err.(type) { + case *UnmarshalTypeError: + err.Struct = d.errorContext.Struct.Name() + err.Field = strings.Join(d.errorContext.FieldStack, ".") + } + } + return err +} + +// skip scans to the end of what was started. +func (d *decodeState) skip() { + s, data, i := &d.scan, d.data, d.off + depth := len(s.parseState) + for { + op := s.step(s, data[i]) + i++ + if len(s.parseState) < depth { + d.off = i + d.opcode = op + return + } + } +} + +// scanNext processes the byte at d.data[d.off]. +func (d *decodeState) scanNext() { + if d.off < len(d.data) { + d.opcode = d.scan.step(&d.scan, d.data[d.off]) + d.off++ + } else { + d.opcode = d.scan.eof() + d.off = len(d.data) + 1 // mark processed EOF with len+1 + } +} + +// scanWhile processes bytes in d.data[d.off:] until it +// receives a scan code not equal to op. +func (d *decodeState) scanWhile(op int) { + s, data, i := &d.scan, d.data, d.off + for i < len(data) { + newOp := s.step(s, data[i]) + i++ + if newOp != op { + d.opcode = newOp + d.off = i + return + } + } + + d.off = len(data) + 1 // mark processed EOF with len+1 + d.opcode = d.scan.eof() +} + +// rescanLiteral is similar to scanWhile(scanContinue), but it specialises the +// common case where we're decoding a literal. The decoder scans the input +// twice, once for syntax errors and to check the length of the value, and the +// second to perform the decoding. +// +// Only in the second step do we use decodeState to tokenize literals, so we +// know there aren't any syntax errors. We can take advantage of that knowledge, +// and scan a literal's bytes much more quickly. +func (d *decodeState) rescanLiteral() { + data, i := d.data, d.off +Switch: + switch data[i-1] { + case '"': // string + for ; i < len(data); i++ { + switch data[i] { + case '\\': + i++ // escaped char + case '"': + i++ // tokenize the closing quote too + break Switch + } + } + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': // number + for ; i < len(data); i++ { + switch data[i] { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '.', 'e', 'E', '+', '-': + default: + break Switch + } + } + case 't': // true + i += len("rue") + case 'f': // false + i += len("alse") + case 'n': // null + i += len("ull") + } + if i < len(data) { + d.opcode = stateEndValue(&d.scan, data[i]) + } else { + d.opcode = scanEnd + } + d.off = i + 1 +} + +// value consumes a JSON value from d.data[d.off-1:], decoding into v, and +// reads the following byte ahead. If v is invalid, the value is discarded. +// The first byte of the value has been read already. +func (d *decodeState) value(v reflect.Value) error { + switch d.opcode { + default: + panic(phasePanicMsg) + + case scanBeginArray: + if v.IsValid() { + if err := d.array(v); err != nil { + return err + } + } else { + d.skip() + } + d.scanNext() + + case scanBeginObject: + if v.IsValid() { + if err := d.object(v); err != nil { + return err + } + } else { + d.skip() + } + d.scanNext() + + case scanBeginLiteral: + // All bytes inside literal return scanContinue op code. + start := d.readIndex() + d.rescanLiteral() + + if v.IsValid() { + if err := d.literalStore(d.data[start:d.readIndex()], v, false); err != nil { + return err + } + } + } + return nil +} + +type unquotedValue struct{} + +// valueQuoted is like value but decodes a +// quoted string literal or literal null into an interface value. +// If it finds anything other than a quoted string literal or null, +// valueQuoted returns unquotedValue{}. +func (d *decodeState) valueQuoted() interface{} { + switch d.opcode { + default: + panic(phasePanicMsg) + + case scanBeginArray, scanBeginObject: + d.skip() + d.scanNext() + + case scanBeginLiteral: + v := d.literalInterface() + switch v.(type) { + case nil, string: + return v + } + } + return unquotedValue{} +} + +// indirect walks down v allocating pointers as needed, +// until it gets to a non-pointer. +// If it encounters an Unmarshaler, indirect stops and returns that. +// If decodingNull is true, indirect stops at the first settable pointer so it +// can be set to nil. +func indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { + // Issue #24153 indicates that it is generally not a guaranteed property + // that you may round-trip a reflect.Value by calling Value.Addr().Elem() + // and expect the value to still be settable for values derived from + // unexported embedded struct fields. + // + // The logic below effectively does this when it first addresses the value + // (to satisfy possible pointer methods) and continues to dereference + // subsequent pointers as necessary. + // + // After the first round-trip, we set v back to the original value to + // preserve the original RW flags contained in reflect.Value. + v0 := v + haveAddr := false + + // If v is a named type and is addressable, + // start with its address, so that if the type has pointer methods, + // we find them. + if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() { + haveAddr = true + v = v.Addr() + } + for { + // Load value from interface, but only if the result will be + // usefully addressable. + if v.Kind() == reflect.Interface && !v.IsNil() { + e := v.Elem() + if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) { + haveAddr = false + v = e + continue + } + } + + if v.Kind() != reflect.Ptr { + break + } + + if decodingNull && v.CanSet() { + break + } + + // Prevent infinite loop if v is an interface pointing to its own address: + // var v interface{} + // v = &v + if v.Elem().Kind() == reflect.Interface && v.Elem().Elem() == v { + v = v.Elem() + break + } + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + if v.Type().NumMethod() > 0 && v.CanInterface() { + if u, ok := v.Interface().(Unmarshaler); ok { + return u, nil, reflect.Value{} + } + if !decodingNull { + if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { + return nil, u, reflect.Value{} + } + } + } + + if haveAddr { + v = v0 // restore original value after round-trip Value.Addr().Elem() + haveAddr = false + } else { + v = v.Elem() + } + } + return nil, nil, v +} + +// array consumes an array from d.data[d.off-1:], decoding into v. +// The first byte of the array ('[') has been read already. +func (d *decodeState) array(v reflect.Value) error { + // Check for unmarshaler. + u, ut, pv := indirect(v, false) + if u != nil { + start := d.readIndex() + d.skip() + return u.UnmarshalJSON(d.data[start:d.off]) + } + if ut != nil { + d.saveError(&UnmarshalTypeError{Value: "array", Type: v.Type(), Offset: int64(d.off)}) + d.skip() + return nil + } + v = pv + + // Check type of target. + switch v.Kind() { + case reflect.Interface: + if v.NumMethod() == 0 { + // Decoding into nil interface? Switch to non-reflect code. + ai := d.arrayInterface() + v.Set(reflect.ValueOf(ai)) + return nil + } + // Otherwise it's invalid. + fallthrough + default: + d.saveError(&UnmarshalTypeError{Value: "array", Type: v.Type(), Offset: int64(d.off)}) + d.skip() + return nil + case reflect.Array, reflect.Slice: + break + } + + i := 0 + for { + // Look ahead for ] - can only happen on first iteration. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndArray { + break + } + + // Get element of array, growing if necessary. + if v.Kind() == reflect.Slice { + // Grow slice if necessary + if i >= v.Cap() { + newcap := v.Cap() + v.Cap()/2 + if newcap < 4 { + newcap = 4 + } + newv := reflect.MakeSlice(v.Type(), v.Len(), newcap) + reflect.Copy(newv, v) + v.Set(newv) + } + if i >= v.Len() { + v.SetLen(i + 1) + } + } + + if i < v.Len() { + // Decode into element. + if err := d.value(v.Index(i)); err != nil { + return err + } + } else { + // Ran out of fixed array: skip. + if err := d.value(reflect.Value{}); err != nil { + return err + } + } + i++ + + // Next token must be , or ]. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndArray { + break + } + if d.opcode != scanArrayValue { + panic(phasePanicMsg) + } + } + + if i < v.Len() { + if v.Kind() == reflect.Array { + // Array. Zero the rest. + z := reflect.Zero(v.Type().Elem()) + for ; i < v.Len(); i++ { + v.Index(i).Set(z) + } + } else { + v.SetLen(i) + } + } + if i == 0 && v.Kind() == reflect.Slice { + v.Set(reflect.MakeSlice(v.Type(), 0, 0)) + } + return nil +} + +var nullLiteral = []byte("null") +var textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() + +// object consumes an object from d.data[d.off-1:], decoding into v. +// The first byte ('{') of the object has been read already. +func (d *decodeState) object(v reflect.Value) error { + // Check for unmarshaler. + u, ut, pv := indirect(v, false) + if u != nil { + start := d.readIndex() + d.skip() + return u.UnmarshalJSON(d.data[start:d.off]) + } + if ut != nil { + d.saveError(&UnmarshalTypeError{Value: "object", Type: v.Type(), Offset: int64(d.off)}) + d.skip() + return nil + } + v = pv + t := v.Type() + + // Decoding into nil interface? Switch to non-reflect code. + if v.Kind() == reflect.Interface && v.NumMethod() == 0 { + oi := d.objectInterface() + v.Set(reflect.ValueOf(oi)) + return nil + } + + var fields structFields + + // Check type of target: + // struct or + // map[T1]T2 where T1 is string, an integer type, + // or an encoding.TextUnmarshaler + switch v.Kind() { + case reflect.Map: + // Map key must either have string kind, have an integer kind, + // or be an encoding.TextUnmarshaler. + switch t.Key().Kind() { + case reflect.String, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + default: + if !reflect.PtrTo(t.Key()).Implements(textUnmarshalerType) { + d.saveError(&UnmarshalTypeError{Value: "object", Type: t, Offset: int64(d.off)}) + d.skip() + return nil + } + } + if v.IsNil() { + v.Set(reflect.MakeMap(t)) + } + case reflect.Struct: + fields = cachedTypeFields(t) + // ok + default: + d.saveError(&UnmarshalTypeError{Value: "object", Type: t, Offset: int64(d.off)}) + d.skip() + return nil + } + + var mapElem reflect.Value + var origErrorContext errorContext + if d.errorContext != nil { + origErrorContext = *d.errorContext + } + + for { + // Read opening " of string key or closing }. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if d.opcode != scanBeginLiteral { + panic(phasePanicMsg) + } + + // Read key. + start := d.readIndex() + d.rescanLiteral() + item := d.data[start:d.readIndex()] + key, ok := unquoteBytes(item) + if !ok { + panic(phasePanicMsg) + } + + // Figure out field corresponding to key. + var subv reflect.Value + destring := false // whether the value is wrapped in a string to be decoded first + + if v.Kind() == reflect.Map { + elemType := t.Elem() + if !mapElem.IsValid() { + mapElem = reflect.New(elemType).Elem() + } else { + mapElem.Set(reflect.Zero(elemType)) + } + subv = mapElem + } else { + var f *field + if i, ok := fields.nameIndex[string(key)]; ok { + // Found an exact name match. + f = &fields.list[i] + } else { + // Fall back to the expensive case-insensitive + // linear search. + for i := range fields.list { + ff := &fields.list[i] + if ff.equalFold(ff.nameBytes, key) { + f = ff + break + } + } + } + if f != nil { + subv = v + destring = f.quoted + for _, i := range f.index { + if subv.Kind() == reflect.Ptr { + if subv.IsNil() { + // If a struct embeds a pointer to an unexported type, + // it is not possible to set a newly allocated value + // since the field is unexported. + // + // See https://golang.org/issue/21357 + if !subv.CanSet() { + d.saveError(fmt.Errorf("json: cannot set embedded pointer to unexported struct: %v", subv.Type().Elem())) + // Invalidate subv to ensure d.value(subv) skips over + // the JSON value without assigning it to subv. + subv = reflect.Value{} + destring = false + break + } + subv.Set(reflect.New(subv.Type().Elem())) + } + subv = subv.Elem() + } + subv = subv.Field(i) + } + if d.errorContext == nil { + d.errorContext = new(errorContext) + } + d.errorContext.FieldStack = append(d.errorContext.FieldStack, f.name) + d.errorContext.Struct = t + } else if d.disallowUnknownFields { + d.saveError(fmt.Errorf("json: unknown field %q", key)) + } + } + + // Read : before value. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode != scanObjectKey { + panic(phasePanicMsg) + } + d.scanWhile(scanSkipSpace) + + if destring { + switch qv := d.valueQuoted().(type) { + case nil: + if err := d.literalStore(nullLiteral, subv, false); err != nil { + return err + } + case string: + if err := d.literalStore([]byte(qv), subv, true); err != nil { + return err + } + default: + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", subv.Type())) + } + } else { + if err := d.value(subv); err != nil { + return err + } + } + + // Write value back to map; + // if using struct, subv points into struct already. + if v.Kind() == reflect.Map { + kt := t.Key() + var kv reflect.Value + switch { + case reflect.PtrTo(kt).Implements(textUnmarshalerType): + kv = reflect.New(kt) + if err := d.literalStore(item, kv, true); err != nil { + return err + } + kv = kv.Elem() + case kt.Kind() == reflect.String: + kv = reflect.ValueOf(key).Convert(kt) + default: + switch kt.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + s := string(key) + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || reflect.Zero(kt).OverflowInt(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) + break + } + kv = reflect.ValueOf(n).Convert(kt) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + s := string(key) + n, err := strconv.ParseUint(s, 10, 64) + if err != nil || reflect.Zero(kt).OverflowUint(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) + break + } + kv = reflect.ValueOf(n).Convert(kt) + default: + panic("json: Unexpected key type") // should never occur + } + } + if kv.IsValid() { + v.SetMapIndex(kv, subv) + } + } + + // Next token must be , or }. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.errorContext != nil { + // Reset errorContext to its original state. + // Keep the same underlying array for FieldStack, to reuse the + // space and avoid unnecessary allocs. + d.errorContext.FieldStack = d.errorContext.FieldStack[:len(origErrorContext.FieldStack)] + d.errorContext.Struct = origErrorContext.Struct + } + if d.opcode == scanEndObject { + break + } + if d.opcode != scanObjectValue { + panic(phasePanicMsg) + } + } + return nil +} + +// convertNumber converts the number literal s to a float64 or a Number +// depending on the setting of d.useNumber. +func (d *decodeState) convertNumber(s string) (interface{}, error) { + if d.useNumber { + return Number(s), nil + } + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil, &UnmarshalTypeError{Value: "number " + s, Type: reflect.TypeOf(0.0), Offset: int64(d.off)} + } + return f, nil +} + +var numberType = reflect.TypeOf(Number("")) + +// literalStore decodes a literal stored in item into v. +// +// fromQuoted indicates whether this literal came from unwrapping a +// string from the ",string" struct tag option. this is used only to +// produce more helpful error messages. +func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool) error { + // Check for unmarshaler. + if len(item) == 0 { + //Empty string given + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + return nil + } + isNull := item[0] == 'n' // null + u, ut, pv := indirect(v, isNull) + if u != nil { + return u.UnmarshalJSON(item) + } + if ut != nil { + if item[0] != '"' { + if fromQuoted { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + return nil + } + val := "number" + switch item[0] { + case 'n': + val = "null" + case 't', 'f': + val = "bool" + } + d.saveError(&UnmarshalTypeError{Value: val, Type: v.Type(), Offset: int64(d.readIndex())}) + return nil + } + s, ok := unquoteBytes(item) + if !ok { + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + panic(phasePanicMsg) + } + return ut.UnmarshalText(s) + } + + v = pv + + switch c := item[0]; c { + case 'n': // null + // The main parser checks that only true and false can reach here, + // but if this was a quoted string input, it could be anything. + if fromQuoted && string(item) != "null" { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + break + } + switch v.Kind() { + case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: + v.Set(reflect.Zero(v.Type())) + // otherwise, ignore null for primitives/string + } + case 't', 'f': // true, false + value := item[0] == 't' + // The main parser checks that only true and false can reach here, + // but if this was a quoted string input, it could be anything. + if fromQuoted && string(item) != "true" && string(item) != "false" { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + break + } + switch v.Kind() { + default: + if fromQuoted { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.readIndex())}) + } + case reflect.Bool: + v.SetBool(value) + case reflect.Interface: + if v.NumMethod() == 0 { + v.Set(reflect.ValueOf(value)) + } else { + d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.readIndex())}) + } + } + + case '"': // string + s, ok := unquoteBytes(item) + if !ok { + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + panic(phasePanicMsg) + } + switch v.Kind() { + default: + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.readIndex())}) + case reflect.Slice: + if v.Type().Elem().Kind() != reflect.Uint8 { + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) + n, err := base64.StdEncoding.Decode(b, s) + if err != nil { + d.saveError(err) + break + } + v.SetBytes(b[:n]) + case reflect.String: + if v.Type() == numberType && !isValidNumber(string(s)) { + return fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", item) + } + v.SetString(string(s)) + case reflect.Interface: + if v.NumMethod() == 0 { + v.Set(reflect.ValueOf(string(s))) + } else { + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.readIndex())}) + } + } + + default: // number + if c != '-' && (c < '0' || c > '9') { + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + panic(phasePanicMsg) + } + s := string(item) + switch v.Kind() { + default: + if v.Kind() == reflect.String && v.Type() == numberType { + // s must be a valid number, because it's + // already been tokenized. + v.SetString(s) + break + } + if fromQuoted { + return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) + } + d.saveError(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.readIndex())}) + case reflect.Interface: + n, err := d.convertNumber(s) + if err != nil { + d.saveError(err) + break + } + if v.NumMethod() != 0 { + d.saveError(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.Set(reflect.ValueOf(n)) + + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || v.OverflowInt(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.SetInt(n) + + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + n, err := strconv.ParseUint(s, 10, 64) + if err != nil || v.OverflowUint(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.SetUint(n) + + case reflect.Float32, reflect.Float64: + n, err := strconv.ParseFloat(s, v.Type().Bits()) + if err != nil || v.OverflowFloat(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.readIndex())}) + break + } + v.SetFloat(n) + } + } + return nil +} + +// The xxxInterface routines build up a value to be stored +// in an empty interface. They are not strictly necessary, +// but they avoid the weight of reflection in this common case. + +// valueInterface is like value but returns interface{} +func (d *decodeState) valueInterface() (val interface{}) { + switch d.opcode { + default: + panic(phasePanicMsg) + case scanBeginArray: + val = d.arrayInterface() + d.scanNext() + case scanBeginObject: + val = d.objectInterface() + d.scanNext() + case scanBeginLiteral: + val = d.literalInterface() + } + return +} + +// arrayInterface is like array but returns []interface{}. +func (d *decodeState) arrayInterface() []interface{} { + var v = make([]interface{}, 0) + for { + // Look ahead for ] - can only happen on first iteration. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndArray { + break + } + + v = append(v, d.valueInterface()) + + // Next token must be , or ]. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndArray { + break + } + if d.opcode != scanArrayValue { + panic(phasePanicMsg) + } + } + return v +} + +// objectInterface is like object but returns map[string]interface{}. +func (d *decodeState) objectInterface() map[string]interface{} { + m := make(map[string]interface{}) + for { + // Read opening " of string key or closing }. + d.scanWhile(scanSkipSpace) + if d.opcode == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if d.opcode != scanBeginLiteral { + panic(phasePanicMsg) + } + + // Read string key. + start := d.readIndex() + d.rescanLiteral() + item := d.data[start:d.readIndex()] + key, ok := unquote(item) + if !ok { + panic(phasePanicMsg) + } + + // Read : before value. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode != scanObjectKey { + panic(phasePanicMsg) + } + d.scanWhile(scanSkipSpace) + + // Read value. + m[key] = d.valueInterface() + + // Next token must be , or }. + if d.opcode == scanSkipSpace { + d.scanWhile(scanSkipSpace) + } + if d.opcode == scanEndObject { + break + } + if d.opcode != scanObjectValue { + panic(phasePanicMsg) + } + } + return m +} + +// literalInterface consumes and returns a literal from d.data[d.off-1:] and +// it reads the following byte ahead. The first byte of the literal has been +// read already (that's how the caller knows it's a literal). +func (d *decodeState) literalInterface() interface{} { + // All bytes inside literal return scanContinue op code. + start := d.readIndex() + d.rescanLiteral() + + item := d.data[start:d.readIndex()] + + switch c := item[0]; c { + case 'n': // null + return nil + + case 't', 'f': // true, false + return c == 't' + + case '"': // string + s, ok := unquote(item) + if !ok { + panic(phasePanicMsg) + } + return s + + default: // number + if c != '-' && (c < '0' || c > '9') { + panic(phasePanicMsg) + } + n, err := d.convertNumber(string(item)) + if err != nil { + d.saveError(err) + } + return n + } +} + +// getu4 decodes \uXXXX from the beginning of s, returning the hex value, +// or it returns -1. +func getu4(s []byte) rune { + if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { + return -1 + } + var r rune + for _, c := range s[2:6] { + switch { + case '0' <= c && c <= '9': + c = c - '0' + case 'a' <= c && c <= 'f': + c = c - 'a' + 10 + case 'A' <= c && c <= 'F': + c = c - 'A' + 10 + default: + return -1 + } + r = r*16 + rune(c) + } + return r +} + +// unquote converts a quoted JSON string literal s into an actual string t. +// The rules are different than for Go, so cannot use strconv.Unquote. +func unquote(s []byte) (t string, ok bool) { + s, ok = unquoteBytes(s) + t = string(s) + return +} + +func unquoteBytes(s []byte) (t []byte, ok bool) { + if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { + return + } + s = s[1 : len(s)-1] + + // Check for unusual characters. If there are none, + // then no unquoting is needed, so return a slice of the + // original bytes. + r := 0 + for r < len(s) { + c := s[r] + if c == '\\' || c == '"' || c < ' ' { + break + } + if c < utf8.RuneSelf { + r++ + continue + } + rr, size := utf8.DecodeRune(s[r:]) + if rr == utf8.RuneError && size == 1 { + break + } + r += size + } + if r == len(s) { + return s, true + } + + b := make([]byte, len(s)+2*utf8.UTFMax) + w := copy(b, s[0:r]) + for r < len(s) { + // Out of room? Can only happen if s is full of + // malformed UTF-8 and we're replacing each + // byte with RuneError. + if w >= len(b)-2*utf8.UTFMax { + nb := make([]byte, (len(b)+utf8.UTFMax)*2) + copy(nb, b[0:w]) + b = nb + } + switch c := s[r]; { + case c == '\\': + r++ + if r >= len(s) { + return + } + switch s[r] { + default: + return + case '"', '\\', '/', '\'': + b[w] = s[r] + r++ + w++ + case 'b': + b[w] = '\b' + r++ + w++ + case 'f': + b[w] = '\f' + r++ + w++ + case 'n': + b[w] = '\n' + r++ + w++ + case 'r': + b[w] = '\r' + r++ + w++ + case 't': + b[w] = '\t' + r++ + w++ + case 'u': + r-- + rr := getu4(s[r:]) + if rr < 0 { + return + } + r += 6 + if utf16.IsSurrogate(rr) { + rr1 := getu4(s[r:]) + if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar { + // A valid pair; consume. + r += 6 + w += utf8.EncodeRune(b[w:], dec) + break + } + // Invalid surrogate; fall back to replacement rune. + rr = unicode.ReplacementChar + } + w += utf8.EncodeRune(b[w:], rr) + } + + // Quote, control characters are invalid. + case c == '"', c < ' ': + return + + // ASCII + case c < utf8.RuneSelf: + b[w] = c + r++ + w++ + + // Coerce to well-formed UTF-8. + default: + rr, size := utf8.DecodeRune(s[r:]) + r += size + w += utf8.EncodeRune(b[w:], rr) + } + } + return b[0:w], true +} diff --git a/vendor/github.com/clarketm/json/encode.go b/vendor/github.com/clarketm/json/encode.go new file mode 100644 index 0000000..06b2f75 --- /dev/null +++ b/vendor/github.com/clarketm/json/encode.go @@ -0,0 +1,1430 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package json implements encoding and decoding of JSON as defined in +// RFC 7159. The mapping between JSON and Go values is described +// in the documentation for the Marshal and Unmarshal functions. +// +// See "JSON and Go" for an introduction to this package: +// https://golang.org/doc/articles/json_and_go.html +package json + +import ( + "bytes" + "encoding" + "encoding/base64" + "fmt" + "math" + "reflect" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +// Marshal returns the JSON encoding of v. +// +// Marshal traverses the value v recursively. +// If an encountered value implements the Marshaler interface +// and is not a nil pointer, Marshal calls its MarshalJSON method +// to produce JSON. If no MarshalJSON method is present but the +// value implements encoding.TextMarshaler instead, Marshal calls +// its MarshalText method and encodes the result as a JSON string. +// The nil pointer exception is not strictly necessary +// but mimics a similar, necessary exception in the behavior of +// UnmarshalJSON. +// +// Otherwise, Marshal uses the following type-dependent default encodings: +// +// Boolean values encode as JSON booleans. +// +// Floating point, integer, and Number values encode as JSON numbers. +// +// String values encode as JSON strings coerced to valid UTF-8, +// replacing invalid bytes with the Unicode replacement rune. +// So that the JSON will be safe to embed inside HTML