feat(engine)!: let compilers own format detection and options

README.md

@@ -122,6 +122,7 @@ Help always prints to stdout and exits `0`.
 | `-o <file>` | `compile` | Write IR JSON to `<file>` instead of stdout, compact rather than indented. |
 | `--pretty` | `compile` | Indent the JSON `-o` writes; stdout is indented either way. |
 | `--explain <json-pointer>` | `compile` | Report what compiling produced at this source coordinate instead of writing the document. |
+| `--opt <key>=<value>` | both | Set one option on the compiler the spec selects. Repeatable; a repeated key is refused. |
 
 Diagnostics print one per line as `<severity> <code> <location>: <message>`, where `<location>` is
 `<path>#<pointer>` for a finding in a spec file, a bare pointer for one an IR pass made about the
@@ -133,10 +134,29 @@ covers an undecodable file, an unrecognized or unsupported format, and a version
 `2` the invocation or the filesystem was wrong — a bad flag or argument, a spec that could not be
 read, an output that could not be written. Nothing about the spec's own contents reaches `2`.
 
+#### Compiler options
+
+`--opt` names an option in the vocabulary of whichever compiler recognizes the spec — morphic
+itself knows none of them, and an unknown name is refused by the compiler rather than ignored.
+The OpenAPI compiler accepts:
+
+| Option | Values | Meaning |
+|---|---|---|
+| `grouping` | `tags` (default), `path-prefix` | How operations are grouped into operation groups. |
+| `allow-external-refs` | `true`, `false` (default) | Let `$ref` resolution leave the source document, reading files and fetching URLs. |
+| `overlay` | a file path | Apply an [OpenAPI Overlay](https://spec.openapis.org/overlay/latest.html) document to the source before lowering. |
+| `overlay-lax` | `true`, `false` (default) | Do not refuse when an overlay action's selector matches nothing. |
+
+```bash
+morphic compile openapi.yaml --opt grouping=path-prefix --opt overlay=patch.yaml
+```
+
 ### Library
 
 The same pipeline is available as a package. `engine.New` builds the default registry (OpenAPI
-compiler + `validate` pass); `Run` sniffs the format, compiles, and runs passes.
+compiler + `validate` pass); `Run` asks the registered compilers which of them recognizes the
+source, compiles, and runs passes. A Go caller can set compiler options as a typed value through
+`RunOptions.FormatOptions` instead of as text through `RunOptions.CompilerOptions`.
 
 ```go
 eng, err := engine.New()

cmd/morphic/command_test.go

@@ -65,6 +65,9 @@ func TestNewCompileFlags_DefinesEveryFlag(t *testing.T) {
 	assert.Empty(t, opts.outPath)
 	assert.False(t, opts.skipValidate)
 	assert.Empty(t, opts.explain)
+	assert.NotNil(t, opts.settings,
+		"the settings map must exist before Parse writes into it")
+	assert.Empty(t, opts.settings)
 }
 
 func TestNewValidateFlags_DefinesEveryFlag(t *testing.T) {
@@ -105,8 +108,8 @@ func TestSpecFlags_SharedFlagsAgree(t *testing.T) {
 // asserted from both the constructor and the command-table entry so the two
 // cannot drift.
 var (
-	compileFlagNames  = []string{"o", "fail-on", "skip-validate", "explain", "pretty"}
-	validateFlagNames = []string{"fail-on", "skip-validate"}
+	compileFlagNames  = []string{"o", "fail-on", "skip-validate", "explain", "pretty", "opt"}
+	validateFlagNames = []string{"fail-on", "skip-validate", "opt"}
 )
 
 func TestCommand_PrintFlagsDocumentsTheCommandsOwnFlags(t *testing.T) {

cmd/morphic/compile.go

@@ -72,6 +72,9 @@ func newCompileCommand() command {
 			"--explain reports what compiling produced at one source coordinate — the\n" +
 			"type node interned there, the coordinates interned beneath it, and the\n" +
 			"diagnostics stamped at it — instead of writing the document.\n\n" +
+			"--opt passes a setting to the compiler the spec selects, which names and\n" +
+			"validates its own options; morphic itself knows none of them. The OpenAPI\n" +
+			"compiler's are listed in the README.\n\n" +
 			"A -- argument ends flag parsing: every argument after it is an operand,\n" +
 			"even one that begins with a dash, which is how a spec file named like a\n" +
 			"flag is passed.",
@@ -84,11 +87,14 @@ func newCompileCommand() command {
 	}
 }
 
-// specOptions holds the values the flags shared by every spec-taking command
-// parse into.
+// specOptions holds what the shared pipeline runner reads, parsed by the flags
+// every spec-taking command defines. validate is compile's pipeline with the
+// document dropped, so the two configure that pipeline the same way: a spec that
+// needs an option to compile needs it to be validated as it will be compiled.
 type specOptions struct {
 	failOn       string
 	skipValidate bool
+	settings     settingFlag
 }
 
 // compileOptions holds the values compile's flags parse into.
@@ -103,10 +109,14 @@ type compileOptions struct {
 // Sharing the registration is what keeps their spellings, defaults and help
 // text identical across commands rather than identical-looking.
 func bindSpecFlags(fs *flag.FlagSet, opts *specOptions) {
+	opts.settings = settingFlag{}
+
 	fs.StringVar(&opts.failOn, "fail-on", "error",
 		"fail (exit 1) on diagnostics at or above this severity: error|warning")
 	fs.BoolVar(&opts.skipValidate, "skip-validate", false,
 		"skip the referential-integrity validate pass")
+	fs.Var(opts.settings, "opt",
+		"set one `key=value` option on the compiler the spec selects (repeatable)")
 }
 
 // newCompileFlags returns compile's FlagSet and the options its flags write
@@ -180,7 +190,10 @@ func runPipeline(specPath string, opts specOptions, stderr io.Writer) (*engine.R
 		return nil, 2, false
 	}
 
-	res, err := eng.Run(context.Background(), specPath, engine.RunOptions{SkipValidate: opts.skipValidate})
+	res, err := eng.Run(context.Background(), specPath, engine.RunOptions{
+		CompilerOptions: opts.settings,
+		SkipValidate:    opts.skipValidate,
+	})
 	if err != nil {
 		emitf(stderr, "morphic: %v\n", err)
 		return nil, 2, false

cmd/morphic/compile_test.go

@@ -56,8 +56,8 @@ func TestRun_SpecProblemsExitOne(t *testing.T) {
 		name, contents, code string
 	}{
 		{"unrecognized format", "hello: world\n", "engine/unrecognized-format"},
-		{"swagger 2.0", "swagger: \"2.0\"\n", "engine/unsupported-format"},
-		{"undecodable source", "openapi: [unterminated\n", "engine/undecodable-source"},
+		{"swagger 2.0", "swagger: \"2.0\"\n", "engine/no-compiler-for-format"},
+		{"undecodable source", "openapi: [unterminated\n", "openapi/undecodable-source"},
 		{"no compiler for version", "openapi: 4.0.0\ninfo: {title: T, version: \"1\"}\npaths: {}\n",
 			"engine/no-compiler-for-format"},
 	}
@@ -140,6 +140,95 @@ func TestRun_UsageErrors(t *testing.T) {
 	}
 }
 
+// groupingSpec tags its one operation "zoo" while its path starts with "a", so
+// the two grouping strategies name the group differently and the assertion below
+// cannot pass by accident.
+const groupingSpec = `openapi: 3.1.0
+info: {title: T, version: "1"}
+paths:
+  /a/b:
+    get:
+      operationId: ab
+      tags: [zoo]
+      responses: {"200": {description: ok}}
+`
+
+// TestRun_CompilerOptionReachesTheCompiler asserts from the CLI what the Go API
+// could already do: a compiler option the user set changes the document. The
+// control run pins that the difference is the flag and not the spec.
+func TestRun_CompilerOptionReachesTheCompiler(t *testing.T) {
+	t.Parallel()
+	spec := writeFile(t, "spec.yaml", groupingSpec)
+
+	assert.Equal(t, "zoo", groupNameOf(t, spec), "the default groups by tag")
+	assert.Equal(t, "a", groupNameOf(t, spec, "-opt", "grouping=path-prefix"),
+		"the compiler option must reach the compiler")
+}
+
+// groupNameOf compiles spec with args and returns its first operation group's
+// source name.
+func groupNameOf(t *testing.T, spec string, args ...string) string {
+	t.Helper()
+	var stdout, stderr bytes.Buffer
+	code := run(append([]string{"compile", spec}, args...), &stdout, &stderr)
+	require.Equal(t, 0, code, "stderr: %s", stderr.String())
+
+	var doc ir.Document
+	require.NoError(t, json.Unmarshal(stdout.Bytes(), &doc))
+	require.Len(t, doc.Services, 1)
+	require.NotEmpty(t, doc.Services[0].Groups)
+	return doc.Services[0].Groups[0].Name.Source
+}
+
+// TestRun_OverlayOptionReachesTheCompiler covers the option whose value is a
+// file. A compiler reads no files of its own, so the whole chain has to work for
+// this to compile anything: the CLI collects the path, the engine loads it, and
+// the compiler decodes the bytes into its own overlay type.
+func TestRun_OverlayOptionReachesTheCompiler(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+	spec := filepath.Join(dir, "spec.yaml")
+	require.NoError(t, os.WriteFile(spec, []byte(testspec.Tiny), 0o644))
+	overlay := filepath.Join(dir, "patch.yaml")
+	require.NoError(t, os.WriteFile(overlay, []byte(`overlay: 1.0.0
+info: {title: Patch, version: "1"}
+actions:
+  - target: $.info
+    update: {title: Patched}
+`), 0o644))
+
+	var stdout, stderr bytes.Buffer
+	code := run([]string{"compile", spec, "--opt", "overlay=" + overlay}, &stdout, &stderr)
+	require.Equal(t, 0, code, "stderr: %s", stderr.String())
+
+	var doc ir.Document
+	require.NoError(t, json.Unmarshal(stdout.Bytes(), &doc))
+	assert.Equal(t, "Patched", doc.Name, "the overlay must have been applied")
+	assert.Len(t, doc.Sources, 2, "an applied overlay is a second source")
+}
+
+func TestRun_BadCompilerOptionIsRefused(t *testing.T) {
+	t.Parallel()
+	spec := writeFile(t, "spec.yaml", testspec.Tiny)
+	tests := []struct {
+		name, arg, wantErr string
+	}{
+		{"malformed pair", "grouping", "want key=value"},
+		{"unknown name", "gruoping=tags", `unknown option "gruoping"`},
+		{"unusable value", "grouping=alphabetical", `want "tags" or "path-prefix"`},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			var stdout, stderr bytes.Buffer
+			code := run([]string{"compile", spec, "--opt", tc.arg}, &stdout, &stderr)
+			assert.Equal(t, 2, code)
+			assert.Contains(t, stderr.String(), tc.wantErr)
+			assert.Empty(t, stdout.String(), "a refused option must write no document")
+		})
+	}
+}
+
 // unsortedSpec declares its schemas out of alphabetical order and its paths in
 // an order of their own, so a document compiled from it exercises both halves
 // of the determinism invariant: maps emitted in sorted-key order and slices in

cmd/morphic/edgecases_test.go

@@ -89,6 +89,12 @@ func (nilDocCompiler) Formats() []compilers.SourceFormat {
 	return []compilers.SourceFormat{{Name: "openapi", Version: "3.1"}}
 }
 
+func (nilDocCompiler) Detect(compilers.Source) (compilers.SourceFormat, []ir.Diagnostic, bool) {
+	return compilers.SourceFormat{Name: "openapi", Version: "3.1"}, nil, true
+}
+
+func (nilDocCompiler) DecodeOptions(compilers.OptionSet) (any, error) { return nil, nil }
+
 func (nilDocCompiler) Compile(context.Context, []compilers.Source, compilers.Options) (*ir.Document, []ir.Diagnostic, error) {
 	return nil, []ir.Diagnostic{{
 		Severity: ir.SeverityError,

cmd/morphic/options.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"fmt"
+	"maps"
+	"slices"
+	"strings"
+)
+
+// settingFlag collects the repeated `-opt key=value` settings that configure
+// whichever compiler the spec turns out to need.
+//
+// It carries the pairs verbatim rather than parsing them, because the CLI does
+// not know which compiler will read them: the names, the accepted values and
+// what counts as a file path are the compiler's own, and a CLI that validated
+// any of them would be holding a second copy of a vocabulary it cannot see
+// change. What is checked here is the shape every compiler's settings share —
+// that a pair has both halves, and that no key was given twice.
+type settingFlag map[string]string
+
+// String renders the settings as they were typed, sorted so the rendering does
+// not depend on map order. The flag package calls it on a zero value to decide
+// whether to print a default, so a nil map must render as the empty string.
+func (s settingFlag) String() string {
+	pairs := make([]string, 0, len(s))
+	for _, key := range slices.Sorted(maps.Keys(s)) {
+		pairs = append(pairs, key+"="+s[key])
+	}
+	return strings.Join(pairs, " ")
+}
+
+// Set records one key=value pair. A repeated key is refused rather than
+// overwritten: one of the two values would take effect with nothing to say
+// which, and a user who typed both meant something by each.
+func (s settingFlag) Set(raw string) error {
+	if s == nil {
+		return fmt.Errorf("no option set to write %q into", raw)
+	}
+	key, value, ok := strings.Cut(raw, "=")
+	if !ok {
+		return fmt.Errorf("want key=value, got %q", raw)
+	}
+	if key == "" {
+		return fmt.Errorf("empty option name in %q", raw)
+	}
+	if _, repeated := s[key]; repeated {
+		return fmt.Errorf("option %q set more than once", key)
+	}
+	s[key] = value
+	return nil
+}

cmd/morphic/options_test.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSettingFlag_SetCollectsPairs(t *testing.T) {
+	t.Parallel()
+	got := settingFlag{}
+	require.NoError(t, got.Set("grouping=path-prefix"))
+	require.NoError(t, got.Set("overlay=patch.yaml"))
+	// A value may carry the separator; only the first one splits the pair.
+	require.NoError(t, got.Set("note=a=b"))
+	// An empty value is a value, not an absent setting: the compiler decides
+	// whether it can use one.
+	require.NoError(t, got.Set("quiet="))
+
+	assert.Equal(t, settingFlag{
+		"grouping": "path-prefix",
+		"overlay":  "patch.yaml",
+		"note":     "a=b",
+		"quiet":    "",
+	}, got)
+	assert.Equal(t, "grouping=path-prefix note=a=b overlay=patch.yaml quiet=", got.String())
+}
+
+func TestSettingFlag_StringOfNilRendersEmpty(t *testing.T) {
+	t.Parallel()
+	// The flag package renders a zero value to decide whether to print a
+	// default, so this is reached before any -opt is typed.
+	var unset settingFlag
+	assert.Empty(t, unset.String())
+}
+
+func TestSettingFlag_SetRefusals(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name, raw, wantErr string
+	}{
+		{"no separator", "grouping", `want key=value, got "grouping"`},
+		{"empty name", "=tags", `empty option name in "=tags"`},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			err := settingFlag{}.Set(tc.raw)
+			require.Error(t, err)
+			assert.EqualError(t, err, tc.wantErr)
+		})
+	}
+
+	repeated := settingFlag{}
+	require.NoError(t, repeated.Set("grouping=tags"))
+	err := repeated.Set("grouping=path-prefix")
+	require.Error(t, err, "one of the two values would win with nothing to say which")
+	assert.EqualError(t, err, `option "grouping" set more than once`)
+	assert.Equal(t, "tags", repeated["grouping"], "the refused value must not land")
+
+	var unset settingFlag
+	assert.Error(t, unset.Set("grouping=tags"), "a nil set must refuse rather than panic")
+}

cmd/morphic/testdata/compile-help.txt

@@ -9,6 +9,10 @@ written with -o is compact unless --pretty asks for the indented form.
 type node interned there, the coordinates interned beneath it, and the
 diagnostics stamped at it — instead of writing the document.
 
+--opt passes a setting to the compiler the spec selects, which names and
+validates its own options; morphic itself knows none of them. The OpenAPI
+compiler's are listed in the README.
+
 A -- argument ends flag parsing: every argument after it is an operand,
 even one that begins with a dash, which is how a spec file named like a
 flag is passed.
@@ -20,6 +24,8 @@ flags:
     	fail (exit 1) on diagnostics at or above this severity: error|warning (default "error")
   -o string
     	write IR JSON to this file instead of stdout
+  -opt key=value
+    	set one key=value option on the compiler the spec selects (repeatable)
   -pretty
     	indent the IR JSON -o writes; stdout is indented either way
   -skip-validate

cmd/morphic/testdata/validate-help.txt

@@ -13,8 +13,13 @@ The diagnostics and the exit code are compile's: 0 when nothing reaches
 --skip-validate names the referential-integrity pass, not this command: it
 drops that pass's diagnostics and keeps the compiler's.
 
+--opt is compile's, and takes the same settings: a spec that needs one to
+compile needs it here too, or what is checked is not what would be built.
+
 flags:
   -fail-on string
     	fail (exit 1) on diagnostics at or above this severity: error|warning (default "error")
+  -opt key=value
+    	set one key=value option on the compiler the spec selects (repeatable)
   -skip-validate
     	skip the referential-integrity validate pass

cmd/morphic/validate.go

@@ -20,7 +20,9 @@ func newValidateCommand() command {
 			"--fail-on, 1 when something does or the spec lowered to no document at all,\n" +
 			"2 for a misuse or an I/O error.\n\n" +
 			"--skip-validate names the referential-integrity pass, not this command: it\n" +
-			"drops that pass's diagnostics and keeps the compiler's.",
+			"drops that pass's diagnostics and keeps the compiler's.\n\n" +
+			"--opt is compile's, and takes the same settings: a spec that needs one to\n" +
+			"compile needs it here too, or what is checked is not what would be built.",
 		printFlags: func(w io.Writer) {
 			fs, _ := newValidateFlags()
 			fs.SetOutput(w)