diff --git a/charly.yml b/charly.yml index 934f1b3..9adb95c 100644 --- a/charly.yml +++ b/charly.yml @@ -23,6 +23,9 @@ repo: github.com/opencharly/spec # ----------------------------------------------------------------------------- # bootstrap-cue — ensure ./bin/cue is the pinned cue CLI (checksum-verified). # Idempotent: a present, correct bin/cue short-circuits without a download. +# NOT part of cue-gen: generation is in-process. The CLI is the pinned tool for +# AUTHORING the schema as a module (`cue eval ./schema`, `cue vet`, a scratch +# `cue exp`), so it stays available without being on the generation leg. # ----------------------------------------------------------------------------- bootstrap-cue: task: @@ -33,8 +36,8 @@ bootstrap-cue: command: | set -euo pipefail # The version, URL, checksum and arch rules live in ONE place — - # internal/cuesetup -> cuetoolchain — shared with the charly-side generation - # verb, so this task and that verb can never run different toolchains. + # internal/cuesetup -> cuetoolchain — so this task and any other consumer + # of the pinned CLI can never run different toolchains. GOWORK=off go run ./internal/cuesetup bin context: [deploy] @@ -112,25 +115,29 @@ proto-gen: # ----------------------------------------------------------------------------- # cue-gen — regenerate every CUE-owned artifact. # -# spec/cue_types_gen.go (cue exp gengotypes — the Go param structs) +# spec/cue_types_gen.go (schemaparams — the Go param structs, generated in-process +# from the CUE Go API; no `cue` CLI in the loop) # spec/vocab_gen.go (internal/schemagen — the vocab lists) # # Reproducible: two consecutive runs produce no diff (TestGenReproducible). +# No bootstrap-cue dependency: the params generator is in-process, so the pinned +# CUE CLI is not part of this task (bootstrap-cue remains for `cue eval ./schema`). # ----------------------------------------------------------------------------- cue-gen: task: description: Regenerate all CUE-owned artifacts, including the protobuf wire contract dir: . - depends_on: [bootstrap-cue, wire-gen] + depends_on: [wire-gen] plan: - - run: gengotypes + schemagen the generated Go + - run: generate the Go params in-process + schemagen the vocab lists command: | set -euo pipefail # 1) The params file comes from the ONE pipeline (schemaparams): concatenate - # schema/*.cue under `package spec` + `@go(spec)`, run the pinned cue - # gengotypes (provisioned checksum-verified into ./bin), normalize the - # struct tags. No step of it is re-implemented here or in schemagen, and - # the charly-side verb calls the same function. + # schema/*.cue under `package spec` + `@go(spec)`, compose them as a CUE + # module instance, generate the Go types in-process with the CUE Go API, + # normalize the struct tags. No step of it is re-implemented here or in + # schemagen, the charly-side verb calls the same function, and no `cue` + # CLI is provisioned: composition and generation share ONE resolution path. GOWORK=off go run ./internal/schemagen -mode=params -schema=schema -pkg=spec -out=spec/cue_types_gen.go # 2) Companion generator -> the vocab lists. GOWORK=off go run ./internal/schemagen -mode=vocab -schema=schema -out=spec/vocab_gen.go diff --git a/cue.mod/module.cue b/cue.mod/module.cue index 283cf5f..938610a 100644 --- a/cue.mod/module.cue +++ b/cue.mod/module.cue @@ -5,8 +5,9 @@ // The language version is PINNED, not incidental: the schema's `close()` + // embedded-definition idiom (schema/00_common.cue's #Op arms) is legal through // v0.17 and rejected by v0.18.0's semantics ("field not allowed"), so v0.16.1 — -// the version spec's own cuetoolchain pins for `cue exp gengotypes` — is the -// version that keeps the package's VALUE identical to the concatenation's. +// the CUE library version this module pins in go.mod, which both the in-process +// params generator and every composing consumer evaluate under — is the version +// that keeps the package's VALUE identical to the concatenation's. // Raising it is the §10.1 pattern rewrite (TODO 2.16), not a bump. module: "github.com/opencharly/spec" diff --git a/cuetoolchain/cuetoolchain.go b/cuetoolchain/cuetoolchain.go index 189d3e3..a7ecb23 100644 --- a/cuetoolchain/cuetoolchain.go +++ b/cuetoolchain/cuetoolchain.go @@ -1,19 +1,19 @@ // Package cuetoolchain is THE single home for the pinned `cue` CLI (R3). // -// The schema→Go pipeline runs `cue exp gengotypes`, and that generator lives in -// `cuelang.org/go`'s INTERNAL tree (`internal/encoding/gotypes`), so no other -// module can call it in-process. The CLI is therefore a toolchain the pipeline -// provisions itself — checksum-verified, from a pinned release — so a reader who -// is told to run a `charly` command never sees `cue`, a checkout path, or a `cd`. +// The schema→Go params pipeline no longer runs this CLI: `schemaparams` composes and +// generates in-process through the CUE Go API (opencharly/spec#271), so generation and +// composition share one offline resolution path and no binary is provisioned. The CLI +// remains the pinned tool for AUTHORING the schema as a module — `cue eval ./schema`, +// `cue vet`, a scratch `cue exp` — provisioned checksum-verified from a pinned release +// by the `bootstrap-cue` task and by the charly-side cache directory helper. // -// The pin AND the provisioning live here, once: the `charly`-side generation verb -// calls Ensure to obtain the toolchain, and spec's own `bootstrap-cue` task is -// re-keyed onto it too (it used to carry the same version, URL and checksum inline -// in shell — two declarations of one toolchain, which is exactly what this package -// removes). One pin, one home, one fetcher. +// The pin AND the provisioning live here, once: `cuetoolchain.Ensure` is what every +// caller uses to obtain the toolchain (the `bootstrap-cue` task used to carry the same +// version, URL and checksum inline in shell — two declarations of one toolchain, which +// is exactly what this package removes). One pin, one home, one fetcher. // // STANDALONE: stdlib only, so both callers can import it without pulling in the -// packages they regenerate. +// packages they generate. package cuetoolchain import ( @@ -97,7 +97,7 @@ func Ensure(dir string) (string, error) { } sum, ok := SHA256(runtime.GOARCH) if !ok { - return "", fmt.Errorf("the pinned cue %s is not checksum-pinned for %s/%s (only linux/amd64 is) — pin it in github.com/opencharly/spec/cuetoolchain, or run the pipeline in a container", + return "", fmt.Errorf("the pinned cue %s is not checksum-pinned for %s/%s (only linux/amd64 is) — pin it in github.com/opencharly/spec/cuetoolchain, or use a linux/amd64 environment", Version, runtime.GOOS, runtime.GOARCH) } url := URL(runtime.GOARCH) diff --git a/internal/cuesetup/main.go b/internal/cuesetup/main.go index 3bff0c3..c636d1c 100644 --- a/internal/cuesetup/main.go +++ b/internal/cuesetup/main.go @@ -1,9 +1,9 @@ // Command cuesetup provisions the pinned `cue` CLI into a directory. It is the -// task-facing entry point to cuetoolchain.Ensure, so `charly task bootstrap-cue` -// and the `charly`-side generation verb (opencharly/charly#829) cannot disagree -// about which toolchain the schema→Go pipeline runs, and so the pin, its URL and -// its checksum exist in exactly one place (cuetoolchain) instead of once in Go and -// once in shell. +// task-facing entry point to cuetoolchain.Ensure, so `charly task bootstrap-cue` and +// any other consumer of the pinned CLI cannot disagree about which toolchain they run, +// and so the pin, its URL and its checksum exist in exactly one place (cuetoolchain) +// instead of once in Go and once in shell. It is NOT on the generation leg: the +// schema→Go params pipeline composes and generates in-process (no CLI). package main import ( diff --git a/internal/schemagen/main.go b/internal/schemagen/main.go index 16fb82f..eaaaf43 100644 --- a/internal/schemagen/main.go +++ b/internal/schemagen/main.go @@ -4,7 +4,8 @@ // // -mode=params -out= Concatenate schema/*.cue into one file // headed `package spec` + the file-level `@go(spec)` -// attribute, ready for `cue exp gengotypes`. +// attribute, and generate the Go types in-process +// through schemaparams (CUE Go API). // -mode=vocab -out= Compile the concatenation via the embedded // cuelang.org/go API and emit spec/vocab_gen.go // — the single-source vocabulary lists (kind keywords, @@ -24,10 +25,10 @@ // PARAM-GEN SCOPE (the two modes diverge by INPUT, not by mechanism): the // `vocab` mode reads the FULL schema (every schema/*.cue) — it needs #Node's // arms (node.cue) to derive KindWords, so its concatenation stays byte-identical -// to the runtime `sharedCueSchema`. The `params` mode (which feeds `cue exp -// gengotypes` → the Go param structs) reads the schema MINUS node.cue: it +// to the runtime `sharedCueSchema`. The `params` mode (which feeds the in-process +// Go generator → the Go param structs) reads the schema MINUS node.cue: it // defines the node-disjunction wrappers (#Node/#NodeDoc/#*Arm/#*Value), which -// gengotypes degrades to `map[string]any` and which are NOT charly param structs. +// the generator degrades to `map[string]any` and which are NOT charly param structs. // (The egress validation schemas moved to candy/plugin-egress in M16, so they no // longer live here to exclude.) The entity defs (#Box/#Deploy/#Op/#Vm/…) do not // reference node.cue, so the exclusion is compile-clean. Both modes share the ONE concatSchema helper @@ -82,9 +83,9 @@ func fatal(format string, a ...any) { } // excludeParamGen reports whether a schema/*.cue file is EXCLUDED from the -// param-gen concatenation (the gengotypes input). node.cue defines the -// node-disjunction wrappers — not a charly param struct, and gengotypes degrades -// it to `map[string]any`. See the package doc PARAM-GEN SCOPE note. The vocab +// param-gen concatenation (the generator input). node.cue defines the +// node-disjunction wrappers — not a charly param struct, and the generator +// degrades it to `map[string]any`. See the package doc PARAM-GEN SCOPE note. The vocab // mode passes `nil` (no exclusion) so #Node's arms are present for KindWords. // (The egress validation schemas moved to candy/plugin-egress in M16, so there // are no egress_*.cue files here to exclude.) @@ -106,8 +107,8 @@ func concatSchema(dir string, exclude func(name string) bool) (string, []string, } // specSource returns the concatenation headed with the `package ` clause and -// the file-level `@go()` attribute — what `cue exp gengotypes` consumes to -// emit that Go package. pkg is "spec" for the core schema and "params" for a +// the file-level `@go()` attribute — what the generator reads to name the +// Go package it emits. pkg is "spec" for the core schema and "params" for a // plugin's self-contained schema (same one concatenation contract — R3). The cue // API (vocab mode) compiles the same string. The exclude filter scopes the // param-gen input (see excludeParamGen). @@ -120,18 +121,19 @@ func specSource(dir, pkg string, exclude func(name string) bool) (string, error) } // ---------------------------------------------------------------------------- -// retag mode — the Go-native yaml-tag doubling (replaces the former cue:gen sed) +// params mode — the ONE schema→Go pipeline (concatenation, module-aware compose, +// in-process generation, tag normalization) // ---------------------------------------------------------------------------- // writeParams emits the GENERATED Go params file by calling the ONE pipeline // (schemaparams): concatenate the param-gen schema subset under the `package ` + -// `@go()` header, run the pinned cue gengotypes (provisioned through -// cuetoolchain.Ensure into ./bin), normalize the struct tags. None of those steps is -// implemented here any more — that is what makes the pipeline ONE thing rather than a -// copy per caller. `excludeParamGen` (node.cue) is this caller's only difference, and -// it is a parameter. +// `@go()` header, compose it as a CUE module instance, generate the Go types +// in-process (CUE Go API, no `cue` CLI), normalize the struct tags. None of those steps +// is implemented here any more — that is what makes the pipeline ONE thing rather than +// a copy per caller. `excludeParamGen` (node.cue) is this caller's only difference, +// and it is a parameter. func writeParams(dir, out, pkg string) error { - src, err := schemaparams.GenerateExcluding(dir, pkg, "bin", excludeParamGen) + src, err := schemaparams.GenerateExcluding(dir, pkg, excludeParamGen) if err != nil { return err } diff --git a/schema/README.md b/schema/README.md index 7aa3368..bfde989 100644 --- a/schema/README.md +++ b/schema/README.md @@ -28,8 +28,8 @@ are the single source of truth for these consumers: > package load, which is why the shared definitions live in `00_common.cue`: the > `00_` sorts first, keeping the concatenation's order — and so the generated > Go — byte-identical.) The `package spec` clause + the file-level `@go(spec)` -> attribute that `cue exp gengotypes` needs are injected by the gen pipeline -> (`internal/schemagen -mode=params`), NEVER written into the source. +> attribute that the generator reads are injected by the gen pipeline +> (`internal/schemagen -mode=params` → `schemaparams`), NEVER written into the source. ## The package form @@ -47,11 +47,13 @@ the module that owns it is `specmodule.FS` (package `github.com/opencharly/spec` Composing with only the files would evaluate them under the consumer's toolchain default instead of the pinned version above. -`cue.mod/module.cue` pins language version `v0.16.1` — the version spec's own -`cuetoolchain` pins for `cue exp gengotypes`. It is not incidental: the schema's -`close()` + embedded-definition idiom (`#Step`'s arms embedding the closed `#Op`) -is legal through v0.17 and rejected by v0.18's semantics with `field not allowed`. -Raising the language version is the §10.1 pattern rewrite (TODO 2.16), not a bump. +`cue.mod/module.cue` pins language version `v0.16.1` — the version the embedded +`cuelang.org/go` library is pinned to in `go.mod`, and the ONE declaration both the +in-process generator and every composing consumer evaluate under. It is not +incidental: the schema's `close()` + embedded-definition idiom (`#Step`'s arms +embedding the closed `#Op`) is legal through v0.17 and rejected by v0.18's semantics +with `field not allowed`. Raising the language version is the §10.1 pattern rewrite +(TODO 2.16), not a bump. ## Regenerating `spec` — `charly task cue-gen` @@ -62,31 +64,32 @@ charly task cue-gen ``` The task is `cue-gen` in this repo's `charly.yml` (the `kind: task` surface). It -depends on `bootstrap-cue` + `wire-gen` (which in turn depends on -`bootstrap-protoc`): - -1. **Bootstraps the pinned cue CLI** into `./bin/cue` (gitignored) — `v0.16.1`, - the SAME version as charly's embedded `cuelang.org/go` library, so the CLI that - runs `gengotypes` matches the library that compiles the schema. It downloads - the checksum-pinned release tarball (a no-op once `./bin/cue` is at the - pinned version). -2. **Concatenates** `schema/*.cue` via `internal/schemagen -mode=params` +depends on `wire-gen` (which in turn depends on `bootstrap-protoc`): + +1. **Concatenates** `schema/*.cue` via `internal/schemagen -mode=params` (the SAME `schemaconcat.ConcatSchema` contract as the runtime `sharedCueSchema` — one concatenation contract; the per-file `package schema` clause is stripped), headed `package spec` + `@go(spec)`. -3. **`cue exp gengotypes`** → `spec/cue_types_gen.go` (the Go param structs). -4. **`internal/schemagen -mode=vocab`** → `spec/vocab_gen.go` +2. **Composes and generates in-process** (`schemaparams`): the concatenation is + loaded as ONE CUE module instance through `schemacompose` (the same offline + resolution a plugin's served schema uses), and the Go param structs are + generated from the CUE Go API → `spec/cue_types_gen.go`. No `cue` CLI is + provisioned or executed. +3. **`internal/schemagen -mode=vocab`** → `spec/vocab_gen.go` (`KindWords`, `DocDirectives`, `StepKeywords`, `ContextWords`, `OpFields`, `OpVerbs`, `AuthoringVerbs`, … — all read straight from the compiled schema via the cue API). -5. **`gofmt`** the committed generated files. +4. **`gofmt`** the committed generated files. `wire-gen` (chained first) additionally renders `protocol/schema/*.cue` into `proto/plugin.proto` and compiles the Go gRPC stubs. -Each plugin's OWN `params/cue_types_gen.go` is generated from its self-contained +Each plugin's OWN `params/cue_types_gen.go` is generated from its own `candy/plugin-*/schema/*.cue` in the plugin's OWN repository, via the SAME -`internal/schemagen` + pinned cue pipeline (R3). The former superproject +`schemaparams` pipeline (R3) — including a schema that imports +`github.com/opencharly/spec/schema`, which resolves through the same offline +compose and emits the type through the Go package that owns it +(`github.com/opencharly/spec/spec`). The former superproject `taskfiles/Cue.yml` wrapper — which chained this repo's `cue:gen` and then looped over each plugin — was removed with the candy de-submodule work (charly#452, merged 2026-08-29); plugin param generation now lives with the plugin. @@ -103,11 +106,13 @@ the `*_gen.go` files** — change the `*.cue` source and regenerate. ## The `@go(...)` annotations -`cue exp gengotypes` is lossy: every CUE disjunction degrades to `any` / +The params generator is lossy: every CUE disjunction degrades to `any` / `map[string]any` / an empty struct (Go cannot express a disjunction), and field names get a naive `leadingCap + underscores-preserved` mapping. Two annotation classes in the `*.cue` source fix this — both are **inert for runtime validation** -(CUE attributes never participate in unification): +(CUE attributes never participate in unification). Schema comments that name +`gengotypes` describe this mapping: the generator is a faithful in-process port of +`cue exp gengotypes` v0.16.1 (`schemaparams/gotypes.go`), so each statement still holds: - **Field renames** — `@go()` makes the generated Go field name match the hand-written charly param struct (`http` → `HTTP`, `cap_add` → `CapAdd`, diff --git a/schemacompose/compose.go b/schemacompose/compose.go index 832eb9e..f8f315c 100644 --- a/schemacompose/compose.go +++ b/schemacompose/compose.go @@ -10,8 +10,8 @@ package schemacompose // contract — the serve side and the host gate — must judge the same schema with the same // code, so the ONE composer has to live where both of them may reach it. This module also // already owns the family it belongs to: schemaconcat (the concatenation contract), -// schemaparams and cuetoolchain (the generation pipeline), and the embedded schema source -// plus the module declaration (`schema.FS`, `FS`). +// schemaparams (the generation pipeline), cuetoolchain (the pinned authoring CLI), and the +// embedded schema source plus the module declaration (`schema.FS`, `FS`). // // A plugin's served schema used to be TEXT all the way down: `CompileString` on the // concatenated `schema/*.cue` body, in a context with no CUE module in scope. That made @@ -28,12 +28,14 @@ package schemacompose // no `cue.mod/pkg` vendoring, and no local `replace` (which the pinned modfile schema does // not have). // -// TWO entry points, ONE compose. `ServedSchema` composes the plugin's schema FS (the -// serve side, `BuildCapabilitiesWithRequires`), and `ServedText` composes the -// package-less body that reaches the host over the wire (`Capabilities.schema_cue`, the -// host gate). Both funnel into composeServedBody, so the two halves of the contract judge -// the same schema with the same code — which is the property sdk#490 asks for: a builtin -// and an out-of-process plugin, and the serve side and the host, agree byte for byte. +// THREE entry points, ONE compose. `ServedSchema` composes the plugin's schema FS (the +// serve side, `BuildCapabilitiesWithRequires`), `ServedText` composes the package-less body +// that reaches the host over the wire (`Capabilities.schema_cue`, the host gate), and +// `ComposeText` hands the same loaded instance to the Go projection (`schemaparams`), so a +// plugin's params are generated from the schema text both halves of the contract judge. +// All three funnel into composeServedBody, so the two halves of the contract judge the same +// schema with the same code — which is the property sdk#490 asks for: a builtin and an +// out-of-process plugin, and the serve side and the host, agree byte for byte. // // THREE measured constraints decide the shape (sdk#490, measured live): // - the loader requires a dependency's source FS to implement `module.OSRootFS`, because @@ -61,6 +63,7 @@ import ( "sync" "cuelang.org/go/cue" + "cuelang.org/go/cue/build" "cuelang.org/go/cue/cuecontext" "cuelang.org/go/cue/load" "cuelang.org/go/mod/modfile" @@ -112,6 +115,16 @@ const ( derivedDepVersion = "v0.0.0" ) +// Instance is ONE composed CUE module instance: the context its value belongs to, the +// loader's build instance and the built value. The two halves of the contract read the +// VALUE; the Go projection reads the build instance as well, because the import graph and +// the package attributes live there and not in the value. +type Instance struct { + Ctx *cue.Context + Build *build.Instance + Value cue.Value +} + // ServedSchema composes a plugin's embedded schema as one CUE module instance: the // concatenated `schema/*.cue` body of schemaFS under dir, evaluated inside the plugin's own // module declaration when its FS ships `cue.mod/module.cue`, and under the derived @@ -126,7 +139,11 @@ func ServedSchema(ctx *cue.Context, schemaFS fs.FS, dir string) (cue.Value, erro if err != nil { return cue.Value{}, err } - return composeServedBody(ctx, body, decl) + inst, err := composeServedBody(ctx, body, pluginPackage, decl) + if err != nil { + return cue.Value{}, err + } + return inst.Value, nil } // ServedText composes the package-less body a plugin serves on @@ -137,27 +154,45 @@ func ServedSchema(ctx *cue.Context, schemaFS fs.FS, dir string) (cue.Value, erro // derive. For a plugin that ships no declaration — every plugin today — the two entry // points are byte-for-byte the same compose. func ServedText(ctx *cue.Context, body string) (cue.Value, error) { - decl, err := derivedPluginModuleDecl() + inst, err := ComposeText(ctx, body, pluginPackage) if err != nil { return cue.Value{}, err } - return composeServedBody(ctx, body, decl) + return inst.Value, nil +} + +// ComposeText composes a package-less schema body under `package pkg`, in the virtual +// plugin module, under the DERIVED declaration, with spec served by the offline registry. +// It is the same compose as ServedText and is exported for the Go projection +// (schemaparams): a plugin's params are generated from the SAME source text the two halves +// of the contract judge, which is what lets that source carry +// `import "github.com/opencharly/spec/schema"` — the import resolves against the same +// offline registry, with no published CUE module and no `cue` CLI in the loop. +func ComposeText(ctx *cue.Context, body, pkg string) (*Instance, error) { + decl, err := derivedPluginModuleDecl() + if err != nil { + return nil, err + } + return composeServedBody(ctx, body, pkg, decl) } // composeServedBody is the ONE compose: the package-less schema body plus the module // declaration it is evaluated under, loaded as ./schema out of the virtual plugin module, // with spec served by the offline registry. -func composeServedBody(ctx *cue.Context, body string, decl []byte) (cue.Value, error) { +func composeServedBody(ctx *cue.Context, body, pkg string, decl []byte) (*Instance, error) { if ctx == nil { ctx = cuecontext.New() } loc, err := specModuleLoc() if err != nil { - return cue.Value{}, err + return nil, err + } + if pkg == "" { + return nil, fmt.Errorf("compose: the synthesized package name is empty") } overlay := map[string]load.Source{ filepath.Join(pluginModuleRoot, filepath.FromSlash(pluginModuleFile)): load.FromBytes(decl), - filepath.Join(pluginModuleRoot, pluginSchemaDir, "plugin.cue"): load.FromBytes([]byte("package " + pluginPackage + "\n\n" + body)), + filepath.Join(pluginModuleRoot, pluginSchemaDir, "plugin.cue"): load.FromBytes([]byte("package " + pkg + "\n\n" + body)), } instances := load.Instances([]string{"./" + pluginSchemaDir}, &load.Config{ Dir: pluginModuleRoot, @@ -165,17 +200,20 @@ func composeServedBody(ctx *cue.Context, body string, decl []byte) (cue.Value, e Registry: specRegistry{loc: loc}, }) if len(instances) != 1 { - return cue.Value{}, fmt.Errorf("composing ./%s from a plugin module produced %d instances, want 1", pluginSchemaDir, len(instances)) + return nil, fmt.Errorf("composing ./%s from a plugin module produced %d instances, want 1", pluginSchemaDir, len(instances)) } inst := instances[0] if err := inst.Err; err != nil { - return cue.Value{}, err + return nil, err } if inst.Module == "" || inst.ModuleFile == nil { - return cue.Value{}, fmt.Errorf("the plugin's ./%s loaded OUTSIDE its CUE module (module %q): the module's language.version must govern evaluation", pluginSchemaDir, inst.Module) + return nil, fmt.Errorf("the plugin's ./%s loaded OUTSIDE its CUE module (module %q): the module's language.version must govern evaluation", pluginSchemaDir, inst.Module) } v := ctx.BuildInstance(inst) - return v, v.Err() + if err := v.Err(); err != nil { + return nil, err + } + return &Instance{Ctx: ctx, Build: inst, Value: v}, nil } // pluginModuleDecl returns the declaration a plugin's schema is composed under. A plugin diff --git a/schemaconcat/schemaconcat.go b/schemaconcat/schemaconcat.go index b0800b5..301cae2 100644 --- a/schemaconcat/schemaconcat.go +++ b/schemaconcat/schemaconcat.go @@ -5,7 +5,7 @@ // file into ONE package-less compilation unit. They used to carry two // byte-identical-by-comment copies of the same loop; this is the ONE copy they // both call, so the compiled schema (what the runtime validates against) and -// the generated Go types (`cue exp gengotypes`) can never drift. +// the generated Go types (`schemaparams`) can never drift. // // The schema files are ALSO a real CUE package in the `github.com/opencharly/spec` // module (`cue.mod/module.cue`; `package schema`, loaded by `cue eval ./schema`). diff --git a/schemaparams/gohome.go b/schemaparams/gohome.go new file mode 100644 index 0000000..8e15347 --- /dev/null +++ b/schemaparams/gohome.go @@ -0,0 +1,43 @@ +package schemaparams + +import ( + "sort" + "strings" +) + +// goHome names the Go package that owns a CUE package's generated shapes. +type goHome struct { + ImportPath string + Package string +} + +// goHomes is THE relation between a CUE package a schema may import and the Go package +// that owns its shapes. It cannot be derived: CUE import paths and Go import paths are two +// namespaces, and a schema's CUE package is the SOURCE of shapes whose generated Go lives in +// a different package. spec's CUE package `github.com/opencharly/spec/schema` is the source; +// its generated params (`spec/cue_types_gen.go`, package spec) live in +// `github.com/opencharly/spec/spec`. Emitting the CUE path as a Go import names a package +// that exports only FS, so the referenced type is undefined (opencharly/spec#298, F2). +// +// ONE entry today: spec's schema is the only CUE package the compose serves offline (a +// plugin's module can import nothing else — schemacompose's registry refuses every other +// path), so it is the only package a projection can reference across an import. A package +// that gains a Go home is an edit HERE, and the projection's test compiles a generated file +// against it. +var goHomes = map[string]goHome{ + "github.com/opencharly/spec/schema": { + ImportPath: "github.com/opencharly/spec/spec", + Package: "spec", + }, +} + +// registeredGoHomes lists the relations, for an error that tells the reader where to add +// one. +func registeredGoHomes() string { + out := make([]string, 0, len(goHomes)) + for cuePath, home := range goHomes { + out = append(out, cuePath+" => "+home.ImportPath) + } + sort.Strings(out) + return strings.Join(out, ", ") +} diff --git a/schemaparams/gotypes.go b/schemaparams/gotypes.go new file mode 100644 index 0000000..eca1b79 --- /dev/null +++ b/schemaparams/gotypes.go @@ -0,0 +1,913 @@ +package schemaparams + +// gotypes.go — charly's own CUE-API Go type generator for schema params. +// +// The mapping of definitions, docs, optional fields, @go annotations, nillability, the +// generated-file banner and the final gofmt is a faithful port of +// cuelang.org/go v0.16.1's internal/encoding/gotypes (Copyright 2024 CUE Authors, Apache +// License 2.0), reproduced here because that package is INTERNAL to cuelang.org/go and no +// other module can import it — which is why the pipeline used to shell out to the `cue` +// CLI. In-process, the projection can use schemacompose's offline module resolution, so a +// plugin's schema may import the contract module (opencharly/spec#271, Direction 1). +// +// FAITHFUL BY CONSTRUCTION. Nothing about the emitted Go changes for a schema WITHOUT an +// import: schemaparams_test.go compares this generator's output byte for byte against the +// pinned `cue exp gengotypes` on the same source. The one thing not delegated to the local +// `go/format` is doc-comment canonicalization (formatDocComment): Go 1.27's printer stopped +// rewriting a struct field's doc comment, so leaving it to the toolchain would make the same +// schema regenerate differently on a 1.27 box and in the 1.26 CI. Two deliberate differences +// exist only for the import case: +// +// - an imported CUE package is NOT generated into a sibling Go package of its own; it is +// REFERENCED through the Go package that owns its shapes (gohome.go), because the +// consumer's module already contains that package and a generated twin would never be +// compiled; +// - an imported package with no registered Go home is a loud error, never a reference to a +// Go path that may not compile. + +import ( + "bytes" + "fmt" + goast "go/ast" + "go/doc/comment" + goformat "go/format" + goparser "go/parser" + goscanner "go/scanner" + gotoken "go/token" + "sort" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/ast" + "cuelang.org/go/cue/build" + + "github.com/opencharly/spec/schemacompose" +) + +// generateGo renders the Go source for the definitions of ONE composed schema instance. +// Imported CUE packages are referenced, not generated (see goImportFor). +func generateGo(inst *schemacompose.Instance) ([]byte, error) { + pkg := inst.Build + root := inst.Value + if err := root.Validate(); err != nil { + return nil, err + } + g := &generator{ + generatedTypes: make(map[string]*generatedDef), + pkg: pkg, + goImports: make(map[string]goImport), + exprImports: make(map[string]string), + pkgRoot: root, + } + aliases, err := gatherImportAliases(pkg) + if err != nil { + return nil, err + } + g.importAliases = aliases + + iter, err := root.Fields(cue.Definitions(true)) + if err != nil { + return nil, err + } + for iter.Next() { + sel := iter.Selector() + if !sel.IsDefinition() { + continue + } + path := cue.MakePath(sel) + if _, err := g.genDef(path, iter.Value()); err != nil { + return nil, err + } + } + + var buf []byte + printf := func(format string, args ...any) { + buf = fmt.Appendf(buf, format, args...) + } + // The banner is preserved verbatim: it is the generated-file marker the committed + // params files already carry, and the no-import output must stay byte-identical. + printf("// Code generated by \"cue exp gengotypes\"; DO NOT EDIT.\n\n") + printf("package %s\n\n", goPkgNameForInstance(pkg, root, nil)) + renderImports(printf, g) + for _, path := range g.emitDefs { + qpath := g.qualifiedPath(path) + + val := root.LookupPath(path) + goName := goNameFromPath(path, true) + if goName == "" { + return nil, fmt.Errorf("unexpected path in emitDefs: %q", qpath) + } + goAttr := goValueAttr(val) + if s, _ := goAttr.String(0); s != "" { + if s == "-" { + continue + } + goName = s + } + + emitDocs(printf, val.Doc()) + printf("type %s ", goName) + + // As we grab the generated source, do some sanity checks too. + gen, ok := g.generatedTypes[qpath] + if !ok { + return nil, fmt.Errorf("expected type in generatedTypes: %q", qpath) + } + if gen.inProgress { + return nil, fmt.Errorf("unexpected in-progress type in generatedTypes: %q", qpath) + } + if len(gen.src) == 0 { + return nil, fmt.Errorf("unexpected empty type in generatedTypes: %q", qpath) + } + buf = append(buf, gen.src...) + printf("\n\n") + } + + formatted, err := goformat.Source(buf) + if err != nil { + return nil, fmt.Errorf("formatting generated Go: %w\n%s", err, numberedLines(buf)) + } + return formatted, nil +} + +// numberedLines renders generated source with line numbers, the way gengotypes reports a +// formatting failure (it means the generator itself is buggy). +func numberedLines(buf []byte) string { + var b strings.Builder + for i, line := range bytes.Split(buf, []byte("\n")) { + fmt.Fprintf(&b, "% 4d: %s\n", i+1, line) + } + return b.String() +} + +// goImport describes how ONE imported CUE package appears in the generated Go file. +type goImport struct { + Path string // the Go import path + Package string // the imported Go package's declared name + Qualifier string // the name the generated source uses for it +} + +// generator holds the state for generating Go code for one CUE package instance. +type generator struct { + // generatedTypes records CUE definitions which we have analyzed and translated + // to Go type expressions. + generatedTypes map[string]*generatedDef + + // pkg is the package instance being generated. + pkg *build.Instance + + // emitDefs records paths for the definitions we should emit as Go types. + emitDefs []cue.Path + + // goImports records the imported CUE packages this file references, keyed by the + // package's CUE import path. The value names the Go package that owns its shapes — + // the CUE path and the Go path are not the same namespace (gohome.go). + goImports map[string]goImport + + // exprImports records Go imports named by an @go(,type=...) expression, keyed by Go + // import path. Those are already Go paths, so no home lookup applies. + exprImports map[string]string + + // pkgRoot is the root value of the CUE package, necessary to tell if a referenced + // value belongs to the current package or not. + pkgRoot cue.Value + + // importAliases maps a CUE import path to the explicit alias a file gives it. + importAliases map[string]string + + // def tracks the generation state for a single CUE definition. + def *generatedDef +} + +type qualifiedPath = string // [build.Instance.ImportPath] + " " + [cue.Path.String] + +func (g *generator) qualifiedPath(path cue.Path) qualifiedPath { + return g.pkg.ImportPath + " " + path.String() +} + +// generatedDef holds information about a Go type generated for a CUE definition. +type generatedDef struct { + // inProgress helps detect cyclic definitions and prevents emitting any Go source + // before we are done analyzing and generating the relevant types. + inProgress bool + + // facts records useful information about the generated type. + facts typeFacts + + // src is the generated Go type expression source. + src []byte +} + +// typeFacts holds useful information about a generated Go type. +type typeFacts struct { + // isTypeOverride records whether the generated type came from a @go(,type=expr). + isTypeOverride bool + + // isNillable records whether the generated Go type can be compared to nil, such + // that @go(,optional=nillable) can avoid wrapping it in a pointer. + isNillable bool +} + +func (g *generatedDef) printf(format string, args ...any) { + if !g.inProgress { + panic("generatedDef.printf called when inProgress is false") + } + g.src = fmt.Appendf(g.src, format, args...) +} + +type optionalStrategy int + +const ( + _ optionalStrategy = iota + // optional=zero (default); emit the Go type as-is and rely on the zero value. + optionalZero + // optional=nillable; emit the Go type with a pointer unless it can already + // be compared to nil. + optionalNillable +) + +// genDef analyzes and generates a CUE definition as a Go type, +// adding it to generatedTypes as well as to emitDefs. +func (g *generator) genDef(path cue.Path, val cue.Value) (*generatedDef, error) { + qpath := g.qualifiedPath(path) + if def, ok := g.generatedTypes[qpath]; ok { + return def, nil // already done or in progress + } + g.emitDefs = append(g.emitDefs, path) + + parentDef := g.def + def := &generatedDef{inProgress: true} + g.def = def + g.generatedTypes[qpath] = def + facts, err := g.emitType(val, optionalZero) + if err != nil { + return nil, err + } + g.def.facts = facts + g.def.inProgress = false + g.def = parentDef + return def, nil +} + +// emitType generates a CUE value as a Go type. +// When possible, the Go type is emitted in the form of a reference. +// Otherwise, an inline Go type expression is used. +func (g *generator) emitType(val cue.Value, optionalStg optionalStrategy) (typeFacts, error) { + var facts typeFacts + goAttr := goValueAttr(val) + // We prefer the form @go(Name,type=pkg.Baz) as it is explicit and extensible, + // but we are also backwards compatible with @go(Name,pkg.Baz) as emitted by `cue get go`. + // Make sure that we don't mistake @go(,foo=bar) for a type though. + attrType, _, _ := goAttr.Lookup(1, "type") + if attrType == "" { + if s, _ := goAttr.String(1); !strings.Contains(s, "=") { + attrType = s + } + } + if attrType != "" { + fset := gotoken.NewFileSet() + expr, importedByName, err := parseTypeExpr(fset, attrType) + if err != nil { + return facts, fmt.Errorf("cannot parse @go type expression: %w", err) + } + for _, pkgPath := range importedByName { + g.exprImports[pkgPath] = goImportQualifier(pkgPath) + } + collectSelectorImports(expr, importedByName, func(name string) { + g.exprImports[name] = name + }) + var buf bytes.Buffer + // We emit in plaintext, so format the parsed Go expression and print it out. + if err := goformat.Node(&buf, fset, expr); err != nil { + return facts, err + } + g.def.printf("%s", buf.Bytes()) + facts.isTypeOverride = true + return facts, nil + } + + // Note that type references don't get optionalStg, + // as @go(,optional=) only affects fields under the current type expression. + if done, facts, err := g.emitTypeReference(val); err != nil { + return typeFacts{}, err + } else if done { + return facts, nil + } + + // Inline types are below. + + switch k := val.IncompleteKind(); k { + case cue.StructKind: + return g.emitStructKind(val, optionalStg) + case cue.ListKind: + // We mainly care about patterns like [...string]. + facts.isNillable = true // slices can be nil + g.def.printf("[]") + elem := val.LookupPath(cue.MakePath(cue.AnyIndex)) + if !elem.Exists() { + g.def.printf("any /* CUE closed list */") + } else if _, err := g.emitType(elem, optionalStg); err != nil { + return facts, err + } + + case cue.NullKind: + facts.isNillable = true // pointers can be nil + g.def.printf("*struct{} /* CUE null */") + case cue.BoolKind: + g.def.printf("bool") + case cue.IntKind: + g.def.printf("int64") + case cue.FloatKind: + g.def.printf("float64") + case cue.StringKind: + g.def.printf("string") + case cue.BytesKind: + facts.isNillable = true // slices can be nil + g.def.printf("[]byte") + + case cue.NumberKind: + facts.isNillable = true // interfaces can be nil + g.def.printf("any /* CUE number; int64 or float64 */") + + case cue.TopKind: + facts.isNillable = true // interfaces can be nil + g.def.printf("any /* CUE top */") + + default: + // A disjunction of various kinds cannot be represented in Go. + if op, _ := val.Expr(); op == cue.OrOp { + facts.isNillable = true // interfaces can be nil + g.def.printf("any /* CUE disjunction: %s */", k) + break + } + facts.isNillable = true // interfaces can be nil + g.def.printf("any /* TODO: IncompleteKind: %s */", k) + } + return facts, nil +} + +// emitStructKind emits a CUE struct as a Go map, a `map[string]any` fallback for a +// disjunction of structs, or a struct with one field per CUE field. Extracted from +// emitType so each function stays under the complexity gate; the emitted bytes are +// unchanged. +func (g *generator) emitStructKind(val cue.Value, optionalStg optionalStrategy) (typeFacts, error) { + var facts typeFacts + if elem := val.LookupPath(cue.MakePath(cue.AnyString)); elem.Err() == nil { + facts.isNillable = true // maps can be nil + g.def.printf("map[string]") + if _, err := g.emitType(elem, optionalStg); err != nil { + return facts, err + } + return facts, nil + } + // A disjunction of structs cannot be represented in Go, as it does not have sum types. + // Fall back to a map of string to any. + if op, _ := val.Expr(); op == cue.OrOp { + facts.isNillable = true // maps can be nil + g.def.printf("map[string]any") + return facts, nil + } + g.def.printf("struct {\n") + iter, err := val.Fields(cue.Definitions(true), cue.Optional(true)) + if err != nil { + return facts, err + } + for iter.Next() { + sel := iter.Selector() + fieldVal := iter.Value() + if sel.IsDefinition() { + continue + } + cueName := sel.String() + if sel.IsString() { + cueName = sel.Unquoted() + } + cueName = strings.TrimRight(cueName, "?!") + emitDocs(g.def.printf, fieldVal.Doc()) + + // We want the Go name from just this selector, even when it's not a definition. + goName := goNameFromPath(cue.MakePath(sel), false) + + goAttr := fieldVal.Attribute("go") + if s, _ := goAttr.String(0); s != "" { + if s == "-" { + continue + } + goName = s + } + + optional := sel.ConstraintType()&cue.OptionalConstraint != 0 + fieldStg := optionalStg // only for this field + + switch s, ok, _ := goAttr.Lookup(1, "optional"); s { + case "zero": + fieldStg = optionalZero + case "nillable": + fieldStg = optionalNillable + default: + if ok { + return facts, fmt.Errorf("unknown optional strategy %q", s) + } + } + + // Fields whose names encoding/json cannot express are skipped with a note. + if strings.ContainsAny(cueName, "\\\"`,\n") { + g.def.printf("// CUE field %q: encoding/json does not support this field name\n\n", cueName) + continue + } + g.def.printf("%s ", goName) + + // Pointers in Go are a prefix in the syntax, but we won't find out the generated type facts + // until we have emitted its Go source, which we do into the same buffer to avoid copies. + // Luckily, since a pointer is always one byte, and we gofmt the result anyway for nice formatting, + // we can add the pointer first and replace it with whitespace later if not wanted. + ptrOffset := len(g.def.src) + g.def.printf("*") + + fieldFacts, err := g.emitType(fieldVal, fieldStg) + if err != nil { + return facts, err + } + if !usePointer(fieldFacts, optional, fieldStg) { + g.def.src[ptrOffset] = ' ' + } + + omitEmpty := "" + if optional { + omitEmpty = ",omitempty" + } + g.def.printf(" `json:\"%s%s\"`", cueName, omitEmpty) + g.def.printf("\n\n") + } + g.def.printf("}") + return facts, nil +} + +func usePointer(facts typeFacts, optional bool, strategy optionalStrategy) bool { + if facts.isTypeOverride { + // @(,type=) overrides any @(,optional=) setting + return false + } + if !optional { + // Regular and required fields never use pointers. + return false + } + switch strategy { + case optionalZero: + return false + case optionalNillable: + // Only use a pointer when the type isn't already nillable. + return !facts.isNillable + default: + panic("unreachable") + } +} + +// parseTypeExpr extends goparser.ParseExpr to allow selecting from full import paths. +// Accept `[]"go/constant".Kind` with a pre-processing step to find quoted strings, +// record them as imports keyed by package name in the returned map, and rewrite the Go +// expression to be in terms of the imported package. +func parseTypeExpr(fset *gotoken.FileSet, src string) (goast.Expr, map[string]string, error) { + var goSrc strings.Builder + importedByName := make(map[string]string) + + var scan goscanner.Scanner + scan.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), nil, 0) + lastStringLit := "" + for { + _, tok, lit := scan.Scan() + if tok == gotoken.EOF { + break + } + if lastStringLit != "" { + if tok == gotoken.PERIOD { + imp, err := strconv.Unquote(lastStringLit) + if err != nil { + panic(err) // should never happen + } + // We assume the package name is the last path component. + impName := lastPathElement(imp) + importedByName[impName] = imp + goSrc.WriteString(impName) + } else { + goSrc.WriteString(lastStringLit) + } + lastStringLit = "" + } + switch tok { + case gotoken.STRING: + lastStringLit = lit + case gotoken.IDENT, gotoken.INT, gotoken.FLOAT, gotoken.IMAG, gotoken.CHAR: + goSrc.WriteString(lit) + case gotoken.SEMICOLON: + // EOF inserts a semicolon, which breaks goparser.ParseExpr. + if lit == "\n" { + break // inserted semicolon at EOF + } + fallthrough + default: + goSrc.WriteString(tok.String()) + } + } + expr, err := goparser.ParseExpr(goSrc.String()) + return expr, importedByName, err +} + +// collectSelectorImports records the single-element package an unquoted selector names +// (`time.Duration`), skipping any package the quoted-path pre-pass already imported. +// It is the upstream astutil.Apply pass over the parsed expression, without the +// golang.org/x/tools dependency. +func collectSelectorImports(expr goast.Expr, importedByName map[string]string, add func(pkgPath string)) { + goast.Inspect(expr, func(n goast.Node) bool { + sel, ok := n.(*goast.SelectorExpr) + if !ok { + return true + } + id, ok := sel.X.(*goast.Ident) + if !ok { + return true + } + if importedByName[id.Name] != "" { + return true + } + add(id.Name) + return true + }) +} + +// lastPathElement returns the part of a slash-separated path after its last "/", or the +// whole string when it has none. +func lastPathElement(s string) string { + if i := strings.LastIndex(s, "/"); i >= 0 { + return s[i+1:] + } + return s +} + +// goNameFromPath transforms a CUE path, such as "#foo.bar?", into a suitable name for +// a generated Go type, such as "Foo_bar". When defsOnly is true, all path elements must +// be definitions, or "" is returned. +func goNameFromPath(path cue.Path, defsOnly bool) string { + export := true + var sb strings.Builder + for i, sel := range path.Selectors() { + if defsOnly && !sel.IsDefinition() { + return "" + } + if i > 0 { + // To aid in readability, nested names are separated with underscores. + sb.WriteString("_") + } + str := sel.String() + if sel.IsString() { + str = sel.Unquoted() + } + str, hidden := strings.CutPrefix(str, "_") + if hidden { + // If any part of the path is hidden, we are not exporting. + export = false + } + // Leading or trailing characters for definitions, optional, or required + // are not included as part of Go names. + str = strings.TrimPrefix(str, "#") + str = strings.TrimRight(str, "?!") + // CUE allows quoted field names such as "foo-bar" or "123baz", none of which are + // valid Go identifiers: replace forbidden characters with underscores, adding a + // leading "F" if the name begins with a digit. + for i, r := range str { + switch { + case unicode.IsLetter(r): + sb.WriteRune(r) + case unicode.IsDigit(r): + if i == 0 { + sb.WriteRune('F') + } + sb.WriteRune(r) + default: + sb.WriteRune('_') + } + } + } + name := sb.String() + if export { + // Capitalize the first letter to export the name in Go. + first, size := utf8.DecodeRuneInString(name) + name = string(unicode.ToTitle(first)) + name[size:] + } + return name +} + +// goValueAttr is like cue.Value.Attribute with the string parameter "go", but it +// supports DeclAttr attributes as well and not just FieldAttr. +func goValueAttr(val cue.Value) cue.Attribute { + attrs := val.Attributes(cue.ValueAttr) + for _, attr := range attrs { + if attr.Name() == "go" { + return attr + } + } + return cue.Attribute{} +} + +// goPkgNameForInstance determines what to name a Go package generated from a CUE +// instance. By default this is the CUE package name, but it can be overridden by a @go() +// package attribute. When supplying importAliases, and if no package attribute is found, +// the returned package name reflects the alias name that the package is being imported as. +func goPkgNameForInstance(inst *build.Instance, instVal cue.Value, importAliases map[string]string) string { + attr := goValueAttr(instVal) + if s, _ := attr.String(0); s != "" { + return s + } + if alias, ok := importAliases[inst.ImportPath]; ok { + return alias + } + return inst.PkgName +} + +// emitTypeReference attempts to generate a CUE value as a Go type via a reference, +// either to a type in the same Go package, or to a type in an imported package. +func (g *generator) emitTypeReference(val cue.Value) (bool, typeFacts, error) { + root, path := val.ReferencePath() + if len(path.Selectors()) == 0 { + return false, typeFacts{}, nil + } + inst := root.BuildInstance() + // Go has no notion of qualified import paths; if a CUE file imports + // "foo.com/bar:qualified", we import just "foo.com/bar" on the Go side. + unqualifiedPath := ast.ParseImportPath(inst.ImportPath).Unqualified().String() + + // As a special case, some CUE standard library types are allowed as references + // even though they aren't definitions. + defsOnly := true + switch fmt.Sprintf("%s.%s", unqualifiedPath, path) { + case "time.Duration": + // Note that CUE represents durations as strings, but Go as int64. + g.def.printf("string /* CUE time.Duration */") + return true, typeFacts{}, nil + case "time.Time": + defsOnly = false + } + + name := goNameFromPath(path, defsOnly) + if name == "" { + return false, typeFacts{}, nil // Not a path we are generating. + } + + var facts typeFacts + inProgress := false + // We did use a reference; if the referenced name was from another package, we + // need to ensure that package is imported. Otherwise, we need to ensure that the + // referenced local definition is generated. Either way, return the facts about the + // referenced type. + if root != g.pkgRoot { + if _, err := g.goImportFor(unqualifiedPath); err != nil { + return false, typeFacts{}, err + } + } else { + def, err := g.genDef(path, cue.Dereference(val)) + if err != nil { + return false, typeFacts{}, err + } + facts = def.facts + inProgress = def.inProgress + } + // We generate types depth-first; if the type referenced here is still in progress, + // it means that we are in a cyclic type, so be nillable to avoid a Go type of + // infinite size. + if inProgress && !facts.isNillable { + g.def.printf("*") + facts.isNillable = true // pointers can be nil + } + if root != g.pkgRoot { + g.def.printf("%s.", g.goImports[unqualifiedPath].Qualifier) + } + g.def.printf("%s", name) + return true, facts, nil +} + +// goImportFor registers the Go package a referenced CUE package is imported as and +// returns how the source names it. The CUE import path and the Go import path are +// different namespaces: the contract module's CUE package +// `github.com/opencharly/spec/schema` is the SOURCE of the shapes, while their generated +// Go types live in `github.com/opencharly/spec/spec`. goHomes carries that relation; an +// unregistered import is a loud error, because emitting the CUE path as a Go import names +// a package that does not compile (spec#298, F2). +func (g *generator) goImportFor(cuePath string) (goImport, error) { + home, ok := goHomes[cuePath] + if !ok { + return goImport{}, fmt.Errorf( + "no Go package is registered for CUE import %q: the projection references only packages whose Go home it knows (registered: %s) — add the relation to goHomes", + cuePath, registeredGoHomes()) + } + qualifier := home.Package + if alias, ok := g.importAliases[cuePath]; ok { + qualifier = alias + } + gi := goImport{Path: home.ImportPath, Package: home.Package, Qualifier: qualifier} + g.goImports[cuePath] = gi + return gi, nil +} + +// renderImports writes the import block for the packages the emitted definitions +// referenced, sorted by Go path and deduplicated across CUE packages that share a Go +// home. A qualifier that differs from the Go package's declared name is written as an +// explicit alias. +func renderImports(printf func(string, ...any), g *generator) { + type imp struct { + path string + packageNm string + qualifier string + } + byPath := make(map[string]imp) + for _, gi := range g.goImports { + byPath[gi.Path] = imp{path: gi.Path, packageNm: gi.Package, qualifier: gi.Qualifier} + } + for path, qualifier := range g.exprImports { + if _, ok := byPath[path]; ok { + continue + } + byPath[path] = imp{path: path, packageNm: goImportQualifier(path), qualifier: qualifier} + } + if len(byPath) == 0 { + return + } + paths := make([]string, 0, len(byPath)) + for p := range byPath { + paths = append(paths, p) + } + sort.Strings(paths) + printf("import (\n") + for _, p := range paths { + e := byPath[p] + if e.qualifier != e.packageNm { + printf("\t%s %q\n", e.qualifier, e.path) + } else { + printf("\t%q\n", e.path) + } + } + printf(")\n") +} + +// goImportQualifier is the conventional package name for a Go import path: its last +// element. It is only used where the path is already a Go path (an @go(,type=...) import), +// not for a CUE import (whose Go home is registered in goHomes). +func goImportQualifier(path string) string { + return lastPathElement(path) +} + +// emitDocs generates the documentation comments attached to the following declaration. +func emitDocs(printf func(string, ...any), groups []*ast.CommentGroup) { + for i, group := range groups { + if i > 0 { + printf("//\n") + } + for _, line := range formatDocComment(group.List) { + printf("%s\n", line) + } + } +} + +// formatDocComment canonicalizes a doc comment exactly as go/printer does for a Go 1.26 +// toolchain, using the SAME public go/doc/comment parser and printer the standard library +// uses. It is done here rather than left to go/format because Go 1.27's go/printer stopped +// rewriting a doc comment whose next token is an IDENT (a struct field's doc comment), so +// delegating the transform would make the generated file depend on which Go built the +// generator — the same schema would regenerate differently on a 1.27 developer box and in +// the 1.26 CI. Applying it up front makes both printers emit one form (the transform is +// idempotent, so a 1.26 printer re-running it changes nothing). +func formatDocComment(list []*ast.Comment) []string { + raw := make([]string, 0, len(list)) + if len(list) == 0 { + return raw + } + if strings.HasPrefix(list[0].Text, "/*") { + // A block doc comment is left alone by the printer (it only reformats a + // well-formed sequence of `//` lines). + for _, c := range list { + raw = append(raw, c.Text) + } + return raw + } + var b strings.Builder + var directives []string + for _, c := range list { + after, found := strings.CutPrefix(c.Text, "//") + if !found { + for _, d := range list { + raw = append(raw, d.Text) + } + return raw + } + // Accumulate //go:build etc lines separately, as the printer does. + if isDocDirective(after) { + directives = append(directives, c.Text) + continue + } + b.WriteString(strings.TrimPrefix(after, " ")) + b.WriteString("\n") + } + text := b.String() + if text == "" { + for _, c := range list { + raw = append(raw, c.Text) + } + return raw + } + + var p comment.Parser + d := p.Parse(text) + var pr comment.Printer + text = string(pr.Comment(d)) + + for text != "" { + var line string + line, text, _ = strings.Cut(text, "\n") + switch { + case line == "": + raw = append(raw, "//") + case strings.HasPrefix(line, "\t"): + raw = append(raw, "//"+line) + default: + raw = append(raw, "// "+line) + } + } + if len(directives) > 0 { + raw = append(raw, "//") + raw = append(raw, directives...) + } + return raw +} + +// isDocDirective reports whether c is a comment directive (the printer's isDirective, +// with the `//` removed). +func isDocDirective(c string) bool { + if strings.HasPrefix(c, "line ") || strings.HasPrefix(c, "extern ") || strings.HasPrefix(c, "export ") { + return true + } + colon := strings.Index(c, ":") + if colon <= 0 || colon+1 >= len(c) { + return false + } + for i := 0; i <= colon+1; i++ { + if i == colon { + continue + } + b := c[i] + if !isLowerAlphaNum(b) { + return false + } + } + return true +} + +// isLowerAlphaNum reports whether b is a lowercase ASCII letter or a digit (the character +// class a comment directive's prefix may contain). +func isLowerAlphaNum(b byte) bool { + return ('a' <= b && b <= 'z') || ('0' <= b && b <= '9') +} + +// gatherImportAliases collects the aliases from imports across the instance. +func gatherImportAliases(inst *build.Instance) (map[string]string, error) { + fileAliases := make(map[string]string) + tracked := make(map[string]int) + + type pair struct{ path, alias string } + var explicit []pair + + for _, file := range inst.Files { + for spec := range file.ImportSpecs() { + pkgPath, err := strconv.Unquote(spec.Path.Value) + if err != nil { + return nil, err + } + // Unaliased import: reserve its package name, which can come + // from an explicit or implicit qualifier. + if spec.Name == nil { + pkgName := ast.ParseImportPath(pkgPath).Qualifier + if pkgName == "" { + return nil, fmt.Errorf("could not find an imported package qualifier: %q", pkgPath) + } + tracked[pkgName]++ + continue + } + // Explicit alias: queue for resolution. + alias := spec.Name.Name + explicit = append(explicit, pair{path: pkgPath, alias: alias}) + } + } + + // Resolve explicit aliases with conflict suffixing. + for _, e := range explicit { + alias := e.alias + if count, ok := tracked[alias]; ok { + alias = fmt.Sprintf("%s%d", alias, count) + } + fileAliases[e.path] = alias + tracked[e.alias]++ // track the alias name (unsuffixed) for future conflicts + } + + return fileAliases, nil +} diff --git a/schemaparams/schemaparams.go b/schemaparams/schemaparams.go index 7f32814..be2fee4 100644 --- a/schemaparams/schemaparams.go +++ b/schemaparams/schemaparams.go @@ -1,9 +1,10 @@ // Package schemaparams is THE schema→Go params pipeline (R3). // -// One function runs the whole thing — concatenate a schema directory's `*.cue` -// files under a `package ` + `@go()` header, run the pinned -// `cue exp gengotypes`, then normalize the struct tags — so that the three places -// that need it cannot drift: +// One function runs the whole thing — concatenate a schema directory's `*.cue` files +// under a `package ` + `@go()` header, compose them as ONE CUE module +// instance through `schemacompose`, generate the Go structs in-process from the CUE Go +// API, then normalize the struct tags — so that the three places that need it cannot +// drift: // // - spec's own generator (its `cue-gen` task) produces `spec/cue_types_gen.go`; // - the `charly`-side verb (`charly candy params`) produces a plugin candy's @@ -14,58 +15,38 @@ // binary and without keeping its own copy of the recipe. // // The steps' contracts stay where they are, and this package only sequences them: -// `schemaconcat` (the concatenation), `cuetoolchain` (the pin and its fetcher), -// `schemaretag` (the tag normalization). The `gengotypes` generator itself lives in -// `cuelang.org/go`'s INTERNAL tree, which is why the pinned CLI is provisioned and -// run here rather than called in-process. +// `schemaconcat` (the concatenation), `schemacompose` (the one module-aware compose, +// with its offline resolution of the contract module), `schemaretag` (the tag +// normalization), and the in-process generator (`gotypes.go`). +// +// Composition and generation share ONE resolution path (opencharly/spec#271, Direction +// 1). A plugin's schema may carry `import "github.com/opencharly/spec/schema"`; the +// compose resolves it against the contract module the Go binary already links, with no +// published CUE module, no module cache and no `cue` CLI, and the generator emits the +// definition through the Go package that actually owns the shapes (`gohome.go`). That is +// what a projected params file needs to compile: `cue exp gengotypes` could do neither, +// because the CLI resolves modules only through an OCI registry and its generator lives +// in cuelang.org/go's INTERNAL tree. package schemaparams import ( "fmt" "os" - "os/exec" - "path/filepath" - "strings" - "github.com/opencharly/spec/cuetoolchain" + "github.com/opencharly/spec/schemacompose" "github.com/opencharly/spec/schemaconcat" "github.com/opencharly/spec/schemaretag" ) -// codegenModule names the scratch CUE module the pipeline compiles a schema as. It is -// not published anywhere; it exists so the instance HAS a module (and therefore a -// language version) rather than resolving against an implicit default. -const codegenModule = "github.com/opencharly/spec/schemaparams/codegen" - -// Generate provisions the pinned `cue` CLI into cueDir and returns the Go file the -// schema directory produces. cueDir is the caller's choice (spec's task uses `./bin`, -// the charly-side verb its own cache): the pin, the checksum, the verification and the -// arch rules all come from cuetoolchain, so every caller runs the SAME toolchain. -func Generate(schemaDir, pkg, cueDir string) ([]byte, error) { - return GenerateExcluding(schemaDir, pkg, cueDir, nil) +// Generate returns the Go file the schema directory produces. +func Generate(schemaDir, pkg string) ([]byte, error) { + return GenerateExcluding(schemaDir, pkg, nil) } // GenerateExcluding is Generate with a file filter: a schema whose disjunction -// wrappers (spec's `node.cue`) gengotypes must not see passes an `exclude` that names +// wrappers (spec's `node.cue`) are not param structs passes an `exclude` that names // them. nil includes every `*.cue` file, which is what a plugin candy wants. -func GenerateExcluding(schemaDir, pkg, cueDir string, exclude func(name string) bool) ([]byte, error) { - cueBin, err := cuetoolchain.Ensure(cueDir) - if err != nil { - return nil, err - } - return GenerateWithCueExcluding(schemaDir, pkg, cueBin, exclude) -} - -// GenerateWithCue runs the pipeline with an explicit `cue` binary — the seam for a -// caller that has already provisioned one, and for a test that stands in for the -// toolchain rather than downloading it. -func GenerateWithCue(schemaDir, pkg, cueBin string) ([]byte, error) { - return GenerateWithCueExcluding(schemaDir, pkg, cueBin, nil) -} - -// GenerateWithCueExcluding is GenerateWithCue with a file filter (see -// GenerateExcluding). -func GenerateWithCueExcluding(schemaDir, pkg, cueBin string, exclude func(name string) bool) ([]byte, error) { +func GenerateExcluding(schemaDir, pkg string, exclude func(name string) bool) ([]byte, error) { body, files, err := schemaconcat.ConcatSchema(os.DirFS(schemaDir), ".", exclude) if err != nil { return nil, fmt.Errorf("reading %s: %w", schemaDir, err) @@ -73,49 +54,16 @@ func GenerateWithCueExcluding(schemaDir, pkg, cueBin string, exclude func(name s if len(files) == 0 { return nil, fmt.Errorf("no *.cue files in %s", schemaDir) } - tmp, err := os.MkdirTemp("", "schemaparams-") + // The header is part of the contract: `package ` is re-established by the + // compose, and the file-level @go() attribute is what the generator reads to + // name the Go package. + inst, err := schemacompose.ComposeText(nil, "@go("+pkg+")\n\n"+body, pkg) if err != nil { - return nil, err - } - defer func() { _ = os.RemoveAll(tmp) }() - // The CUE module the instance is compiled as. The recipe this pipeline replaced - // wrote this file, and it is what fixes the LANGUAGE VERSION the generator resolves - // against — so it belongs to the pipeline, not to one caller's shell, and the version - // comes from the PIN (cuetoolchain) so the declaration cannot drift from the binary - // that reads it. `codegenModule` is a neutral path: the module is a scratch instance, - // not a published one. - if err := os.MkdirAll(filepath.Join(tmp, "cue.mod"), 0o755); err != nil { - return nil, err - } - mod := fmt.Sprintf("module: %q\nlanguage: version: %q\n", codegenModule, cuetoolchain.Version) - if err := os.WriteFile(filepath.Join(tmp, "cue.mod", "module.cue"), []byte(mod), 0o644); err != nil { - return nil, err - } - // The header is part of the contract: `package ` names the Go package and - // the file-level @go() attribute is what gengotypes reads to name it. - src := "package " + pkg + "\n\n@go(" + pkg + ")\n\n" + body - if err := os.WriteFile(filepath.Join(tmp, pkg+".cue"), []byte(src), 0o644); err != nil { - return nil, err - } - // The generator runs with the TEMP dir as its working directory, so the binary - // path must be absolute first: a caller that passes a relative one (spec's own - // task passes `bin`, i.e. ./bin/cue) would otherwise fail with - // `fork/exec bin/cue: no such file or directory`. - absCue := cueBin - if !filepath.IsAbs(absCue) { - var aerr error - if absCue, aerr = filepath.Abs(cueBin); aerr != nil { - return nil, fmt.Errorf("resolving the toolchain path %q: %w", cueBin, aerr) - } - } - cmd := exec.Command(absCue, "exp", "gengotypes", ".") - cmd.Dir = tmp - if out, err := cmd.CombinedOutput(); err != nil { - return nil, fmt.Errorf("cue exp gengotypes failed in %s: %w\n%s", tmp, err, strings.TrimSpace(string(out))) + return nil, fmt.Errorf("composing %s as package %s: %w", schemaDir, pkg, err) } - gen, err := os.ReadFile(filepath.Join(tmp, "cue_types_"+pkg+"_gen.go")) + gen, err := generateGo(inst) if err != nil { - return nil, fmt.Errorf("gengotypes produced no cue_types_%s_gen.go: %w", pkg, err) + return nil, fmt.Errorf("generating Go for %s: %w", schemaDir, err) } return schemaretag.Normalize(gen), nil } diff --git a/schemaparams/schemaparams_test.go b/schemaparams/schemaparams_test.go index da85ee7..03c4a25 100644 --- a/schemaparams/schemaparams_test.go +++ b/schemaparams/schemaparams_test.go @@ -1,218 +1,251 @@ package schemaparams -// schemaparams_test.go — the pipeline's contract, exercised through the real code -// path with a STAND-IN toolchain (deterministic, no network) plus one arm that runs -// the REAL provisioned `cue`, which skips visibly wherever it cannot be provisioned. +// schemaparams_test.go — the pipeline's contract. +// +// Three kinds of arm, each proving a different claim: +// +// - the IMPORT case (opencharly/spec#298, F1+F2): a plugin schema carrying +// `import "github.com/opencharly/spec/schema"` generates params that reference the +// Go package that OWNS the shapes, and the generated file COMPILES against the +// contract module (a real `go build`, not a string assertion); +// - the NO-IMPORT case: output must stay byte-identical to the pinned +// `cue exp gengotypes`, which the oracle arm proves by running that CLI on the same +// source (it skips visibly when the pinned binary cannot be provisioned); +// - the pipeline's own edges (package name, file filter, empty dir, unregistered +// import). import ( "os" + "os/exec" "path/filepath" "strings" "testing" "github.com/opencharly/spec/cuetoolchain" + "github.com/opencharly/spec/schemaconcat" + "github.com/opencharly/spec/schemaretag" ) -// stubCue writes an executable standing in for `cue exp gengotypes`: it CAPTURES the -// CUE source it was handed (so the concat+header contract can be asserted) and emits -// a fixed generated file with a json-only tag (so the retag contract can be asserted). -func stubCue(t *testing.T, captureDir, body, exitLine string) string { - t.Helper() - bin := filepath.Join(t.TempDir(), "cue") - // The stub derives the generated file's name from the CUE file it is handed - // (`.cue`), exactly as the real `cue exp gengotypes` names its output — - // so the same stub serves any package name. - script := "#!/bin/sh\n" + - "f=$(ls *.cue | head -1); pkg=$(basename \"$f\" .cue)\n" + - "cp \"$f\" \"" + captureDir + "/captured.cue\" 2>/dev/null || true\n" + - "cp cue.mod/module.cue \"" + captureDir + "/captured_module.cue\" 2>/dev/null || true\n" + - exitLine + "\n" + - "cat > \"cue_types_${pkg}_gen.go\" <<'EOF'\n" + body + "\nEOF\n" - if err := os.WriteFile(bin, []byte(script), 0o755); err != nil { - t.Fatal(err) - } - return bin -} - -func schemaDirWith(t *testing.T, files map[string]string) string { - t.Helper() - dir := filepath.Join(t.TempDir(), "schema") - if err := os.MkdirAll(dir, 0o755); err != nil { - t.Fatal(err) +// TestGenerateNoImportIsByteIdenticalToGengotypes is the no-regression proof: the +// in-process generator must produce EXACTLY what the pinned `cue exp gengotypes` produces +// for a schema without an import (the twin shape, every plugin's shape today). The oracle +// re-runs the old recipe verbatim — concat, scratch module, CLI, tag normalization — so +// the comparison is against the real toolchain, never a hand-written expectation. +func TestGenerateNoImportIsByteIdenticalToGengotypes(t *testing.T) { + cue := pinnedCue(t) + if cue == "" { + t.Skipf("pinned cue %s could not be provisioned here — the byte-identity oracle is skipped, not faked", cuetoolchain.Version) + } + dir := filepath.Join("testdata", "twin", "schema") + want := gengotypesOracle(t, dir, "params", cue) + got, err := Generate(dir, "params") + if err != nil { + t.Fatalf("Generate: %v", err) } - for name, body := range files { - if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { - t.Fatal(err) - } + if string(got) != string(want) { + t.Fatalf("in-process output differs from `cue exp gengotypes` (%d vs %d bytes)\n--- gengotypes ---\n%s\n--- in-process ---\n%s", len(want), len(got), want, got) } - return dir } -const stubBody = "package params\n\ntype Foo struct {\n\tName string `json:\"name\"`\n}" - -// TestGenerateWithCueRunsConcatHeaderAndRetag pins all three steps in one arm: the -// schema files arrive concatenated under the `package params` + `@go(params)` header -// (captured from the child), and the returned file has the tags normalized. -func TestGenerateWithCueRunsConcatHeaderAndRetag(t *testing.T) { - capture := t.TempDir() - schema := schemaDirWith(t, map[string]string{ - "a.cue": "#A: {\n\tname?: string\n}\n", - "b.cue": "#B: {\n\tother?: string\n}\n", - }) - bin := stubCue(t, capture, stubBody, "true") - - got, err := GenerateWithCue(schema, "params", bin) +// TestGenerateImportReferencesTheOwningGoPackage is F2: the emitted Go must import the +// package that owns the shapes (`github.com/opencharly/spec/spec`), never the CUE +// package path whose Go package exports only FS. +func TestGenerateImportReferencesTheOwningGoPackage(t *testing.T) { + out, err := Generate(filepath.Join("testdata", "import", "schema"), "params") if err != nil { - t.Fatalf("GenerateWithCue: %v", err) + t.Fatalf("Generate: %v", err) + } + src := string(out) + if !strings.Contains(src, "\"github.com/opencharly/spec/spec\"") { + t.Fatalf("generated file does not import the Go package that owns the shapes:\n%s", src) } - if want := "package params\n\ntype Foo struct {\n\tName string `yaml:\"name,omitempty\" json:\"name\"`\n}\n"; string(got) != want { - t.Fatalf("pipeline output =\n%q\nwant\n%q", got, want) + if strings.Contains(src, "\"github.com/opencharly/spec/schema\"") { + t.Fatalf("generated file imports the CUE package path as Go (F2):\n%s", src) } + if !strings.Contains(src, "Config spec.TunnelConfig ") { + t.Fatalf("generated field does not reference spec.TunnelConfig:\n%s", src) + } +} - handed, err := os.ReadFile(filepath.Join(capture, "captured.cue")) +// TestGenerateImportCompiles proves the import case at its REAL boundary: the generated +// params file is placed in a module that requires this one and `go build` compiles it. +// A string assertion cannot catch a Go package that exists but lacks the type (the exact +// F2 failure), so this arm builds it. +func TestGenerateImportCompiles(t *testing.T) { + out, err := Generate(filepath.Join("testdata", "import", "schema"), "params") if err != nil { - t.Fatalf("the toolchain was not handed a CUE source: %v", err) + t.Fatalf("Generate: %v", err) } - src := string(handed) - if !strings.HasPrefix(src, "package params\n\n@go(params)\n\n") { - t.Fatalf("the concatenation is not headed by the package clause + @go attribute:\n%.120s", src) + root, err := filepath.Abs("..") + if err != nil { + t.Fatal(err) } - for _, want := range []string{"#A: {", "#B: {"} { - if !strings.Contains(src, want) { - t.Fatalf("the concatenation is missing %s:\n%s", want, src) - } + // The temp module mirrors THIS module's declaration: the same go directive and the + // same requirement set, with only the module path rewritten and the contract module + // added as a local replace. A minimal `require spec` makes `go build` EXPAND the + // module graph, which asks the proxy for go.mod files the repo's own build never + // needs — measured in CI, where gopkg.in/check.v1's go.mod was absent from the cache + // and GOPROXY=off refused the lookup. A complete declaration is also what a real + // consumer of this contract module has. + mod, err := os.ReadFile(filepath.Join(root, "go.mod")) + if err != nil { + t.Fatalf("read the module's go.mod: %v", err) } - if strings.Index(src, "#A: {") > strings.Index(src, "#B: {") { - t.Fatalf("the concatenation is not in sorted file order:\n%s", src) + gomod := strings.Replace(string(mod), "module github.com/opencharly/spec", "module example.com/pluginparams", 1) + if gomod == string(mod) { + t.Fatalf("the contract module's go.mod does not start with its module path") } -} - -// TestGenerateWithCueEmitsTheCodegenModule pins the emitted ENVIRONMENT, not just the -// output: the pipeline compiles the schema as a CUE module whose language version comes -// from the pin, because the recipe it replaced wrote exactly that file and the version -// it declares is what the generator resolves against. Without this arm the module could -// silently disappear again while every output assertion still passed. -func TestGenerateWithCueEmitsTheCodegenModule(t *testing.T) { - capture := t.TempDir() - schema := schemaDirWith(t, map[string]string{"a.cue": "#A: {\n\tname?: string\n}\n"}) - if _, err := GenerateWithCue(schema, "params", stubCue(t, capture, stubBody, "true")); err != nil { - t.Fatalf("GenerateWithCue: %v", err) - } - mod, err := os.ReadFile(filepath.Join(capture, "captured_module.cue")) + gomod += "\nrequire github.com/opencharly/spec v0.0.0\n\nreplace github.com/opencharly/spec => " + root + "\n" + sum, err := os.ReadFile(filepath.Join(root, "go.sum")) if err != nil { - t.Fatalf("the pipeline compiled the schema with NO cue.mod/module.cue: %v", err) + t.Fatalf("read the module's go.sum (needed by the temp module's build): %v", err) } - if !strings.Contains(string(mod), "module: \""+codegenModule+"\"") { - t.Fatalf("module.cue does not declare the codegen module:\n%s", mod) + tmp := t.TempDir() + if err := os.MkdirAll(filepath.Join(tmp, "params"), 0o755); err != nil { + t.Fatal(err) } - if !strings.Contains(string(mod), "language: version: \""+cuetoolchain.Version+"\"") { - t.Fatalf("module.cue does not declare the PINNED language version %s:\n%s", cuetoolchain.Version, mod) + if err := os.WriteFile(filepath.Join(tmp, "params", "cue_types_gen.go"), out, 0o644); err != nil { + t.Fatal(err) } -} - -// TestGenerateWithCueHonoursThePackageName: the header follows the caller's package, -// so the same function serves a plugin (`params`) and spec itself (`spec`). -func TestGenerateWithCueHonoursThePackageName(t *testing.T) { - capture := t.TempDir() - schema := schemaDirWith(t, map[string]string{"a.cue": "#A: {\n\tname?: string\n}\n"}) - bin := stubCue(t, capture, "package spec\n\ntype Foo struct {\n\tName string `json:\"name\"`\n}", "true") - - if _, err := GenerateWithCue(schema, "spec", bin); err != nil { - t.Fatalf("GenerateWithCue: %v", err) + if err := os.WriteFile(filepath.Join(tmp, "go.mod"), []byte(gomod), 0o644); err != nil { + t.Fatal(err) } - handed, err := os.ReadFile(filepath.Join(capture, "captured.cue")) - if err != nil { + if err := os.WriteFile(filepath.Join(tmp, "go.sum"), sum, 0o644); err != nil { t.Fatal(err) } - if !strings.HasPrefix(string(handed), "package spec\n\n@go(spec)\n\n") { - t.Fatalf("the header does not follow the caller's package:\n%.80s", handed) + // No GOFLAGS: the default readonly mode uses the complete declaration above and + // never tries to rewrite it (a rewrite is what needed the proxy). + cmd := exec.Command("go", "build", "./params") + cmd.Dir = tmp + cmd.Env = append(os.Environ(), "GOWORK=off", "GOPROXY=off") + if buildOut, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("the generated params do not compile against the contract module: %v\n%s\n--- generated ---\n%s", err, buildOut, out) } } -// TestGenerateWithCueAcceptsARelativeToolchainPath is the arm that would have caught -// the bug this pipeline shipped for one run: the generator is exec'd with the TEMP dir -// as its working directory, so a RELATIVE cueBin (spec's own task passes `bin`) must be -// resolved against the caller's working directory first — otherwise it fails with -// `fork/exec bin/cue: no such file or directory`. -func TestGenerateWithCueAcceptsARelativeToolchainPath(t *testing.T) { - home, err := os.Getwd() - if err != nil { - t.Fatal(err) +// TestGenerateHonoursThePackageName: the header follows the caller's package, so the same +// pipeline serves a plugin (`params`) and spec itself (`spec`). +func TestGenerateHonoursThePackageName(t *testing.T) { + dir := filepath.Join("testdata", "twin", "schema") + for _, pkg := range []string{"params", "spec"} { + out, err := Generate(dir, pkg) + if err != nil { + t.Fatalf("Generate(%q): %v", pkg, err) + } + if !strings.Contains(string(out), "\npackage "+pkg+"\n") { + t.Fatalf("generated file does not declare package %q:\n%.120s", pkg, out) + } } +} + +// TestGenerateExcludingHonoursTheFilter: the file filter is the caller's only input +// difference (spec excludes node.cue), so an excluded file must contribute nothing. +func TestGenerateExcludingHonoursTheFilter(t *testing.T) { dir := t.TempDir() - if err := os.Chdir(dir); err != nil { - t.Fatal(err) + for name, body := range map[string]string{ + "a.cue": "#A: {\n\tname?: string\n}\n", + "b.cue": "#B: {\n\tother?: string\n}\n", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } } - t.Cleanup(func() { _ = os.Chdir(home) }) - binDir := filepath.Join(dir, "bin") - if err := os.MkdirAll(binDir, 0o755); err != nil { - t.Fatal(err) + out, err := GenerateExcluding(dir, "params", func(name string) bool { return name == "b.cue" }) + if err != nil { + t.Fatalf("GenerateExcluding: %v", err) } - bin := stubCue(t, dir, stubBody, "true") - if err := os.Rename(bin, filepath.Join(binDir, "cue")); err != nil { - t.Fatal(err) + if !strings.Contains(string(out), "type A struct") { + t.Fatalf("the included definition is missing:\n%s", out) } - schema := schemaDirWith(t, map[string]string{"a.cue": "#A: {\n\tname?: string\n}\n"}) - if _, err := GenerateWithCue(schema, "params", filepath.Join("bin", "cue")); err != nil { - t.Fatalf("a relative toolchain path must resolve against the caller's working directory: %v", err) + if strings.Contains(string(out), "type B struct") { + t.Fatalf("the EXCLUDED file still generated a type:\n%s", out) } } -// TestGenerateWithCueRejectsAnEmptySchemaDir: "no schema" must be an error naming the -// directory, never an empty generated file. -func TestGenerateWithCueRejectsAnEmptySchemaDir(t *testing.T) { +// TestGenerateRejectsAnEmptySchemaDir: "no schema" must be an error naming the directory, +// never an empty generated file. +func TestGenerateRejectsAnEmptySchemaDir(t *testing.T) { dir := filepath.Join(t.TempDir(), "empty") if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatal(err) } - _, err := GenerateWithCue(dir, "params", stubCue(t, t.TempDir(), stubBody, "true")) + _, err := Generate(dir, "params") if err == nil || !strings.Contains(err.Error(), "no *.cue files") { t.Fatalf("want an empty-schema error, got %v", err) } } -// TestGenerateWithCueSurfacesAToolchainFailure: a failing generator is reported with -// its own output, not as an empty or partial result. -func TestGenerateWithCueSurfacesAToolchainFailure(t *testing.T) { - schema := schemaDirWith(t, map[string]string{"a.cue": "#A: {\n\tname?: string\n}\n"}) - bin := stubCue(t, t.TempDir(), stubBody, "echo 'cue: boom' >&2; exit 1") - _, err := GenerateWithCue(schema, "params", bin) - if err == nil { - t.Fatal("a failing toolchain produced no error") - } - if !strings.Contains(err.Error(), "gengotypes failed") || !strings.Contains(err.Error(), "cue: boom") { - t.Fatalf("the failure must name the step and carry the toolchain's own output, got: %v", err) +// TestGoImportForRejectsAnUnregisteredPackage: an import the projection cannot name a Go +// home for is a loud error, never a reference to a Go path that may not compile. +func TestGoImportForRejectsAnUnregisteredPackage(t *testing.T) { + g := &generator{goImports: map[string]goImport{}, importAliases: map[string]string{}} + _, err := g.goImportFor("example.com/other/schema") + if err == nil || !strings.Contains(err.Error(), "no Go package is registered") { + t.Fatalf("want an unregistered-import error, got %v", err) + } +} + +// TestGoImportForUsesTheGoHome: the relation maps the CUE package to its Go owner, and an +// explicit CUE import alias is preserved as the Go qualifier. +func TestGoImportForUsesTheGoHome(t *testing.T) { + g := &generator{goImports: map[string]goImport{}, importAliases: map[string]string{"github.com/opencharly/spec/schema": "contract"}} + gi, err := g.goImportFor("github.com/opencharly/spec/schema") + if err != nil { + t.Fatal(err) + } + if gi.Path != "github.com/opencharly/spec/spec" || gi.Qualifier != "contract" { + t.Fatalf("goImportFor = %+v, want path spec/spec and qualifier contract", gi) } } -// TestGenerateProvisionsThePinnedToolchain runs the REAL pipeline: Generate resolves -// the pinned `cue` through cuetoolchain.Ensure (no stub), generates from a tiny schema -// and normalizes the tags. It needs the release to be provisionable, so it skips -// VISIBLY — never with a canned substitute — when that is impossible here. -func TestGenerateProvisionsThePinnedToolchain(t *testing.T) { - schema := schemaDirWith(t, map[string]string{ - "a.cue": "// The leading doc comment, which gengotypes reproduces.\n#A: {\n\tname?: string\n\tretry_limit?: int\n}\n", - }) +// pinnedCue resolves the pinned `cue` CLI for the oracle arm, provisioning it through the +// SAME cuetoolchain the repo's bootstrap task uses. "" means it cannot run here. +func pinnedCue(t *testing.T) string { + t.Helper() cache, err := os.UserCacheDir() if err != nil { - t.Skipf("no user cache dir to provision the toolchain into: %v", err) + return "" } - got, err := Generate(schema, "params", filepath.Join(cache, "charly", "cue", cuetoolchain.Version)) + bin, err := cuetoolchain.Ensure(filepath.Join(cache, "charly", "cue", cuetoolchain.Version)) if err != nil { - t.Skipf("the pinned cue toolchain could not be provisioned here (%v) — this arm is skipped, not faked", err) + return "" } - src := string(got) - if !strings.Contains(src, "package params") { - t.Fatalf("generated source has no package clause:\n%.200s", src) + return bin +} + +// gengotypesOracle reproduces the pipeline the shell-out used: concatenate the schema +// directory, head it `package ` + `@go()`, write a scratch CUE module, run the +// pinned `cue exp gengotypes`, normalize the tags. It is a TEST-ONLY oracle: the old +// recipe must be reproducible to compare against, but it is not product code any more. +func gengotypesOracle(t *testing.T, schemaDir, pkg, cueBin string) []byte { + t.Helper() + body, files, err := schemaconcat.ConcatSchema(os.DirFS(schemaDir), ".", nil) + if err != nil { + t.Fatalf("concat: %v", err) } - // The leading doc comment is reproduced by the generator — the trap that makes a - // grep-for-a-retired-string check useless — and the tags are normalized. - if !strings.Contains(src, "The leading doc comment") { - t.Fatalf("the schema's leading doc comment did not survive into the generated file:\n%.300s", src) + if len(files) == 0 { + t.Fatalf("no *.cue files in %s", schemaDir) } - for _, line := range strings.Split(src, "\n") { - if strings.Contains(line, "json:\"") && !strings.Contains(line, "yaml:\"") { - t.Fatalf("a generated line carries a json tag with no yaml tag (retag did not run):\n%s", line) - } + tmp := t.TempDir() + if err := os.MkdirAll(filepath.Join(tmp, "cue.mod"), 0o755); err != nil { + t.Fatal(err) + } + mod := "module: \"github.com/opencharly/spec/schemaparams/codegen\"\nlanguage: version: \"" + cuetoolchain.Version + "\"\n" + if err := os.WriteFile(filepath.Join(tmp, "cue.mod", "module.cue"), []byte(mod), 0o644); err != nil { + t.Fatal(err) + } + src := "package " + pkg + "\n\n@go(" + pkg + ")\n\n" + body + if err := os.WriteFile(filepath.Join(tmp, pkg+".cue"), []byte(src), 0o644); err != nil { + t.Fatal(err) + } + cmd := exec.Command(cueBin, "exp", "gengotypes", ".") + cmd.Dir = tmp + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("cue exp gengotypes failed: %v\n%s", err, out) + } + gen, err := os.ReadFile(filepath.Join(tmp, "cue_types_"+pkg+"_gen.go")) + if err != nil { + t.Fatalf("gengotypes produced no output: %v", err) } + return schemaretag.Normalize(gen) } diff --git a/schemaparams/testdata/import/schema/tunnel.cue b/schemaparams/testdata/import/schema/tunnel.cue new file mode 100644 index 0000000..5701923 --- /dev/null +++ b/schemaparams/testdata/import/schema/tunnel.cue @@ -0,0 +1,45 @@ +// The OUT-OF-TREE plugin-tunnel's OWN CUE schema — the typed params for the +// `tunnel` VERB (verb:tunnel), the externalized tailscale/cloudflare TUNNEL +// EXECUTION LEG. It is the SINGLE SOURCE for this plugin's params, used two ways +// (the same contract core `spec` + the reference examplerunverb use): +// +// 1. GENERATE the Go param struct — the schema→Go pipeline concatenates this file under a +// `package params` + `@go(params)` header and generates the Go struct from it, +// emits +// ../params/cue_types_gen.go, so the provider decodes plugin_input into a TYPED +// struct (params.TunnelInput / TunnelConfig / TunnelPort), never a hand-parsed map. +// 2. VALIDATE authored input AT RUNTIME — the plugin SERVES this source over the +// Describe channel (sdk.BuildCapabilities → schema_cue); the host splices it onto +// its base (base ++ plugin) and validates every authored `plugin: tunnel` step's +// plugin_input against #TunnelInput. The host NEVER reads this file from disk — the +// schema travels with the plugin. +// +// verb:tunnel is NOT a check verb in the usual sense — it is the externalized TUNNEL +// EXECUTION backend. The pod-lifecycle plugins (candy/plugin-deploy-pod, candy/plugin-pod) +// drive its start/stop/setup methods directly over this verb's Invoke envelope +// ({method,config}) via InvokeProvider. It ALSO +// carries a benign `plan` (dry-run) method — a `plugin: tunnel` check step returns the +// EXACT tailscale/cloudflared argv it WOULD run WITHOUT exec, so a disposable bed proves +// the registry dispatch + the TunnelConfig wire round-trip + the moved command-building +// logic with ZERO tailscale/cloudflare credentials. +// +// SELF-CONTAINED: #TunnelInput references only its sibling defs (#TunnelConfig / +// #TunnelPort) WITHIN this schema, NO base def, so it compiles standalone (gengotypes + +// the SDK serve-side check) AND splices onto the base — the splice exists to detect a +// def-name collision with the base, not to resolve base references. + +import "github.com/opencharly/spec/schema" + +// #TunnelInput — the plugin_input shape for the `tunnel` verb. +#TunnelInput: { + // method — the tunnel operation: start | stop | setup | plan. `plan` is the + // creds-free dry-run that returns the argv the operation WOULD run (no exec). + method: string & !="" @go(Method) + // config — the resolved tunnel configuration to act on (byte-compatible with the + // core's TunnelConfig, sent by the calling pod-lifecycle plugin over the Invoke envelope). + config?: schema.#TunnelConfig @go(Config) + // expect — plan only: the expected argv command lines (space-joined). When set, the + // plan method compares the built argv against it (FAIL on mismatch); empty ⇒ the + // step passes as long as dispatch + round-trip succeed, echoing the built argv. + expect?: [...string] @go(Expect) +} diff --git a/schemaparams/testdata/twin/schema/tunnel.cue b/schemaparams/testdata/twin/schema/tunnel.cue new file mode 100644 index 0000000..1de0eba --- /dev/null +++ b/schemaparams/testdata/twin/schema/tunnel.cue @@ -0,0 +1,72 @@ +// The OUT-OF-TREE plugin-tunnel's OWN CUE schema — the typed params for the +// `tunnel` VERB (verb:tunnel), the externalized tailscale/cloudflare TUNNEL +// EXECUTION LEG. It is the SINGLE SOURCE for this plugin's params, used two ways +// (the same contract core `spec` + the reference examplerunverb use): +// +// 1. GENERATE the Go param struct — the schema→Go pipeline concatenates this file under a +// `package params` + `@go(params)` header and generates the Go struct from it, +// emits +// ../params/cue_types_gen.go, so the provider decodes plugin_input into a TYPED +// struct (params.TunnelInput / TunnelConfig / TunnelPort), never a hand-parsed map. +// 2. VALIDATE authored input AT RUNTIME — the plugin SERVES this source over the +// Describe channel (sdk.BuildCapabilities → schema_cue); the host splices it onto +// its base (base ++ plugin) and validates every authored `plugin: tunnel` step's +// plugin_input against #TunnelInput. The host NEVER reads this file from disk — the +// schema travels with the plugin. +// +// verb:tunnel is NOT a check verb in the usual sense — it is the externalized TUNNEL +// EXECUTION backend. The pod-lifecycle plugins (candy/plugin-deploy-pod, candy/plugin-pod) +// drive its start/stop/setup methods directly over this verb's Invoke envelope +// ({method,config}) via InvokeProvider. It ALSO +// carries a benign `plan` (dry-run) method — a `plugin: tunnel` check step returns the +// EXACT tailscale/cloudflared argv it WOULD run WITHOUT exec, so a disposable bed proves +// the registry dispatch + the TunnelConfig wire round-trip + the moved command-building +// logic with ZERO tailscale/cloudflare credentials. +// +// SELF-CONTAINED: #TunnelInput references only its sibling defs (#TunnelConfig / +// #TunnelPort) WITHIN this schema, NO base def, so it compiles standalone (gengotypes + +// the SDK serve-side check) AND splices onto the base — the splice exists to detect a +// def-name collision with the base, not to resolve base references. + +// #TunnelInput — the plugin_input shape for the `tunnel` verb. +#TunnelInput: { + // method — the tunnel operation: start | stop | setup | plan. `plan` is the + // creds-free dry-run that returns the argv the operation WOULD run (no exec). + method: string & !="" @go(Method) + // config — the resolved tunnel configuration to act on (byte-compatible with the + // core's TunnelConfig, sent by the calling pod-lifecycle plugin over the Invoke envelope). + config?: #TunnelConfig @go(Config) + // expect — plan only: the expected argv command lines (space-joined). When set, the + // plan method compares the built argv against it (FAIL on mismatch); empty ⇒ the + // step passes as long as dispatch + round-trip succeed, echoing the built argv. + expect?: [...string] @go(Expect) +} + +// #TunnelConfig — the resolved, ready-to-execute tunnel configuration. +#TunnelConfig: { + // provider — "tailscale" or "cloudflare". + provider?: string @go(Provider) + // tunnel_name — cloudflare: tunnel name. + tunnel_name?: string @go(TunnelName) + // hostname — cloudflare: default hostname (from the image dns field). + hostname?: string @go(Hostname) + // box_name — for PID file naming / cloudflare tunnel-name default. + box_name?: string @go(BoxName) + // ports — all tunneled ports with their access scope. + ports?: [...#TunnelPort] @go(Ports) +} + +// #TunnelPort — a single port to tunnel with its protocol and access scope. +#TunnelPort: { + // port — the tailscale HTTPS listen port (must be a valid serve/funnel port). + port?: int @go(Port) + // backend_port — the localhost backend port (0 means same as port). + backend_port?: int @go(BackendPort) + // protocol — backend scheme: http | https | https+insecure | tcp | + // tls-terminated-tcp | ssh | rdp | smb (udp is skipped, never tunneled). + protocol?: string @go(Protocol) + // public — true = internet-accessible (funnel), false = private (tailnet-only serve). + public?: bool @go(Public) + // hostname — cloudflare: per-port hostname (from the map form). + hostname?: string @go(Hostname) +} diff --git a/schemaretag/schemaretag.go b/schemaretag/schemaretag.go index 8946949..c8667ff 100644 --- a/schemaretag/schemaretag.go +++ b/schemaretag/schemaretag.go @@ -1,21 +1,21 @@ // Package schemaretag is THE single struct-tag normalization contract for the // schema→Go pipeline (R3). // -// `cue exp gengotypes` emits json tags only, but every committed +// The params generator emits json tags only, but every committed // `cue_types_gen.go` in this org carries yaml tags too — so the generation -// pipeline has a THIRD step after concatenation (`schemaconcat`) and generation: -// normalize the struct tags, doubling every json tag with a yaml tag of the same -// name and giving a bare yaml key `,omitempty`. +// pipeline has a THIRD step after concatenation (`schemaconcat`) and generation +// (`schemaparams`): normalize the struct tags, doubling every json tag with a yaml +// tag of the same name and giving a bare yaml key `,omitempty`. // // That step used to live only inside `internal/schemagen`, which no other module // may import (Go's internal rule) — which is why a charly-native regeneration // verb could not reuse it, and why the recipe had to be published as -// checkout-relative shell. It lives here now: `internal/schemagen`'s retag mode -// and the `charly`-side generation verb both call THIS, so a generated file can -// never depend on which of them produced it. +// checkout-relative shell. It lives here now: `schemaparams` calls THIS, and the +// `charly`-side generation verb reaches it through `schemaparams`, so a generated +// file can never depend on which of them produced it. // -// STANDALONE: depends only on the stdlib `regexp`, so both callers can import it -// without pulling in the packages they regenerate. +// STANDALONE: depends only on the stdlib `regexp`, so its caller can import it +// without pulling in the packages it regenerates. package schemaretag import "regexp" diff --git a/spec/gen_repro_test.go b/spec/gen_repro_test.go index 50c9bb1..fe5f79c 100644 --- a/spec/gen_repro_test.go +++ b/spec/gen_repro_test.go @@ -2,14 +2,13 @@ package spec // Reproducibility gate: the committed cue_types_gen.go and vocab_gen.go MUST // equal a fresh `charly task cue-gen`. This re-runs the SAME tools the task runs (the -// internal/schemagen concat + the pinned cue exp gengotypes + the -// schemagen vocab emitter) into a temp dir and diffs the result against the -// committed files. It skips gracefully when the pinned cue CLI is unavailable -// (a dev box without ./bin/cue), but in CI — where `charly task cue-gen` has run — it -// catches any drift between schema/*.cue and the committed generated Go. +// internal/schemagen concat + the in-process schemaparams generator + the +// schemagen vocab emitter) into a temp dir and compares the result BYTE FOR BYTE +// against the committed files. Nothing here needs the `cue` CLI — generation is +// in-process — so the gate never skips, and it catches any drift between +// schema/*.cue and the committed generated Go. import ( - "go/format" "os" "os/exec" "path/filepath" @@ -17,34 +16,12 @@ import ( "testing" ) -const cueVersion = "v0.16.1" - -// findCue resolves the pinned cue CLI: ../bin/cue (repo bin) or a PATH `cue` — -// whichever reports cueVersion. Returns "" if none. -func findCue(t *testing.T) string { - t.Helper() - candidates := []string{ - filepath.Join("..", "bin", "cue"), // repoRoot/bin/cue (cwd = spec/) - } - if p, err := exec.LookPath("cue"); err == nil { - candidates = append(candidates, p) - } - for _, c := range candidates { - out, err := exec.Command(c, "version").CombinedOutput() - if err == nil && strings.Contains(string(out), cueVersion) { - abs, _ := filepath.Abs(c) - return abs - } - } - return "" -} - // freshTypesGen reproduces spec/cue_types_gen.go through the ONE pipeline: the same // `schemagen -mode=params` the `cue-gen` task runs, which delegates to // schemaparams.GenerateExcluding. The test does not re-run the individual steps — -// concatenation, `cue exp gengotypes` and the tag normalization all live in that one -// function now, and a test that reproduced them itself would be a second copy of the -// mechanism it is meant to gate (and would not notice the task's path breaking). +// concatenation, composition, generation and the tag normalization all live in that +// one function now, and a test that reproduced them itself would be a second copy of +// the mechanism it is meant to gate (and would not notice the task's path breaking). func freshTypesGen(t *testing.T) []byte { t.Helper() out := filepath.Join(t.TempDir(), "cue_types_gen.go") @@ -53,7 +30,7 @@ func freshTypesGen(t *testing.T) []byte { if err != nil { t.Fatalf("read fresh types: %v", err) } - return gofmt(t, raw) + return raw } // freshVocabGen reproduces spec/vocab_gen.go via schemagen -mode=vocab. @@ -65,7 +42,7 @@ func freshVocabGen(t *testing.T) []byte { if err != nil { t.Fatalf("read fresh vocab: %v", err) } - return gofmt(t, raw) + return raw } func runIn(t *testing.T, dir, name string, args ...string) { @@ -77,30 +54,16 @@ func runIn(t *testing.T, dir, name string, args ...string) { } } -func gofmt(t *testing.T, b []byte) []byte { - t.Helper() - f, err := format.Source(b) - if err != nil { - t.Fatalf("gofmt: %v", err) - } - return f -} - func committed(t *testing.T, name string) []byte { t.Helper() raw, err := os.ReadFile(name) if err != nil { t.Fatalf("read committed %s: %v", name, err) } - return gofmt(t, raw) + return raw } func TestGenReproducible(t *testing.T) { - cue := findCue(t) - if cue == "" { - t.Skipf("pinned cue %s not available (run `charly task cue-gen` to bootstrap ./bin/cue) — skipping reproducibility gate", cueVersion) - } - if got, want := freshTypesGen(t), committed(t, "cue_types_gen.go"); !equalBytes(got, want) { t.Errorf("cue_types_gen.go is STALE: a fresh `charly task cue-gen` differs from the committed file.\n"+ "Run `charly task cue-gen` and commit the result. (fresh=%d bytes, committed=%d bytes)", len(got), len(want))