From 47738b7067571a475f304f9072a29d1470dc6c01 Mon Sep 17 00:00:00 2001 From: aitrawog Date: Mon, 9 Nov 2026 15:11:00 +0100 Subject: [PATCH 1/2] feat(schemaparams): generate params in-process over the offline compose Replace the `cue exp gengotypes` shell-out with charly's own CUE-Go-API generator, loaded through schemacompose's offline module registry, so a plugin schema that imports github.com/opencharly/spec/schema projects to the Go package that owns the shapes (github.com/opencharly/spec/spec). - schemacompose exports ComposeText and Instance, so the ONE compose now feeds the serve side, the host gate and the Go projection from the same module instance and the same offline resolution. - schemaparams/gotypes.go ports gengotypes v0.16.1's mapping in-process. A schema without an import stays byte-identical: the pinned-CLI oracle in the tests compares byte for byte, and the committed spec tree regenerates as a no-op. Doc-comment canonicalization is done locally so Go 1.26 and Go 1.27 emit one form. - schemaparams/gohome.go carries the CUE-package to Go-package relation that closes F2; an import with no registered Go home is a loud error. - cue-gen no longer bootstraps the cue CLI: the CLI leaves the generation leg entirely and remains the pinned authoring tool for `cue eval ./schema`. Generate(dir, pkg) replaces Generate(dir, pkg, cueDir); the consumer call site in opencharly/plugin-candy is the next leg of opencharly/opencharly#950. Closes opencharly/spec#298. Updates opencharly/spec#271. Assisted-by: DeepSeek Harness deepseek-v4.1-flash (fully tested and validated) --- charly.yml | 25 +- cue.mod/module.cue | 5 +- cuetoolchain/cuetoolchain.go | 24 +- internal/cuesetup/main.go | 10 +- internal/schemagen/main.go | 34 +- schema/README.md | 51 +- schemacompose/compose.go | 74 +- schemaconcat/schemaconcat.go | 2 +- schemaparams/gohome.go | 43 + schemaparams/gotypes.go | 913 ++++++++++++++++++ schemaparams/schemaparams.go | 112 +-- schemaparams/schemaparams_test.go | 328 ++++--- .../testdata/import/schema/tunnel.cue | 45 + schemaparams/testdata/twin/schema/tunnel.cue | 72 ++ schemaretag/schemaretag.go | 18 +- spec/gen_repro_test.go | 59 +- 16 files changed, 1434 insertions(+), 381 deletions(-) create mode 100644 schemaparams/gohome.go create mode 100644 schemaparams/gotypes.go create mode 100644 schemaparams/testdata/import/schema/tunnel.cue create mode 100644 schemaparams/testdata/twin/schema/tunnel.cue diff --git a/charly.yml b/charly.yml index 934f1b3..9adb95c 100644 --- a/charly.yml +++ b/charly.yml @@ -23,6 +23,9 @@ repo: github.com/opencharly/spec # ----------------------------------------------------------------------------- # bootstrap-cue — ensure ./bin/cue is the pinned cue CLI (checksum-verified). # Idempotent: a present, correct bin/cue short-circuits without a download. +# NOT part of cue-gen: generation is in-process. The CLI is the pinned tool for +# AUTHORING the schema as a module (`cue eval ./schema`, `cue vet`, a scratch +# `cue exp`), so it stays available without being on the generation leg. # ----------------------------------------------------------------------------- bootstrap-cue: task: @@ -33,8 +36,8 @@ bootstrap-cue: command: | set -euo pipefail # The version, URL, checksum and arch rules live in ONE place — - # internal/cuesetup -> cuetoolchain — shared with the charly-side generation - # verb, so this task and that verb can never run different toolchains. + # internal/cuesetup -> cuetoolchain — so this task and any other consumer + # of the pinned CLI can never run different toolchains. GOWORK=off go run ./internal/cuesetup bin context: [deploy] @@ -112,25 +115,29 @@ proto-gen: # ----------------------------------------------------------------------------- # cue-gen — regenerate every CUE-owned artifact. # -# spec/cue_types_gen.go (cue exp gengotypes — the Go param structs) +# spec/cue_types_gen.go (schemaparams — the Go param structs, generated in-process +# from the CUE Go API; no `cue` CLI in the loop) # spec/vocab_gen.go (internal/schemagen — the vocab lists) # # Reproducible: two consecutive runs produce no diff (TestGenReproducible). +# No bootstrap-cue dependency: the params generator is in-process, so the pinned +# CUE CLI is not part of this task (bootstrap-cue remains for `cue eval ./schema`). # ----------------------------------------------------------------------------- cue-gen: task: description: Regenerate all CUE-owned artifacts, including the protobuf wire contract dir: . - depends_on: [bootstrap-cue, wire-gen] + depends_on: [wire-gen] plan: - - run: gengotypes + schemagen the generated Go + - run: generate the Go params in-process + schemagen the vocab lists command: | set -euo pipefail # 1) The params file comes from the ONE pipeline (schemaparams): concatenate - # schema/*.cue under `package spec` + `@go(spec)`, run the pinned cue - # gengotypes (provisioned checksum-verified into ./bin), normalize the - # struct tags. No step of it is re-implemented here or in schemagen, and - # the charly-side verb calls the same function. + # schema/*.cue under `package spec` + `@go(spec)`, compose them as a CUE + # module instance, generate the Go types in-process with the CUE Go API, + # normalize the struct tags. No step of it is re-implemented here or in + # schemagen, the charly-side verb calls the same function, and no `cue` + # CLI is provisioned: composition and generation share ONE resolution path. GOWORK=off go run ./internal/schemagen -mode=params -schema=schema -pkg=spec -out=spec/cue_types_gen.go # 2) Companion generator -> the vocab lists. GOWORK=off go run ./internal/schemagen -mode=vocab -schema=schema -out=spec/vocab_gen.go diff --git a/cue.mod/module.cue b/cue.mod/module.cue index 283cf5f..938610a 100644 --- a/cue.mod/module.cue +++ b/cue.mod/module.cue @@ -5,8 +5,9 @@ // The language version is PINNED, not incidental: the schema's `close()` + // embedded-definition idiom (schema/00_common.cue's #Op arms) is legal through // v0.17 and rejected by v0.18.0's semantics ("field not allowed"), so v0.16.1 — -// the version spec's own cuetoolchain pins for `cue exp gengotypes` — is the -// version that keeps the package's VALUE identical to the concatenation's. +// the CUE library version this module pins in go.mod, which both the in-process +// params generator and every composing consumer evaluate under — is the version +// that keeps the package's VALUE identical to the concatenation's. // Raising it is the §10.1 pattern rewrite (TODO 2.16), not a bump. module: "github.com/opencharly/spec" diff --git a/cuetoolchain/cuetoolchain.go b/cuetoolchain/cuetoolchain.go index 189d3e3..a7ecb23 100644 --- a/cuetoolchain/cuetoolchain.go +++ b/cuetoolchain/cuetoolchain.go @@ -1,19 +1,19 @@ // Package cuetoolchain is THE single home for the pinned `cue` CLI (R3). // -// The schema→Go pipeline runs `cue exp gengotypes`, and that generator lives in -// `cuelang.org/go`'s INTERNAL tree (`internal/encoding/gotypes`), so no other -// module can call it in-process. The CLI is therefore a toolchain the pipeline -// provisions itself — checksum-verified, from a pinned release — so a reader who -// is told to run a `charly` command never sees `cue`, a checkout path, or a `cd`. +// The schema→Go params pipeline no longer runs this CLI: `schemaparams` composes and +// generates in-process through the CUE Go API (opencharly/spec#271), so generation and +// composition share one offline resolution path and no binary is provisioned. The CLI +// remains the pinned tool for AUTHORING the schema as a module — `cue eval ./schema`, +// `cue vet`, a scratch `cue exp` — provisioned checksum-verified from a pinned release +// by the `bootstrap-cue` task and by the charly-side cache directory helper. // -// The pin AND the provisioning live here, once: the `charly`-side generation verb -// calls Ensure to obtain the toolchain, and spec's own `bootstrap-cue` task is -// re-keyed onto it too (it used to carry the same version, URL and checksum inline -// in shell — two declarations of one toolchain, which is exactly what this package -// removes). One pin, one home, one fetcher. +// The pin AND the provisioning live here, once: `cuetoolchain.Ensure` is what every +// caller uses to obtain the toolchain (the `bootstrap-cue` task used to carry the same +// version, URL and checksum inline in shell — two declarations of one toolchain, which +// is exactly what this package removes). One pin, one home, one fetcher. // // STANDALONE: stdlib only, so both callers can import it without pulling in the -// packages they regenerate. +// packages they generate. package cuetoolchain import ( @@ -97,7 +97,7 @@ func Ensure(dir string) (string, error) { } sum, ok := SHA256(runtime.GOARCH) if !ok { - return "", fmt.Errorf("the pinned cue %s is not checksum-pinned for %s/%s (only linux/amd64 is) — pin it in github.com/opencharly/spec/cuetoolchain, or run the pipeline in a container", + return "", fmt.Errorf("the pinned cue %s is not checksum-pinned for %s/%s (only linux/amd64 is) — pin it in github.com/opencharly/spec/cuetoolchain, or use a linux/amd64 environment", Version, runtime.GOOS, runtime.GOARCH) } url := URL(runtime.GOARCH) diff --git a/internal/cuesetup/main.go b/internal/cuesetup/main.go index 3bff0c3..c636d1c 100644 --- a/internal/cuesetup/main.go +++ b/internal/cuesetup/main.go @@ -1,9 +1,9 @@ // Command cuesetup provisions the pinned `cue` CLI into a directory. It is the -// task-facing entry point to cuetoolchain.Ensure, so `charly task bootstrap-cue` -// and the `charly`-side generation verb (opencharly/charly#829) cannot disagree -// about which toolchain the schema→Go pipeline runs, and so the pin, its URL and -// its checksum exist in exactly one place (cuetoolchain) instead of once in Go and -// once in shell. +// task-facing entry point to cuetoolchain.Ensure, so `charly task bootstrap-cue` and +// any other consumer of the pinned CLI cannot disagree about which toolchain they run, +// and so the pin, its URL and its checksum exist in exactly one place (cuetoolchain) +// instead of once in Go and once in shell. It is NOT on the generation leg: the +// schema→Go params pipeline composes and generates in-process (no CLI). package main import ( diff --git a/internal/schemagen/main.go b/internal/schemagen/main.go index 16fb82f..eaaaf43 100644 --- a/internal/schemagen/main.go +++ b/internal/schemagen/main.go @@ -4,7 +4,8 @@ // // -mode=params -out= Concatenate schema/*.cue into one file // headed `package spec` + the file-level `@go(spec)` -// attribute, ready for `cue exp gengotypes`. +// attribute, and generate the Go types in-process +// through schemaparams (CUE Go API). // -mode=vocab -out= Compile the concatenation via the embedded // cuelang.org/go API and emit spec/vocab_gen.go // — the single-source vocabulary lists (kind keywords, @@ -24,10 +25,10 @@ // PARAM-GEN SCOPE (the two modes diverge by INPUT, not by mechanism): the // `vocab` mode reads the FULL schema (every schema/*.cue) — it needs #Node's // arms (node.cue) to derive KindWords, so its concatenation stays byte-identical -// to the runtime `sharedCueSchema`. The `params` mode (which feeds `cue exp -// gengotypes` → the Go param structs) reads the schema MINUS node.cue: it +// to the runtime `sharedCueSchema`. The `params` mode (which feeds the in-process +// Go generator → the Go param structs) reads the schema MINUS node.cue: it // defines the node-disjunction wrappers (#Node/#NodeDoc/#*Arm/#*Value), which -// gengotypes degrades to `map[string]any` and which are NOT charly param structs. +// the generator degrades to `map[string]any` and which are NOT charly param structs. // (The egress validation schemas moved to candy/plugin-egress in M16, so they no // longer live here to exclude.) The entity defs (#Box/#Deploy/#Op/#Vm/…) do not // reference node.cue, so the exclusion is compile-clean. Both modes share the ONE concatSchema helper @@ -82,9 +83,9 @@ func fatal(format string, a ...any) { } // excludeParamGen reports whether a schema/*.cue file is EXCLUDED from the -// param-gen concatenation (the gengotypes input). node.cue defines the -// node-disjunction wrappers — not a charly param struct, and gengotypes degrades -// it to `map[string]any`. See the package doc PARAM-GEN SCOPE note. The vocab +// param-gen concatenation (the generator input). node.cue defines the +// node-disjunction wrappers — not a charly param struct, and the generator +// degrades it to `map[string]any`. See the package doc PARAM-GEN SCOPE note. The vocab // mode passes `nil` (no exclusion) so #Node's arms are present for KindWords. // (The egress validation schemas moved to candy/plugin-egress in M16, so there // are no egress_*.cue files here to exclude.) @@ -106,8 +107,8 @@ func concatSchema(dir string, exclude func(name string) bool) (string, []string, } // specSource returns the concatenation headed with the `package ` clause and -// the file-level `@go()` attribute — what `cue exp gengotypes` consumes to -// emit that Go package. pkg is "spec" for the core schema and "params" for a +// the file-level `@go()` attribute — what the generator reads to name the +// Go package it emits. pkg is "spec" for the core schema and "params" for a // plugin's self-contained schema (same one concatenation contract — R3). The cue // API (vocab mode) compiles the same string. The exclude filter scopes the // param-gen input (see excludeParamGen). @@ -120,18 +121,19 @@ func specSource(dir, pkg string, exclude func(name string) bool) (string, error) } // ---------------------------------------------------------------------------- -// retag mode — the Go-native yaml-tag doubling (replaces the former cue:gen sed) +// params mode — the ONE schema→Go pipeline (concatenation, module-aware compose, +// in-process generation, tag normalization) // ---------------------------------------------------------------------------- // writeParams emits the GENERATED Go params file by calling the ONE pipeline // (schemaparams): concatenate the param-gen schema subset under the `package ` + -// `@go()` header, run the pinned cue gengotypes (provisioned through -// cuetoolchain.Ensure into ./bin), normalize the struct tags. None of those steps is -// implemented here any more — that is what makes the pipeline ONE thing rather than a -// copy per caller. `excludeParamGen` (node.cue) is this caller's only difference, and -// it is a parameter. +// `@go()` header, compose it as a CUE module instance, generate the Go types +// in-process (CUE Go API, no `cue` CLI), normalize the struct tags. None of those steps +// is implemented here any more — that is what makes the pipeline ONE thing rather than +// a copy per caller. `excludeParamGen` (node.cue) is this caller's only difference, +// and it is a parameter. func writeParams(dir, out, pkg string) error { - src, err := schemaparams.GenerateExcluding(dir, pkg, "bin", excludeParamGen) + src, err := schemaparams.GenerateExcluding(dir, pkg, excludeParamGen) if err != nil { return err } diff --git a/schema/README.md b/schema/README.md index 7aa3368..bfde989 100644 --- a/schema/README.md +++ b/schema/README.md @@ -28,8 +28,8 @@ are the single source of truth for these consumers: > package load, which is why the shared definitions live in `00_common.cue`: the > `00_` sorts first, keeping the concatenation's order — and so the generated > Go — byte-identical.) The `package spec` clause + the file-level `@go(spec)` -> attribute that `cue exp gengotypes` needs are injected by the gen pipeline -> (`internal/schemagen -mode=params`), NEVER written into the source. +> attribute that the generator reads are injected by the gen pipeline +> (`internal/schemagen -mode=params` → `schemaparams`), NEVER written into the source. ## The package form @@ -47,11 +47,13 @@ the module that owns it is `specmodule.FS` (package `github.com/opencharly/spec` Composing with only the files would evaluate them under the consumer's toolchain default instead of the pinned version above. -`cue.mod/module.cue` pins language version `v0.16.1` — the version spec's own -`cuetoolchain` pins for `cue exp gengotypes`. It is not incidental: the schema's -`close()` + embedded-definition idiom (`#Step`'s arms embedding the closed `#Op`) -is legal through v0.17 and rejected by v0.18's semantics with `field not allowed`. -Raising the language version is the §10.1 pattern rewrite (TODO 2.16), not a bump. +`cue.mod/module.cue` pins language version `v0.16.1` — the version the embedded +`cuelang.org/go` library is pinned to in `go.mod`, and the ONE declaration both the +in-process generator and every composing consumer evaluate under. It is not +incidental: the schema's `close()` + embedded-definition idiom (`#Step`'s arms +embedding the closed `#Op`) is legal through v0.17 and rejected by v0.18's semantics +with `field not allowed`. Raising the language version is the §10.1 pattern rewrite +(TODO 2.16), not a bump. ## Regenerating `spec` — `charly task cue-gen` @@ -62,31 +64,32 @@ charly task cue-gen ``` The task is `cue-gen` in this repo's `charly.yml` (the `kind: task` surface). It -depends on `bootstrap-cue` + `wire-gen` (which in turn depends on -`bootstrap-protoc`): - -1. **Bootstraps the pinned cue CLI** into `./bin/cue` (gitignored) — `v0.16.1`, - the SAME version as charly's embedded `cuelang.org/go` library, so the CLI that - runs `gengotypes` matches the library that compiles the schema. It downloads - the checksum-pinned release tarball (a no-op once `./bin/cue` is at the - pinned version). -2. **Concatenates** `schema/*.cue` via `internal/schemagen -mode=params` +depends on `wire-gen` (which in turn depends on `bootstrap-protoc`): + +1. **Concatenates** `schema/*.cue` via `internal/schemagen -mode=params` (the SAME `schemaconcat.ConcatSchema` contract as the runtime `sharedCueSchema` — one concatenation contract; the per-file `package schema` clause is stripped), headed `package spec` + `@go(spec)`. -3. **`cue exp gengotypes`** → `spec/cue_types_gen.go` (the Go param structs). -4. **`internal/schemagen -mode=vocab`** → `spec/vocab_gen.go` +2. **Composes and generates in-process** (`schemaparams`): the concatenation is + loaded as ONE CUE module instance through `schemacompose` (the same offline + resolution a plugin's served schema uses), and the Go param structs are + generated from the CUE Go API → `spec/cue_types_gen.go`. No `cue` CLI is + provisioned or executed. +3. **`internal/schemagen -mode=vocab`** → `spec/vocab_gen.go` (`KindWords`, `DocDirectives`, `StepKeywords`, `ContextWords`, `OpFields`, `OpVerbs`, `AuthoringVerbs`, … — all read straight from the compiled schema via the cue API). -5. **`gofmt`** the committed generated files. +4. **`gofmt`** the committed generated files. `wire-gen` (chained first) additionally renders `protocol/schema/*.cue` into `proto/plugin.proto` and compiles the Go gRPC stubs. -Each plugin's OWN `params/cue_types_gen.go` is generated from its self-contained +Each plugin's OWN `params/cue_types_gen.go` is generated from its own `candy/plugin-*/schema/*.cue` in the plugin's OWN repository, via the SAME -`internal/schemagen` + pinned cue pipeline (R3). The former superproject +`schemaparams` pipeline (R3) — including a schema that imports +`github.com/opencharly/spec/schema`, which resolves through the same offline +compose and emits the type through the Go package that owns it +(`github.com/opencharly/spec/spec`). The former superproject `taskfiles/Cue.yml` wrapper — which chained this repo's `cue:gen` and then looped over each plugin — was removed with the candy de-submodule work (charly#452, merged 2026-08-29); plugin param generation now lives with the plugin. @@ -103,11 +106,13 @@ the `*_gen.go` files** — change the `*.cue` source and regenerate. ## The `@go(...)` annotations -`cue exp gengotypes` is lossy: every CUE disjunction degrades to `any` / +The params generator is lossy: every CUE disjunction degrades to `any` / `map[string]any` / an empty struct (Go cannot express a disjunction), and field names get a naive `leadingCap + underscores-preserved` mapping. Two annotation classes in the `*.cue` source fix this — both are **inert for runtime validation** -(CUE attributes never participate in unification): +(CUE attributes never participate in unification). Schema comments that name +`gengotypes` describe this mapping: the generator is a faithful in-process port of +`cue exp gengotypes` v0.16.1 (`schemaparams/gotypes.go`), so each statement still holds: - **Field renames** — `@go()` makes the generated Go field name match the hand-written charly param struct (`http` → `HTTP`, `cap_add` → `CapAdd`, diff --git a/schemacompose/compose.go b/schemacompose/compose.go index 832eb9e..f8f315c 100644 --- a/schemacompose/compose.go +++ b/schemacompose/compose.go @@ -10,8 +10,8 @@ package schemacompose // contract — the serve side and the host gate — must judge the same schema with the same // code, so the ONE composer has to live where both of them may reach it. This module also // already owns the family it belongs to: schemaconcat (the concatenation contract), -// schemaparams and cuetoolchain (the generation pipeline), and the embedded schema source -// plus the module declaration (`schema.FS`, `FS`). +// schemaparams (the generation pipeline), cuetoolchain (the pinned authoring CLI), and the +// embedded schema source plus the module declaration (`schema.FS`, `FS`). // // A plugin's served schema used to be TEXT all the way down: `CompileString` on the // concatenated `schema/*.cue` body, in a context with no CUE module in scope. That made @@ -28,12 +28,14 @@ package schemacompose // no `cue.mod/pkg` vendoring, and no local `replace` (which the pinned modfile schema does // not have). // -// TWO entry points, ONE compose. `ServedSchema` composes the plugin's schema FS (the -// serve side, `BuildCapabilitiesWithRequires`), and `ServedText` composes the -// package-less body that reaches the host over the wire (`Capabilities.schema_cue`, the -// host gate). Both funnel into composeServedBody, so the two halves of the contract judge -// the same schema with the same code — which is the property sdk#490 asks for: a builtin -// and an out-of-process plugin, and the serve side and the host, agree byte for byte. +// THREE entry points, ONE compose. `ServedSchema` composes the plugin's schema FS (the +// serve side, `BuildCapabilitiesWithRequires`), `ServedText` composes the package-less body +// that reaches the host over the wire (`Capabilities.schema_cue`, the host gate), and +// `ComposeText` hands the same loaded instance to the Go projection (`schemaparams`), so a +// plugin's params are generated from the schema text both halves of the contract judge. +// All three funnel into composeServedBody, so the two halves of the contract judge the same +// schema with the same code — which is the property sdk#490 asks for: a builtin and an +// out-of-process plugin, and the serve side and the host, agree byte for byte. // // THREE measured constraints decide the shape (sdk#490, measured live): // - the loader requires a dependency's source FS to implement `module.OSRootFS`, because @@ -61,6 +63,7 @@ import ( "sync" "cuelang.org/go/cue" + "cuelang.org/go/cue/build" "cuelang.org/go/cue/cuecontext" "cuelang.org/go/cue/load" "cuelang.org/go/mod/modfile" @@ -112,6 +115,16 @@ const ( derivedDepVersion = "v0.0.0" ) +// Instance is ONE composed CUE module instance: the context its value belongs to, the +// loader's build instance and the built value. The two halves of the contract read the +// VALUE; the Go projection reads the build instance as well, because the import graph and +// the package attributes live there and not in the value. +type Instance struct { + Ctx *cue.Context + Build *build.Instance + Value cue.Value +} + // ServedSchema composes a plugin's embedded schema as one CUE module instance: the // concatenated `schema/*.cue` body of schemaFS under dir, evaluated inside the plugin's own // module declaration when its FS ships `cue.mod/module.cue`, and under the derived @@ -126,7 +139,11 @@ func ServedSchema(ctx *cue.Context, schemaFS fs.FS, dir string) (cue.Value, erro if err != nil { return cue.Value{}, err } - return composeServedBody(ctx, body, decl) + inst, err := composeServedBody(ctx, body, pluginPackage, decl) + if err != nil { + return cue.Value{}, err + } + return inst.Value, nil } // ServedText composes the package-less body a plugin serves on @@ -137,27 +154,45 @@ func ServedSchema(ctx *cue.Context, schemaFS fs.FS, dir string) (cue.Value, erro // derive. For a plugin that ships no declaration — every plugin today — the two entry // points are byte-for-byte the same compose. func ServedText(ctx *cue.Context, body string) (cue.Value, error) { - decl, err := derivedPluginModuleDecl() + inst, err := ComposeText(ctx, body, pluginPackage) if err != nil { return cue.Value{}, err } - return composeServedBody(ctx, body, decl) + return inst.Value, nil +} + +// ComposeText composes a package-less schema body under `package pkg`, in the virtual +// plugin module, under the DERIVED declaration, with spec served by the offline registry. +// It is the same compose as ServedText and is exported for the Go projection +// (schemaparams): a plugin's params are generated from the SAME source text the two halves +// of the contract judge, which is what lets that source carry +// `import "github.com/opencharly/spec/schema"` — the import resolves against the same +// offline registry, with no published CUE module and no `cue` CLI in the loop. +func ComposeText(ctx *cue.Context, body, pkg string) (*Instance, error) { + decl, err := derivedPluginModuleDecl() + if err != nil { + return nil, err + } + return composeServedBody(ctx, body, pkg, decl) } // composeServedBody is the ONE compose: the package-less schema body plus the module // declaration it is evaluated under, loaded as ./schema out of the virtual plugin module, // with spec served by the offline registry. -func composeServedBody(ctx *cue.Context, body string, decl []byte) (cue.Value, error) { +func composeServedBody(ctx *cue.Context, body, pkg string, decl []byte) (*Instance, error) { if ctx == nil { ctx = cuecontext.New() } loc, err := specModuleLoc() if err != nil { - return cue.Value{}, err + return nil, err + } + if pkg == "" { + return nil, fmt.Errorf("compose: the synthesized package name is empty") } overlay := map[string]load.Source{ filepath.Join(pluginModuleRoot, filepath.FromSlash(pluginModuleFile)): load.FromBytes(decl), - filepath.Join(pluginModuleRoot, pluginSchemaDir, "plugin.cue"): load.FromBytes([]byte("package " + pluginPackage + "\n\n" + body)), + filepath.Join(pluginModuleRoot, pluginSchemaDir, "plugin.cue"): load.FromBytes([]byte("package " + pkg + "\n\n" + body)), } instances := load.Instances([]string{"./" + pluginSchemaDir}, &load.Config{ Dir: pluginModuleRoot, @@ -165,17 +200,20 @@ func composeServedBody(ctx *cue.Context, body string, decl []byte) (cue.Value, e Registry: specRegistry{loc: loc}, }) if len(instances) != 1 { - return cue.Value{}, fmt.Errorf("composing ./%s from a plugin module produced %d instances, want 1", pluginSchemaDir, len(instances)) + return nil, fmt.Errorf("composing ./%s from a plugin module produced %d instances, want 1", pluginSchemaDir, len(instances)) } inst := instances[0] if err := inst.Err; err != nil { - return cue.Value{}, err + return nil, err } if inst.Module == "" || inst.ModuleFile == nil { - return cue.Value{}, fmt.Errorf("the plugin's ./%s loaded OUTSIDE its CUE module (module %q): the module's language.version must govern evaluation", pluginSchemaDir, inst.Module) + return nil, fmt.Errorf("the plugin's ./%s loaded OUTSIDE its CUE module (module %q): the module's language.version must govern evaluation", pluginSchemaDir, inst.Module) } v := ctx.BuildInstance(inst) - return v, v.Err() + if err := v.Err(); err != nil { + return nil, err + } + return &Instance{Ctx: ctx, Build: inst, Value: v}, nil } // pluginModuleDecl returns the declaration a plugin's schema is composed under. A plugin diff --git a/schemaconcat/schemaconcat.go b/schemaconcat/schemaconcat.go index b0800b5..301cae2 100644 --- a/schemaconcat/schemaconcat.go +++ b/schemaconcat/schemaconcat.go @@ -5,7 +5,7 @@ // file into ONE package-less compilation unit. They used to carry two // byte-identical-by-comment copies of the same loop; this is the ONE copy they // both call, so the compiled schema (what the runtime validates against) and -// the generated Go types (`cue exp gengotypes`) can never drift. +// the generated Go types (`schemaparams`) can never drift. // // The schema files are ALSO a real CUE package in the `github.com/opencharly/spec` // module (`cue.mod/module.cue`; `package schema`, loaded by `cue eval ./schema`). diff --git a/schemaparams/gohome.go b/schemaparams/gohome.go new file mode 100644 index 0000000..8e15347 --- /dev/null +++ b/schemaparams/gohome.go @@ -0,0 +1,43 @@ +package schemaparams + +import ( + "sort" + "strings" +) + +// goHome names the Go package that owns a CUE package's generated shapes. +type goHome struct { + ImportPath string + Package string +} + +// goHomes is THE relation between a CUE package a schema may import and the Go package +// that owns its shapes. It cannot be derived: CUE import paths and Go import paths are two +// namespaces, and a schema's CUE package is the SOURCE of shapes whose generated Go lives in +// a different package. spec's CUE package `github.com/opencharly/spec/schema` is the source; +// its generated params (`spec/cue_types_gen.go`, package spec) live in +// `github.com/opencharly/spec/spec`. Emitting the CUE path as a Go import names a package +// that exports only FS, so the referenced type is undefined (opencharly/spec#298, F2). +// +// ONE entry today: spec's schema is the only CUE package the compose serves offline (a +// plugin's module can import nothing else — schemacompose's registry refuses every other +// path), so it is the only package a projection can reference across an import. A package +// that gains a Go home is an edit HERE, and the projection's test compiles a generated file +// against it. +var goHomes = map[string]goHome{ + "github.com/opencharly/spec/schema": { + ImportPath: "github.com/opencharly/spec/spec", + Package: "spec", + }, +} + +// registeredGoHomes lists the relations, for an error that tells the reader where to add +// one. +func registeredGoHomes() string { + out := make([]string, 0, len(goHomes)) + for cuePath, home := range goHomes { + out = append(out, cuePath+" => "+home.ImportPath) + } + sort.Strings(out) + return strings.Join(out, ", ") +} diff --git a/schemaparams/gotypes.go b/schemaparams/gotypes.go new file mode 100644 index 0000000..eca1b79 --- /dev/null +++ b/schemaparams/gotypes.go @@ -0,0 +1,913 @@ +package schemaparams + +// gotypes.go — charly's own CUE-API Go type generator for schema params. +// +// The mapping of definitions, docs, optional fields, @go annotations, nillability, the +// generated-file banner and the final gofmt is a faithful port of +// cuelang.org/go v0.16.1's internal/encoding/gotypes (Copyright 2024 CUE Authors, Apache +// License 2.0), reproduced here because that package is INTERNAL to cuelang.org/go and no +// other module can import it — which is why the pipeline used to shell out to the `cue` +// CLI. In-process, the projection can use schemacompose's offline module resolution, so a +// plugin's schema may import the contract module (opencharly/spec#271, Direction 1). +// +// FAITHFUL BY CONSTRUCTION. Nothing about the emitted Go changes for a schema WITHOUT an +// import: schemaparams_test.go compares this generator's output byte for byte against the +// pinned `cue exp gengotypes` on the same source. The one thing not delegated to the local +// `go/format` is doc-comment canonicalization (formatDocComment): Go 1.27's printer stopped +// rewriting a struct field's doc comment, so leaving it to the toolchain would make the same +// schema regenerate differently on a 1.27 box and in the 1.26 CI. Two deliberate differences +// exist only for the import case: +// +// - an imported CUE package is NOT generated into a sibling Go package of its own; it is +// REFERENCED through the Go package that owns its shapes (gohome.go), because the +// consumer's module already contains that package and a generated twin would never be +// compiled; +// - an imported package with no registered Go home is a loud error, never a reference to a +// Go path that may not compile. + +import ( + "bytes" + "fmt" + goast "go/ast" + "go/doc/comment" + goformat "go/format" + goparser "go/parser" + goscanner "go/scanner" + gotoken "go/token" + "sort" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/ast" + "cuelang.org/go/cue/build" + + "github.com/opencharly/spec/schemacompose" +) + +// generateGo renders the Go source for the definitions of ONE composed schema instance. +// Imported CUE packages are referenced, not generated (see goImportFor). +func generateGo(inst *schemacompose.Instance) ([]byte, error) { + pkg := inst.Build + root := inst.Value + if err := root.Validate(); err != nil { + return nil, err + } + g := &generator{ + generatedTypes: make(map[string]*generatedDef), + pkg: pkg, + goImports: make(map[string]goImport), + exprImports: make(map[string]string), + pkgRoot: root, + } + aliases, err := gatherImportAliases(pkg) + if err != nil { + return nil, err + } + g.importAliases = aliases + + iter, err := root.Fields(cue.Definitions(true)) + if err != nil { + return nil, err + } + for iter.Next() { + sel := iter.Selector() + if !sel.IsDefinition() { + continue + } + path := cue.MakePath(sel) + if _, err := g.genDef(path, iter.Value()); err != nil { + return nil, err + } + } + + var buf []byte + printf := func(format string, args ...any) { + buf = fmt.Appendf(buf, format, args...) + } + // The banner is preserved verbatim: it is the generated-file marker the committed + // params files already carry, and the no-import output must stay byte-identical. + printf("// Code generated by \"cue exp gengotypes\"; DO NOT EDIT.\n\n") + printf("package %s\n\n", goPkgNameForInstance(pkg, root, nil)) + renderImports(printf, g) + for _, path := range g.emitDefs { + qpath := g.qualifiedPath(path) + + val := root.LookupPath(path) + goName := goNameFromPath(path, true) + if goName == "" { + return nil, fmt.Errorf("unexpected path in emitDefs: %q", qpath) + } + goAttr := goValueAttr(val) + if s, _ := goAttr.String(0); s != "" { + if s == "-" { + continue + } + goName = s + } + + emitDocs(printf, val.Doc()) + printf("type %s ", goName) + + // As we grab the generated source, do some sanity checks too. + gen, ok := g.generatedTypes[qpath] + if !ok { + return nil, fmt.Errorf("expected type in generatedTypes: %q", qpath) + } + if gen.inProgress { + return nil, fmt.Errorf("unexpected in-progress type in generatedTypes: %q", qpath) + } + if len(gen.src) == 0 { + return nil, fmt.Errorf("unexpected empty type in generatedTypes: %q", qpath) + } + buf = append(buf, gen.src...) + printf("\n\n") + } + + formatted, err := goformat.Source(buf) + if err != nil { + return nil, fmt.Errorf("formatting generated Go: %w\n%s", err, numberedLines(buf)) + } + return formatted, nil +} + +// numberedLines renders generated source with line numbers, the way gengotypes reports a +// formatting failure (it means the generator itself is buggy). +func numberedLines(buf []byte) string { + var b strings.Builder + for i, line := range bytes.Split(buf, []byte("\n")) { + fmt.Fprintf(&b, "% 4d: %s\n", i+1, line) + } + return b.String() +} + +// goImport describes how ONE imported CUE package appears in the generated Go file. +type goImport struct { + Path string // the Go import path + Package string // the imported Go package's declared name + Qualifier string // the name the generated source uses for it +} + +// generator holds the state for generating Go code for one CUE package instance. +type generator struct { + // generatedTypes records CUE definitions which we have analyzed and translated + // to Go type expressions. + generatedTypes map[string]*generatedDef + + // pkg is the package instance being generated. + pkg *build.Instance + + // emitDefs records paths for the definitions we should emit as Go types. + emitDefs []cue.Path + + // goImports records the imported CUE packages this file references, keyed by the + // package's CUE import path. The value names the Go package that owns its shapes — + // the CUE path and the Go path are not the same namespace (gohome.go). + goImports map[string]goImport + + // exprImports records Go imports named by an @go(,type=...) expression, keyed by Go + // import path. Those are already Go paths, so no home lookup applies. + exprImports map[string]string + + // pkgRoot is the root value of the CUE package, necessary to tell if a referenced + // value belongs to the current package or not. + pkgRoot cue.Value + + // importAliases maps a CUE import path to the explicit alias a file gives it. + importAliases map[string]string + + // def tracks the generation state for a single CUE definition. + def *generatedDef +} + +type qualifiedPath = string // [build.Instance.ImportPath] + " " + [cue.Path.String] + +func (g *generator) qualifiedPath(path cue.Path) qualifiedPath { + return g.pkg.ImportPath + " " + path.String() +} + +// generatedDef holds information about a Go type generated for a CUE definition. +type generatedDef struct { + // inProgress helps detect cyclic definitions and prevents emitting any Go source + // before we are done analyzing and generating the relevant types. + inProgress bool + + // facts records useful information about the generated type. + facts typeFacts + + // src is the generated Go type expression source. + src []byte +} + +// typeFacts holds useful information about a generated Go type. +type typeFacts struct { + // isTypeOverride records whether the generated type came from a @go(,type=expr). + isTypeOverride bool + + // isNillable records whether the generated Go type can be compared to nil, such + // that @go(,optional=nillable) can avoid wrapping it in a pointer. + isNillable bool +} + +func (g *generatedDef) printf(format string, args ...any) { + if !g.inProgress { + panic("generatedDef.printf called when inProgress is false") + } + g.src = fmt.Appendf(g.src, format, args...) +} + +type optionalStrategy int + +const ( + _ optionalStrategy = iota + // optional=zero (default); emit the Go type as-is and rely on the zero value. + optionalZero + // optional=nillable; emit the Go type with a pointer unless it can already + // be compared to nil. + optionalNillable +) + +// genDef analyzes and generates a CUE definition as a Go type, +// adding it to generatedTypes as well as to emitDefs. +func (g *generator) genDef(path cue.Path, val cue.Value) (*generatedDef, error) { + qpath := g.qualifiedPath(path) + if def, ok := g.generatedTypes[qpath]; ok { + return def, nil // already done or in progress + } + g.emitDefs = append(g.emitDefs, path) + + parentDef := g.def + def := &generatedDef{inProgress: true} + g.def = def + g.generatedTypes[qpath] = def + facts, err := g.emitType(val, optionalZero) + if err != nil { + return nil, err + } + g.def.facts = facts + g.def.inProgress = false + g.def = parentDef + return def, nil +} + +// emitType generates a CUE value as a Go type. +// When possible, the Go type is emitted in the form of a reference. +// Otherwise, an inline Go type expression is used. +func (g *generator) emitType(val cue.Value, optionalStg optionalStrategy) (typeFacts, error) { + var facts typeFacts + goAttr := goValueAttr(val) + // We prefer the form @go(Name,type=pkg.Baz) as it is explicit and extensible, + // but we are also backwards compatible with @go(Name,pkg.Baz) as emitted by `cue get go`. + // Make sure that we don't mistake @go(,foo=bar) for a type though. + attrType, _, _ := goAttr.Lookup(1, "type") + if attrType == "" { + if s, _ := goAttr.String(1); !strings.Contains(s, "=") { + attrType = s + } + } + if attrType != "" { + fset := gotoken.NewFileSet() + expr, importedByName, err := parseTypeExpr(fset, attrType) + if err != nil { + return facts, fmt.Errorf("cannot parse @go type expression: %w", err) + } + for _, pkgPath := range importedByName { + g.exprImports[pkgPath] = goImportQualifier(pkgPath) + } + collectSelectorImports(expr, importedByName, func(name string) { + g.exprImports[name] = name + }) + var buf bytes.Buffer + // We emit in plaintext, so format the parsed Go expression and print it out. + if err := goformat.Node(&buf, fset, expr); err != nil { + return facts, err + } + g.def.printf("%s", buf.Bytes()) + facts.isTypeOverride = true + return facts, nil + } + + // Note that type references don't get optionalStg, + // as @go(,optional=) only affects fields under the current type expression. + if done, facts, err := g.emitTypeReference(val); err != nil { + return typeFacts{}, err + } else if done { + return facts, nil + } + + // Inline types are below. + + switch k := val.IncompleteKind(); k { + case cue.StructKind: + return g.emitStructKind(val, optionalStg) + case cue.ListKind: + // We mainly care about patterns like [...string]. + facts.isNillable = true // slices can be nil + g.def.printf("[]") + elem := val.LookupPath(cue.MakePath(cue.AnyIndex)) + if !elem.Exists() { + g.def.printf("any /* CUE closed list */") + } else if _, err := g.emitType(elem, optionalStg); err != nil { + return facts, err + } + + case cue.NullKind: + facts.isNillable = true // pointers can be nil + g.def.printf("*struct{} /* CUE null */") + case cue.BoolKind: + g.def.printf("bool") + case cue.IntKind: + g.def.printf("int64") + case cue.FloatKind: + g.def.printf("float64") + case cue.StringKind: + g.def.printf("string") + case cue.BytesKind: + facts.isNillable = true // slices can be nil + g.def.printf("[]byte") + + case cue.NumberKind: + facts.isNillable = true // interfaces can be nil + g.def.printf("any /* CUE number; int64 or float64 */") + + case cue.TopKind: + facts.isNillable = true // interfaces can be nil + g.def.printf("any /* CUE top */") + + default: + // A disjunction of various kinds cannot be represented in Go. + if op, _ := val.Expr(); op == cue.OrOp { + facts.isNillable = true // interfaces can be nil + g.def.printf("any /* CUE disjunction: %s */", k) + break + } + facts.isNillable = true // interfaces can be nil + g.def.printf("any /* TODO: IncompleteKind: %s */", k) + } + return facts, nil +} + +// emitStructKind emits a CUE struct as a Go map, a `map[string]any` fallback for a +// disjunction of structs, or a struct with one field per CUE field. Extracted from +// emitType so each function stays under the complexity gate; the emitted bytes are +// unchanged. +func (g *generator) emitStructKind(val cue.Value, optionalStg optionalStrategy) (typeFacts, error) { + var facts typeFacts + if elem := val.LookupPath(cue.MakePath(cue.AnyString)); elem.Err() == nil { + facts.isNillable = true // maps can be nil + g.def.printf("map[string]") + if _, err := g.emitType(elem, optionalStg); err != nil { + return facts, err + } + return facts, nil + } + // A disjunction of structs cannot be represented in Go, as it does not have sum types. + // Fall back to a map of string to any. + if op, _ := val.Expr(); op == cue.OrOp { + facts.isNillable = true // maps can be nil + g.def.printf("map[string]any") + return facts, nil + } + g.def.printf("struct {\n") + iter, err := val.Fields(cue.Definitions(true), cue.Optional(true)) + if err != nil { + return facts, err + } + for iter.Next() { + sel := iter.Selector() + fieldVal := iter.Value() + if sel.IsDefinition() { + continue + } + cueName := sel.String() + if sel.IsString() { + cueName = sel.Unquoted() + } + cueName = strings.TrimRight(cueName, "?!") + emitDocs(g.def.printf, fieldVal.Doc()) + + // We want the Go name from just this selector, even when it's not a definition. + goName := goNameFromPath(cue.MakePath(sel), false) + + goAttr := fieldVal.Attribute("go") + if s, _ := goAttr.String(0); s != "" { + if s == "-" { + continue + } + goName = s + } + + optional := sel.ConstraintType()&cue.OptionalConstraint != 0 + fieldStg := optionalStg // only for this field + + switch s, ok, _ := goAttr.Lookup(1, "optional"); s { + case "zero": + fieldStg = optionalZero + case "nillable": + fieldStg = optionalNillable + default: + if ok { + return facts, fmt.Errorf("unknown optional strategy %q", s) + } + } + + // Fields whose names encoding/json cannot express are skipped with a note. + if strings.ContainsAny(cueName, "\\\"`,\n") { + g.def.printf("// CUE field %q: encoding/json does not support this field name\n\n", cueName) + continue + } + g.def.printf("%s ", goName) + + // Pointers in Go are a prefix in the syntax, but we won't find out the generated type facts + // until we have emitted its Go source, which we do into the same buffer to avoid copies. + // Luckily, since a pointer is always one byte, and we gofmt the result anyway for nice formatting, + // we can add the pointer first and replace it with whitespace later if not wanted. + ptrOffset := len(g.def.src) + g.def.printf("*") + + fieldFacts, err := g.emitType(fieldVal, fieldStg) + if err != nil { + return facts, err + } + if !usePointer(fieldFacts, optional, fieldStg) { + g.def.src[ptrOffset] = ' ' + } + + omitEmpty := "" + if optional { + omitEmpty = ",omitempty" + } + g.def.printf(" `json:\"%s%s\"`", cueName, omitEmpty) + g.def.printf("\n\n") + } + g.def.printf("}") + return facts, nil +} + +func usePointer(facts typeFacts, optional bool, strategy optionalStrategy) bool { + if facts.isTypeOverride { + // @(,type=) overrides any @(,optional=) setting + return false + } + if !optional { + // Regular and required fields never use pointers. + return false + } + switch strategy { + case optionalZero: + return false + case optionalNillable: + // Only use a pointer when the type isn't already nillable. + return !facts.isNillable + default: + panic("unreachable") + } +} + +// parseTypeExpr extends goparser.ParseExpr to allow selecting from full import paths. +// Accept `[]"go/constant".Kind` with a pre-processing step to find quoted strings, +// record them as imports keyed by package name in the returned map, and rewrite the Go +// expression to be in terms of the imported package. +func parseTypeExpr(fset *gotoken.FileSet, src string) (goast.Expr, map[string]string, error) { + var goSrc strings.Builder + importedByName := make(map[string]string) + + var scan goscanner.Scanner + scan.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), nil, 0) + lastStringLit := "" + for { + _, tok, lit := scan.Scan() + if tok == gotoken.EOF { + break + } + if lastStringLit != "" { + if tok == gotoken.PERIOD { + imp, err := strconv.Unquote(lastStringLit) + if err != nil { + panic(err) // should never happen + } + // We assume the package name is the last path component. + impName := lastPathElement(imp) + importedByName[impName] = imp + goSrc.WriteString(impName) + } else { + goSrc.WriteString(lastStringLit) + } + lastStringLit = "" + } + switch tok { + case gotoken.STRING: + lastStringLit = lit + case gotoken.IDENT, gotoken.INT, gotoken.FLOAT, gotoken.IMAG, gotoken.CHAR: + goSrc.WriteString(lit) + case gotoken.SEMICOLON: + // EOF inserts a semicolon, which breaks goparser.ParseExpr. + if lit == "\n" { + break // inserted semicolon at EOF + } + fallthrough + default: + goSrc.WriteString(tok.String()) + } + } + expr, err := goparser.ParseExpr(goSrc.String()) + return expr, importedByName, err +} + +// collectSelectorImports records the single-element package an unquoted selector names +// (`time.Duration`), skipping any package the quoted-path pre-pass already imported. +// It is the upstream astutil.Apply pass over the parsed expression, without the +// golang.org/x/tools dependency. +func collectSelectorImports(expr goast.Expr, importedByName map[string]string, add func(pkgPath string)) { + goast.Inspect(expr, func(n goast.Node) bool { + sel, ok := n.(*goast.SelectorExpr) + if !ok { + return true + } + id, ok := sel.X.(*goast.Ident) + if !ok { + return true + } + if importedByName[id.Name] != "" { + return true + } + add(id.Name) + return true + }) +} + +// lastPathElement returns the part of a slash-separated path after its last "/", or the +// whole string when it has none. +func lastPathElement(s string) string { + if i := strings.LastIndex(s, "/"); i >= 0 { + return s[i+1:] + } + return s +} + +// goNameFromPath transforms a CUE path, such as "#foo.bar?", into a suitable name for +// a generated Go type, such as "Foo_bar". When defsOnly is true, all path elements must +// be definitions, or "" is returned. +func goNameFromPath(path cue.Path, defsOnly bool) string { + export := true + var sb strings.Builder + for i, sel := range path.Selectors() { + if defsOnly && !sel.IsDefinition() { + return "" + } + if i > 0 { + // To aid in readability, nested names are separated with underscores. + sb.WriteString("_") + } + str := sel.String() + if sel.IsString() { + str = sel.Unquoted() + } + str, hidden := strings.CutPrefix(str, "_") + if hidden { + // If any part of the path is hidden, we are not exporting. + export = false + } + // Leading or trailing characters for definitions, optional, or required + // are not included as part of Go names. + str = strings.TrimPrefix(str, "#") + str = strings.TrimRight(str, "?!") + // CUE allows quoted field names such as "foo-bar" or "123baz", none of which are + // valid Go identifiers: replace forbidden characters with underscores, adding a + // leading "F" if the name begins with a digit. + for i, r := range str { + switch { + case unicode.IsLetter(r): + sb.WriteRune(r) + case unicode.IsDigit(r): + if i == 0 { + sb.WriteRune('F') + } + sb.WriteRune(r) + default: + sb.WriteRune('_') + } + } + } + name := sb.String() + if export { + // Capitalize the first letter to export the name in Go. + first, size := utf8.DecodeRuneInString(name) + name = string(unicode.ToTitle(first)) + name[size:] + } + return name +} + +// goValueAttr is like cue.Value.Attribute with the string parameter "go", but it +// supports DeclAttr attributes as well and not just FieldAttr. +func goValueAttr(val cue.Value) cue.Attribute { + attrs := val.Attributes(cue.ValueAttr) + for _, attr := range attrs { + if attr.Name() == "go" { + return attr + } + } + return cue.Attribute{} +} + +// goPkgNameForInstance determines what to name a Go package generated from a CUE +// instance. By default this is the CUE package name, but it can be overridden by a @go() +// package attribute. When supplying importAliases, and if no package attribute is found, +// the returned package name reflects the alias name that the package is being imported as. +func goPkgNameForInstance(inst *build.Instance, instVal cue.Value, importAliases map[string]string) string { + attr := goValueAttr(instVal) + if s, _ := attr.String(0); s != "" { + return s + } + if alias, ok := importAliases[inst.ImportPath]; ok { + return alias + } + return inst.PkgName +} + +// emitTypeReference attempts to generate a CUE value as a Go type via a reference, +// either to a type in the same Go package, or to a type in an imported package. +func (g *generator) emitTypeReference(val cue.Value) (bool, typeFacts, error) { + root, path := val.ReferencePath() + if len(path.Selectors()) == 0 { + return false, typeFacts{}, nil + } + inst := root.BuildInstance() + // Go has no notion of qualified import paths; if a CUE file imports + // "foo.com/bar:qualified", we import just "foo.com/bar" on the Go side. + unqualifiedPath := ast.ParseImportPath(inst.ImportPath).Unqualified().String() + + // As a special case, some CUE standard library types are allowed as references + // even though they aren't definitions. + defsOnly := true + switch fmt.Sprintf("%s.%s", unqualifiedPath, path) { + case "time.Duration": + // Note that CUE represents durations as strings, but Go as int64. + g.def.printf("string /* CUE time.Duration */") + return true, typeFacts{}, nil + case "time.Time": + defsOnly = false + } + + name := goNameFromPath(path, defsOnly) + if name == "" { + return false, typeFacts{}, nil // Not a path we are generating. + } + + var facts typeFacts + inProgress := false + // We did use a reference; if the referenced name was from another package, we + // need to ensure that package is imported. Otherwise, we need to ensure that the + // referenced local definition is generated. Either way, return the facts about the + // referenced type. + if root != g.pkgRoot { + if _, err := g.goImportFor(unqualifiedPath); err != nil { + return false, typeFacts{}, err + } + } else { + def, err := g.genDef(path, cue.Dereference(val)) + if err != nil { + return false, typeFacts{}, err + } + facts = def.facts + inProgress = def.inProgress + } + // We generate types depth-first; if the type referenced here is still in progress, + // it means that we are in a cyclic type, so be nillable to avoid a Go type of + // infinite size. + if inProgress && !facts.isNillable { + g.def.printf("*") + facts.isNillable = true // pointers can be nil + } + if root != g.pkgRoot { + g.def.printf("%s.", g.goImports[unqualifiedPath].Qualifier) + } + g.def.printf("%s", name) + return true, facts, nil +} + +// goImportFor registers the Go package a referenced CUE package is imported as and +// returns how the source names it. The CUE import path and the Go import path are +// different namespaces: the contract module's CUE package +// `github.com/opencharly/spec/schema` is the SOURCE of the shapes, while their generated +// Go types live in `github.com/opencharly/spec/spec`. goHomes carries that relation; an +// unregistered import is a loud error, because emitting the CUE path as a Go import names +// a package that does not compile (spec#298, F2). +func (g *generator) goImportFor(cuePath string) (goImport, error) { + home, ok := goHomes[cuePath] + if !ok { + return goImport{}, fmt.Errorf( + "no Go package is registered for CUE import %q: the projection references only packages whose Go home it knows (registered: %s) — add the relation to goHomes", + cuePath, registeredGoHomes()) + } + qualifier := home.Package + if alias, ok := g.importAliases[cuePath]; ok { + qualifier = alias + } + gi := goImport{Path: home.ImportPath, Package: home.Package, Qualifier: qualifier} + g.goImports[cuePath] = gi + return gi, nil +} + +// renderImports writes the import block for the packages the emitted definitions +// referenced, sorted by Go path and deduplicated across CUE packages that share a Go +// home. A qualifier that differs from the Go package's declared name is written as an +// explicit alias. +func renderImports(printf func(string, ...any), g *generator) { + type imp struct { + path string + packageNm string + qualifier string + } + byPath := make(map[string]imp) + for _, gi := range g.goImports { + byPath[gi.Path] = imp{path: gi.Path, packageNm: gi.Package, qualifier: gi.Qualifier} + } + for path, qualifier := range g.exprImports { + if _, ok := byPath[path]; ok { + continue + } + byPath[path] = imp{path: path, packageNm: goImportQualifier(path), qualifier: qualifier} + } + if len(byPath) == 0 { + return + } + paths := make([]string, 0, len(byPath)) + for p := range byPath { + paths = append(paths, p) + } + sort.Strings(paths) + printf("import (\n") + for _, p := range paths { + e := byPath[p] + if e.qualifier != e.packageNm { + printf("\t%s %q\n", e.qualifier, e.path) + } else { + printf("\t%q\n", e.path) + } + } + printf(")\n") +} + +// goImportQualifier is the conventional package name for a Go import path: its last +// element. It is only used where the path is already a Go path (an @go(,type=...) import), +// not for a CUE import (whose Go home is registered in goHomes). +func goImportQualifier(path string) string { + return lastPathElement(path) +} + +// emitDocs generates the documentation comments attached to the following declaration. +func emitDocs(printf func(string, ...any), groups []*ast.CommentGroup) { + for i, group := range groups { + if i > 0 { + printf("//\n") + } + for _, line := range formatDocComment(group.List) { + printf("%s\n", line) + } + } +} + +// formatDocComment canonicalizes a doc comment exactly as go/printer does for a Go 1.26 +// toolchain, using the SAME public go/doc/comment parser and printer the standard library +// uses. It is done here rather than left to go/format because Go 1.27's go/printer stopped +// rewriting a doc comment whose next token is an IDENT (a struct field's doc comment), so +// delegating the transform would make the generated file depend on which Go built the +// generator — the same schema would regenerate differently on a 1.27 developer box and in +// the 1.26 CI. Applying it up front makes both printers emit one form (the transform is +// idempotent, so a 1.26 printer re-running it changes nothing). +func formatDocComment(list []*ast.Comment) []string { + raw := make([]string, 0, len(list)) + if len(list) == 0 { + return raw + } + if strings.HasPrefix(list[0].Text, "/*") { + // A block doc comment is left alone by the printer (it only reformats a + // well-formed sequence of `//` lines). + for _, c := range list { + raw = append(raw, c.Text) + } + return raw + } + var b strings.Builder + var directives []string + for _, c := range list { + after, found := strings.CutPrefix(c.Text, "//") + if !found { + for _, d := range list { + raw = append(raw, d.Text) + } + return raw + } + // Accumulate //go:build etc lines separately, as the printer does. + if isDocDirective(after) { + directives = append(directives, c.Text) + continue + } + b.WriteString(strings.TrimPrefix(after, " ")) + b.WriteString("\n") + } + text := b.String() + if text == "" { + for _, c := range list { + raw = append(raw, c.Text) + } + return raw + } + + var p comment.Parser + d := p.Parse(text) + var pr comment.Printer + text = string(pr.Comment(d)) + + for text != "" { + var line string + line, text, _ = strings.Cut(text, "\n") + switch { + case line == "": + raw = append(raw, "//") + case strings.HasPrefix(line, "\t"): + raw = append(raw, "//"+line) + default: + raw = append(raw, "// "+line) + } + } + if len(directives) > 0 { + raw = append(raw, "//") + raw = append(raw, directives...) + } + return raw +} + +// isDocDirective reports whether c is a comment directive (the printer's isDirective, +// with the `//` removed). +func isDocDirective(c string) bool { + if strings.HasPrefix(c, "line ") || strings.HasPrefix(c, "extern ") || strings.HasPrefix(c, "export ") { + return true + } + colon := strings.Index(c, ":") + if colon <= 0 || colon+1 >= len(c) { + return false + } + for i := 0; i <= colon+1; i++ { + if i == colon { + continue + } + b := c[i] + if !isLowerAlphaNum(b) { + return false + } + } + return true +} + +// isLowerAlphaNum reports whether b is a lowercase ASCII letter or a digit (the character +// class a comment directive's prefix may contain). +func isLowerAlphaNum(b byte) bool { + return ('a' <= b && b <= 'z') || ('0' <= b && b <= '9') +} + +// gatherImportAliases collects the aliases from imports across the instance. +func gatherImportAliases(inst *build.Instance) (map[string]string, error) { + fileAliases := make(map[string]string) + tracked := make(map[string]int) + + type pair struct{ path, alias string } + var explicit []pair + + for _, file := range inst.Files { + for spec := range file.ImportSpecs() { + pkgPath, err := strconv.Unquote(spec.Path.Value) + if err != nil { + return nil, err + } + // Unaliased import: reserve its package name, which can come + // from an explicit or implicit qualifier. + if spec.Name == nil { + pkgName := ast.ParseImportPath(pkgPath).Qualifier + if pkgName == "" { + return nil, fmt.Errorf("could not find an imported package qualifier: %q", pkgPath) + } + tracked[pkgName]++ + continue + } + // Explicit alias: queue for resolution. + alias := spec.Name.Name + explicit = append(explicit, pair{path: pkgPath, alias: alias}) + } + } + + // Resolve explicit aliases with conflict suffixing. + for _, e := range explicit { + alias := e.alias + if count, ok := tracked[alias]; ok { + alias = fmt.Sprintf("%s%d", alias, count) + } + fileAliases[e.path] = alias + tracked[e.alias]++ // track the alias name (unsuffixed) for future conflicts + } + + return fileAliases, nil +} diff --git a/schemaparams/schemaparams.go b/schemaparams/schemaparams.go index 7f32814..be2fee4 100644 --- a/schemaparams/schemaparams.go +++ b/schemaparams/schemaparams.go @@ -1,9 +1,10 @@ // Package schemaparams is THE schema→Go params pipeline (R3). // -// One function runs the whole thing — concatenate a schema directory's `*.cue` -// files under a `package ` + `@go()` header, run the pinned -// `cue exp gengotypes`, then normalize the struct tags — so that the three places -// that need it cannot drift: +// One function runs the whole thing — concatenate a schema directory's `*.cue` files +// under a `package ` + `@go()` header, compose them as ONE CUE module +// instance through `schemacompose`, generate the Go structs in-process from the CUE Go +// API, then normalize the struct tags — so that the three places that need it cannot +// drift: // // - spec's own generator (its `cue-gen` task) produces `spec/cue_types_gen.go`; // - the `charly`-side verb (`charly candy params`) produces a plugin candy's @@ -14,58 +15,38 @@ // binary and without keeping its own copy of the recipe. // // The steps' contracts stay where they are, and this package only sequences them: -// `schemaconcat` (the concatenation), `cuetoolchain` (the pin and its fetcher), -// `schemaretag` (the tag normalization). The `gengotypes` generator itself lives in -// `cuelang.org/go`'s INTERNAL tree, which is why the pinned CLI is provisioned and -// run here rather than called in-process. +// `schemaconcat` (the concatenation), `schemacompose` (the one module-aware compose, +// with its offline resolution of the contract module), `schemaretag` (the tag +// normalization), and the in-process generator (`gotypes.go`). +// +// Composition and generation share ONE resolution path (opencharly/spec#271, Direction +// 1). A plugin's schema may carry `import "github.com/opencharly/spec/schema"`; the +// compose resolves it against the contract module the Go binary already links, with no +// published CUE module, no module cache and no `cue` CLI, and the generator emits the +// definition through the Go package that actually owns the shapes (`gohome.go`). That is +// what a projected params file needs to compile: `cue exp gengotypes` could do neither, +// because the CLI resolves modules only through an OCI registry and its generator lives +// in cuelang.org/go's INTERNAL tree. package schemaparams import ( "fmt" "os" - "os/exec" - "path/filepath" - "strings" - "github.com/opencharly/spec/cuetoolchain" + "github.com/opencharly/spec/schemacompose" "github.com/opencharly/spec/schemaconcat" "github.com/opencharly/spec/schemaretag" ) -// codegenModule names the scratch CUE module the pipeline compiles a schema as. It is -// not published anywhere; it exists so the instance HAS a module (and therefore a -// language version) rather than resolving against an implicit default. -const codegenModule = "github.com/opencharly/spec/schemaparams/codegen" - -// Generate provisions the pinned `cue` CLI into cueDir and returns the Go file the -// schema directory produces. cueDir is the caller's choice (spec's task uses `./bin`, -// the charly-side verb its own cache): the pin, the checksum, the verification and the -// arch rules all come from cuetoolchain, so every caller runs the SAME toolchain. -func Generate(schemaDir, pkg, cueDir string) ([]byte, error) { - return GenerateExcluding(schemaDir, pkg, cueDir, nil) +// Generate returns the Go file the schema directory produces. +func Generate(schemaDir, pkg string) ([]byte, error) { + return GenerateExcluding(schemaDir, pkg, nil) } // GenerateExcluding is Generate with a file filter: a schema whose disjunction -// wrappers (spec's `node.cue`) gengotypes must not see passes an `exclude` that names +// wrappers (spec's `node.cue`) are not param structs passes an `exclude` that names // them. nil includes every `*.cue` file, which is what a plugin candy wants. -func GenerateExcluding(schemaDir, pkg, cueDir string, exclude func(name string) bool) ([]byte, error) { - cueBin, err := cuetoolchain.Ensure(cueDir) - if err != nil { - return nil, err - } - return GenerateWithCueExcluding(schemaDir, pkg, cueBin, exclude) -} - -// GenerateWithCue runs the pipeline with an explicit `cue` binary — the seam for a -// caller that has already provisioned one, and for a test that stands in for the -// toolchain rather than downloading it. -func GenerateWithCue(schemaDir, pkg, cueBin string) ([]byte, error) { - return GenerateWithCueExcluding(schemaDir, pkg, cueBin, nil) -} - -// GenerateWithCueExcluding is GenerateWithCue with a file filter (see -// GenerateExcluding). -func GenerateWithCueExcluding(schemaDir, pkg, cueBin string, exclude func(name string) bool) ([]byte, error) { +func GenerateExcluding(schemaDir, pkg string, exclude func(name string) bool) ([]byte, error) { body, files, err := schemaconcat.ConcatSchema(os.DirFS(schemaDir), ".", exclude) if err != nil { return nil, fmt.Errorf("reading %s: %w", schemaDir, err) @@ -73,49 +54,16 @@ func GenerateWithCueExcluding(schemaDir, pkg, cueBin string, exclude func(name s if len(files) == 0 { return nil, fmt.Errorf("no *.cue files in %s", schemaDir) } - tmp, err := os.MkdirTemp("", "schemaparams-") + // The header is part of the contract: `package ` is re-established by the + // compose, and the file-level @go() attribute is what the generator reads to + // name the Go package. + inst, err := schemacompose.ComposeText(nil, "@go("+pkg+")\n\n"+body, pkg) if err != nil { - return nil, err - } - defer func() { _ = os.RemoveAll(tmp) }() - // The CUE module the instance is compiled as. The recipe this pipeline replaced - // wrote this file, and it is what fixes the LANGUAGE VERSION the generator resolves - // against — so it belongs to the pipeline, not to one caller's shell, and the version - // comes from the PIN (cuetoolchain) so the declaration cannot drift from the binary - // that reads it. `codegenModule` is a neutral path: the module is a scratch instance, - // not a published one. - if err := os.MkdirAll(filepath.Join(tmp, "cue.mod"), 0o755); err != nil { - return nil, err - } - mod := fmt.Sprintf("module: %q\nlanguage: version: %q\n", codegenModule, cuetoolchain.Version) - if err := os.WriteFile(filepath.Join(tmp, "cue.mod", "module.cue"), []byte(mod), 0o644); err != nil { - return nil, err - } - // The header is part of the contract: `package ` names the Go package and - // the file-level @go() attribute is what gengotypes reads to name it. - src := "package " + pkg + "\n\n@go(" + pkg + ")\n\n" + body - if err := os.WriteFile(filepath.Join(tmp, pkg+".cue"), []byte(src), 0o644); err != nil { - return nil, err - } - // The generator runs with the TEMP dir as its working directory, so the binary - // path must be absolute first: a caller that passes a relative one (spec's own - // task passes `bin`, i.e. ./bin/cue) would otherwise fail with - // `fork/exec bin/cue: no such file or directory`. - absCue := cueBin - if !filepath.IsAbs(absCue) { - var aerr error - if absCue, aerr = filepath.Abs(cueBin); aerr != nil { - return nil, fmt.Errorf("resolving the toolchain path %q: %w", cueBin, aerr) - } - } - cmd := exec.Command(absCue, "exp", "gengotypes", ".") - cmd.Dir = tmp - if out, err := cmd.CombinedOutput(); err != nil { - return nil, fmt.Errorf("cue exp gengotypes failed in %s: %w\n%s", tmp, err, strings.TrimSpace(string(out))) + return nil, fmt.Errorf("composing %s as package %s: %w", schemaDir, pkg, err) } - gen, err := os.ReadFile(filepath.Join(tmp, "cue_types_"+pkg+"_gen.go")) + gen, err := generateGo(inst) if err != nil { - return nil, fmt.Errorf("gengotypes produced no cue_types_%s_gen.go: %w", pkg, err) + return nil, fmt.Errorf("generating Go for %s: %w", schemaDir, err) } return schemaretag.Normalize(gen), nil } diff --git a/schemaparams/schemaparams_test.go b/schemaparams/schemaparams_test.go index da85ee7..b677e11 100644 --- a/schemaparams/schemaparams_test.go +++ b/schemaparams/schemaparams_test.go @@ -1,218 +1,234 @@ package schemaparams -// schemaparams_test.go — the pipeline's contract, exercised through the real code -// path with a STAND-IN toolchain (deterministic, no network) plus one arm that runs -// the REAL provisioned `cue`, which skips visibly wherever it cannot be provisioned. +// schemaparams_test.go — the pipeline's contract. +// +// Three kinds of arm, each proving a different claim: +// +// - the IMPORT case (opencharly/spec#298, F1+F2): a plugin schema carrying +// `import "github.com/opencharly/spec/schema"` generates params that reference the +// Go package that OWNS the shapes, and the generated file COMPILES against the +// contract module (a real `go build`, not a string assertion); +// - the NO-IMPORT case: output must stay byte-identical to the pinned +// `cue exp gengotypes`, which the oracle arm proves by running that CLI on the same +// source (it skips visibly when the pinned binary cannot be provisioned); +// - the pipeline's own edges (package name, file filter, empty dir, unregistered +// import). import ( "os" + "os/exec" "path/filepath" "strings" "testing" "github.com/opencharly/spec/cuetoolchain" + "github.com/opencharly/spec/schemaconcat" + "github.com/opencharly/spec/schemaretag" ) -// stubCue writes an executable standing in for `cue exp gengotypes`: it CAPTURES the -// CUE source it was handed (so the concat+header contract can be asserted) and emits -// a fixed generated file with a json-only tag (so the retag contract can be asserted). -func stubCue(t *testing.T, captureDir, body, exitLine string) string { - t.Helper() - bin := filepath.Join(t.TempDir(), "cue") - // The stub derives the generated file's name from the CUE file it is handed - // (`.cue`), exactly as the real `cue exp gengotypes` names its output — - // so the same stub serves any package name. - script := "#!/bin/sh\n" + - "f=$(ls *.cue | head -1); pkg=$(basename \"$f\" .cue)\n" + - "cp \"$f\" \"" + captureDir + "/captured.cue\" 2>/dev/null || true\n" + - "cp cue.mod/module.cue \"" + captureDir + "/captured_module.cue\" 2>/dev/null || true\n" + - exitLine + "\n" + - "cat > \"cue_types_${pkg}_gen.go\" <<'EOF'\n" + body + "\nEOF\n" - if err := os.WriteFile(bin, []byte(script), 0o755); err != nil { - t.Fatal(err) - } - return bin -} - -func schemaDirWith(t *testing.T, files map[string]string) string { - t.Helper() - dir := filepath.Join(t.TempDir(), "schema") - if err := os.MkdirAll(dir, 0o755); err != nil { - t.Fatal(err) - } - for name, body := range files { - if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { - t.Fatal(err) - } - } - return dir -} - -const stubBody = "package params\n\ntype Foo struct {\n\tName string `json:\"name\"`\n}" - -// TestGenerateWithCueRunsConcatHeaderAndRetag pins all three steps in one arm: the -// schema files arrive concatenated under the `package params` + `@go(params)` header -// (captured from the child), and the returned file has the tags normalized. -func TestGenerateWithCueRunsConcatHeaderAndRetag(t *testing.T) { - capture := t.TempDir() - schema := schemaDirWith(t, map[string]string{ - "a.cue": "#A: {\n\tname?: string\n}\n", - "b.cue": "#B: {\n\tother?: string\n}\n", - }) - bin := stubCue(t, capture, stubBody, "true") - - got, err := GenerateWithCue(schema, "params", bin) +// TestGenerateNoImportIsByteIdenticalToGengotypes is the no-regression proof: the +// in-process generator must produce EXACTLY what the pinned `cue exp gengotypes` produces +// for a schema without an import (the twin shape, every plugin's shape today). The oracle +// re-runs the old recipe verbatim — concat, scratch module, CLI, tag normalization — so +// the comparison is against the real toolchain, never a hand-written expectation. +func TestGenerateNoImportIsByteIdenticalToGengotypes(t *testing.T) { + cue := pinnedCue(t) + if cue == "" { + t.Skipf("pinned cue %s could not be provisioned here — the byte-identity oracle is skipped, not faked", cuetoolchain.Version) + } + dir := filepath.Join("testdata", "twin", "schema") + want := gengotypesOracle(t, dir, "params", cue) + got, err := Generate(dir, "params") if err != nil { - t.Fatalf("GenerateWithCue: %v", err) + t.Fatalf("Generate: %v", err) } - if want := "package params\n\ntype Foo struct {\n\tName string `yaml:\"name,omitempty\" json:\"name\"`\n}\n"; string(got) != want { - t.Fatalf("pipeline output =\n%q\nwant\n%q", got, want) + if string(got) != string(want) { + t.Fatalf("in-process output differs from `cue exp gengotypes` (%d vs %d bytes)\n--- gengotypes ---\n%s\n--- in-process ---\n%s", len(want), len(got), want, got) } +} - handed, err := os.ReadFile(filepath.Join(capture, "captured.cue")) +// TestGenerateImportReferencesTheOwningGoPackage is F2: the emitted Go must import the +// package that owns the shapes (`github.com/opencharly/spec/spec`), never the CUE +// package path whose Go package exports only FS. +func TestGenerateImportReferencesTheOwningGoPackage(t *testing.T) { + out, err := Generate(filepath.Join("testdata", "import", "schema"), "params") if err != nil { - t.Fatalf("the toolchain was not handed a CUE source: %v", err) + t.Fatalf("Generate: %v", err) } - src := string(handed) - if !strings.HasPrefix(src, "package params\n\n@go(params)\n\n") { - t.Fatalf("the concatenation is not headed by the package clause + @go attribute:\n%.120s", src) + src := string(out) + if !strings.Contains(src, "\"github.com/opencharly/spec/spec\"") { + t.Fatalf("generated file does not import the Go package that owns the shapes:\n%s", src) } - for _, want := range []string{"#A: {", "#B: {"} { - if !strings.Contains(src, want) { - t.Fatalf("the concatenation is missing %s:\n%s", want, src) - } + if strings.Contains(src, "\"github.com/opencharly/spec/schema\"") { + t.Fatalf("generated file imports the CUE package path as Go (F2):\n%s", src) } - if strings.Index(src, "#A: {") > strings.Index(src, "#B: {") { - t.Fatalf("the concatenation is not in sorted file order:\n%s", src) + if !strings.Contains(src, "Config spec.TunnelConfig ") { + t.Fatalf("generated field does not reference spec.TunnelConfig:\n%s", src) } } -// TestGenerateWithCueEmitsTheCodegenModule pins the emitted ENVIRONMENT, not just the -// output: the pipeline compiles the schema as a CUE module whose language version comes -// from the pin, because the recipe it replaced wrote exactly that file and the version -// it declares is what the generator resolves against. Without this arm the module could -// silently disappear again while every output assertion still passed. -func TestGenerateWithCueEmitsTheCodegenModule(t *testing.T) { - capture := t.TempDir() - schema := schemaDirWith(t, map[string]string{"a.cue": "#A: {\n\tname?: string\n}\n"}) - if _, err := GenerateWithCue(schema, "params", stubCue(t, capture, stubBody, "true")); err != nil { - t.Fatalf("GenerateWithCue: %v", err) - } - mod, err := os.ReadFile(filepath.Join(capture, "captured_module.cue")) +// TestGenerateImportCompiles proves the import case at its REAL boundary: the generated +// params file is placed in a module that requires this one and `go build` compiles it. +// A string assertion cannot catch a Go package that exists but lacks the type (the exact +// F2 failure), so this arm builds it. +func TestGenerateImportCompiles(t *testing.T) { + out, err := Generate(filepath.Join("testdata", "import", "schema"), "params") if err != nil { - t.Fatalf("the pipeline compiled the schema with NO cue.mod/module.cue: %v", err) + t.Fatalf("Generate: %v", err) } - if !strings.Contains(string(mod), "module: \""+codegenModule+"\"") { - t.Fatalf("module.cue does not declare the codegen module:\n%s", mod) + root, err := filepath.Abs("..") + if err != nil { + t.Fatal(err) } - if !strings.Contains(string(mod), "language: version: \""+cuetoolchain.Version+"\"") { - t.Fatalf("module.cue does not declare the PINNED language version %s:\n%s", cuetoolchain.Version, mod) + sum, err := os.ReadFile(filepath.Join(root, "go.sum")) + if err != nil { + t.Fatalf("read the module's go.sum (needed by the temp module's build): %v", err) } -} - -// TestGenerateWithCueHonoursThePackageName: the header follows the caller's package, -// so the same function serves a plugin (`params`) and spec itself (`spec`). -func TestGenerateWithCueHonoursThePackageName(t *testing.T) { - capture := t.TempDir() - schema := schemaDirWith(t, map[string]string{"a.cue": "#A: {\n\tname?: string\n}\n"}) - bin := stubCue(t, capture, "package spec\n\ntype Foo struct {\n\tName string `json:\"name\"`\n}", "true") - - if _, err := GenerateWithCue(schema, "spec", bin); err != nil { - t.Fatalf("GenerateWithCue: %v", err) + tmp := t.TempDir() + if err := os.MkdirAll(filepath.Join(tmp, "params"), 0o755); err != nil { + t.Fatal(err) } - handed, err := os.ReadFile(filepath.Join(capture, "captured.cue")) - if err != nil { + if err := os.WriteFile(filepath.Join(tmp, "params", "cue_types_gen.go"), out, 0o644); err != nil { + t.Fatal(err) + } + gomod := "module example.com/pluginparams\n\ngo 1.26.0\n\nrequire github.com/opencharly/spec v0.0.0\n\nreplace github.com/opencharly/spec => " + root + "\n" + if err := os.WriteFile(filepath.Join(tmp, "go.mod"), []byte(gomod), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tmp, "go.sum"), sum, 0o644); err != nil { t.Fatal(err) } - if !strings.HasPrefix(string(handed), "package spec\n\n@go(spec)\n\n") { - t.Fatalf("the header does not follow the caller's package:\n%.80s", handed) + cmd := exec.Command("go", "build", "./params") + cmd.Dir = tmp + cmd.Env = append(os.Environ(), "GOWORK=off", "GOFLAGS=-mod=mod", "GOPROXY=off") + if buildOut, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("the generated params do not compile against the contract module: %v\n%s\n--- generated ---\n%s", err, buildOut, out) } } -// TestGenerateWithCueAcceptsARelativeToolchainPath is the arm that would have caught -// the bug this pipeline shipped for one run: the generator is exec'd with the TEMP dir -// as its working directory, so a RELATIVE cueBin (spec's own task passes `bin`) must be -// resolved against the caller's working directory first — otherwise it fails with -// `fork/exec bin/cue: no such file or directory`. -func TestGenerateWithCueAcceptsARelativeToolchainPath(t *testing.T) { - home, err := os.Getwd() - if err != nil { - t.Fatal(err) +// TestGenerateHonoursThePackageName: the header follows the caller's package, so the same +// pipeline serves a plugin (`params`) and spec itself (`spec`). +func TestGenerateHonoursThePackageName(t *testing.T) { + dir := filepath.Join("testdata", "twin", "schema") + for _, pkg := range []string{"params", "spec"} { + out, err := Generate(dir, pkg) + if err != nil { + t.Fatalf("Generate(%q): %v", pkg, err) + } + if !strings.Contains(string(out), "\npackage "+pkg+"\n") { + t.Fatalf("generated file does not declare package %q:\n%.120s", pkg, out) + } } +} + +// TestGenerateExcludingHonoursTheFilter: the file filter is the caller's only input +// difference (spec excludes node.cue), so an excluded file must contribute nothing. +func TestGenerateExcludingHonoursTheFilter(t *testing.T) { dir := t.TempDir() - if err := os.Chdir(dir); err != nil { - t.Fatal(err) + for name, body := range map[string]string{ + "a.cue": "#A: {\n\tname?: string\n}\n", + "b.cue": "#B: {\n\tother?: string\n}\n", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } } - t.Cleanup(func() { _ = os.Chdir(home) }) - binDir := filepath.Join(dir, "bin") - if err := os.MkdirAll(binDir, 0o755); err != nil { - t.Fatal(err) + out, err := GenerateExcluding(dir, "params", func(name string) bool { return name == "b.cue" }) + if err != nil { + t.Fatalf("GenerateExcluding: %v", err) } - bin := stubCue(t, dir, stubBody, "true") - if err := os.Rename(bin, filepath.Join(binDir, "cue")); err != nil { - t.Fatal(err) + if !strings.Contains(string(out), "type A struct") { + t.Fatalf("the included definition is missing:\n%s", out) } - schema := schemaDirWith(t, map[string]string{"a.cue": "#A: {\n\tname?: string\n}\n"}) - if _, err := GenerateWithCue(schema, "params", filepath.Join("bin", "cue")); err != nil { - t.Fatalf("a relative toolchain path must resolve against the caller's working directory: %v", err) + if strings.Contains(string(out), "type B struct") { + t.Fatalf("the EXCLUDED file still generated a type:\n%s", out) } } -// TestGenerateWithCueRejectsAnEmptySchemaDir: "no schema" must be an error naming the -// directory, never an empty generated file. -func TestGenerateWithCueRejectsAnEmptySchemaDir(t *testing.T) { +// TestGenerateRejectsAnEmptySchemaDir: "no schema" must be an error naming the directory, +// never an empty generated file. +func TestGenerateRejectsAnEmptySchemaDir(t *testing.T) { dir := filepath.Join(t.TempDir(), "empty") if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatal(err) } - _, err := GenerateWithCue(dir, "params", stubCue(t, t.TempDir(), stubBody, "true")) + _, err := Generate(dir, "params") if err == nil || !strings.Contains(err.Error(), "no *.cue files") { t.Fatalf("want an empty-schema error, got %v", err) } } -// TestGenerateWithCueSurfacesAToolchainFailure: a failing generator is reported with -// its own output, not as an empty or partial result. -func TestGenerateWithCueSurfacesAToolchainFailure(t *testing.T) { - schema := schemaDirWith(t, map[string]string{"a.cue": "#A: {\n\tname?: string\n}\n"}) - bin := stubCue(t, t.TempDir(), stubBody, "echo 'cue: boom' >&2; exit 1") - _, err := GenerateWithCue(schema, "params", bin) - if err == nil { - t.Fatal("a failing toolchain produced no error") - } - if !strings.Contains(err.Error(), "gengotypes failed") || !strings.Contains(err.Error(), "cue: boom") { - t.Fatalf("the failure must name the step and carry the toolchain's own output, got: %v", err) +// TestGoImportForRejectsAnUnregisteredPackage: an import the projection cannot name a Go +// home for is a loud error, never a reference to a Go path that may not compile. +func TestGoImportForRejectsAnUnregisteredPackage(t *testing.T) { + g := &generator{goImports: map[string]goImport{}, importAliases: map[string]string{}} + _, err := g.goImportFor("example.com/other/schema") + if err == nil || !strings.Contains(err.Error(), "no Go package is registered") { + t.Fatalf("want an unregistered-import error, got %v", err) } } -// TestGenerateProvisionsThePinnedToolchain runs the REAL pipeline: Generate resolves -// the pinned `cue` through cuetoolchain.Ensure (no stub), generates from a tiny schema -// and normalizes the tags. It needs the release to be provisionable, so it skips -// VISIBLY — never with a canned substitute — when that is impossible here. -func TestGenerateProvisionsThePinnedToolchain(t *testing.T) { - schema := schemaDirWith(t, map[string]string{ - "a.cue": "// The leading doc comment, which gengotypes reproduces.\n#A: {\n\tname?: string\n\tretry_limit?: int\n}\n", - }) +// TestGoImportForUsesTheGoHome: the relation maps the CUE package to its Go owner, and an +// explicit CUE import alias is preserved as the Go qualifier. +func TestGoImportForUsesTheGoHome(t *testing.T) { + g := &generator{goImports: map[string]goImport{}, importAliases: map[string]string{"github.com/opencharly/spec/schema": "contract"}} + gi, err := g.goImportFor("github.com/opencharly/spec/schema") + if err != nil { + t.Fatal(err) + } + if gi.Path != "github.com/opencharly/spec/spec" || gi.Qualifier != "contract" { + t.Fatalf("goImportFor = %+v, want path spec/spec and qualifier contract", gi) + } +} + +// pinnedCue resolves the pinned `cue` CLI for the oracle arm, provisioning it through the +// SAME cuetoolchain the repo's bootstrap task uses. "" means it cannot run here. +func pinnedCue(t *testing.T) string { + t.Helper() cache, err := os.UserCacheDir() if err != nil { - t.Skipf("no user cache dir to provision the toolchain into: %v", err) + return "" } - got, err := Generate(schema, "params", filepath.Join(cache, "charly", "cue", cuetoolchain.Version)) + bin, err := cuetoolchain.Ensure(filepath.Join(cache, "charly", "cue", cuetoolchain.Version)) if err != nil { - t.Skipf("the pinned cue toolchain could not be provisioned here (%v) — this arm is skipped, not faked", err) + return "" } - src := string(got) - if !strings.Contains(src, "package params") { - t.Fatalf("generated source has no package clause:\n%.200s", src) + return bin +} + +// gengotypesOracle reproduces the pipeline the shell-out used: concatenate the schema +// directory, head it `package ` + `@go()`, write a scratch CUE module, run the +// pinned `cue exp gengotypes`, normalize the tags. It is a TEST-ONLY oracle: the old +// recipe must be reproducible to compare against, but it is not product code any more. +func gengotypesOracle(t *testing.T, schemaDir, pkg, cueBin string) []byte { + t.Helper() + body, files, err := schemaconcat.ConcatSchema(os.DirFS(schemaDir), ".", nil) + if err != nil { + t.Fatalf("concat: %v", err) } - // The leading doc comment is reproduced by the generator — the trap that makes a - // grep-for-a-retired-string check useless — and the tags are normalized. - if !strings.Contains(src, "The leading doc comment") { - t.Fatalf("the schema's leading doc comment did not survive into the generated file:\n%.300s", src) + if len(files) == 0 { + t.Fatalf("no *.cue files in %s", schemaDir) } - for _, line := range strings.Split(src, "\n") { - if strings.Contains(line, "json:\"") && !strings.Contains(line, "yaml:\"") { - t.Fatalf("a generated line carries a json tag with no yaml tag (retag did not run):\n%s", line) - } + tmp := t.TempDir() + if err := os.MkdirAll(filepath.Join(tmp, "cue.mod"), 0o755); err != nil { + t.Fatal(err) + } + mod := "module: \"github.com/opencharly/spec/schemaparams/codegen\"\nlanguage: version: \"" + cuetoolchain.Version + "\"\n" + if err := os.WriteFile(filepath.Join(tmp, "cue.mod", "module.cue"), []byte(mod), 0o644); err != nil { + t.Fatal(err) + } + src := "package " + pkg + "\n\n@go(" + pkg + ")\n\n" + body + if err := os.WriteFile(filepath.Join(tmp, pkg+".cue"), []byte(src), 0o644); err != nil { + t.Fatal(err) + } + cmd := exec.Command(cueBin, "exp", "gengotypes", ".") + cmd.Dir = tmp + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("cue exp gengotypes failed: %v\n%s", err, out) + } + gen, err := os.ReadFile(filepath.Join(tmp, "cue_types_"+pkg+"_gen.go")) + if err != nil { + t.Fatalf("gengotypes produced no output: %v", err) } + return schemaretag.Normalize(gen) } diff --git a/schemaparams/testdata/import/schema/tunnel.cue b/schemaparams/testdata/import/schema/tunnel.cue new file mode 100644 index 0000000..5701923 --- /dev/null +++ b/schemaparams/testdata/import/schema/tunnel.cue @@ -0,0 +1,45 @@ +// The OUT-OF-TREE plugin-tunnel's OWN CUE schema — the typed params for the +// `tunnel` VERB (verb:tunnel), the externalized tailscale/cloudflare TUNNEL +// EXECUTION LEG. It is the SINGLE SOURCE for this plugin's params, used two ways +// (the same contract core `spec` + the reference examplerunverb use): +// +// 1. GENERATE the Go param struct — the schema→Go pipeline concatenates this file under a +// `package params` + `@go(params)` header and generates the Go struct from it, +// emits +// ../params/cue_types_gen.go, so the provider decodes plugin_input into a TYPED +// struct (params.TunnelInput / TunnelConfig / TunnelPort), never a hand-parsed map. +// 2. VALIDATE authored input AT RUNTIME — the plugin SERVES this source over the +// Describe channel (sdk.BuildCapabilities → schema_cue); the host splices it onto +// its base (base ++ plugin) and validates every authored `plugin: tunnel` step's +// plugin_input against #TunnelInput. The host NEVER reads this file from disk — the +// schema travels with the plugin. +// +// verb:tunnel is NOT a check verb in the usual sense — it is the externalized TUNNEL +// EXECUTION backend. The pod-lifecycle plugins (candy/plugin-deploy-pod, candy/plugin-pod) +// drive its start/stop/setup methods directly over this verb's Invoke envelope +// ({method,config}) via InvokeProvider. It ALSO +// carries a benign `plan` (dry-run) method — a `plugin: tunnel` check step returns the +// EXACT tailscale/cloudflared argv it WOULD run WITHOUT exec, so a disposable bed proves +// the registry dispatch + the TunnelConfig wire round-trip + the moved command-building +// logic with ZERO tailscale/cloudflare credentials. +// +// SELF-CONTAINED: #TunnelInput references only its sibling defs (#TunnelConfig / +// #TunnelPort) WITHIN this schema, NO base def, so it compiles standalone (gengotypes + +// the SDK serve-side check) AND splices onto the base — the splice exists to detect a +// def-name collision with the base, not to resolve base references. + +import "github.com/opencharly/spec/schema" + +// #TunnelInput — the plugin_input shape for the `tunnel` verb. +#TunnelInput: { + // method — the tunnel operation: start | stop | setup | plan. `plan` is the + // creds-free dry-run that returns the argv the operation WOULD run (no exec). + method: string & !="" @go(Method) + // config — the resolved tunnel configuration to act on (byte-compatible with the + // core's TunnelConfig, sent by the calling pod-lifecycle plugin over the Invoke envelope). + config?: schema.#TunnelConfig @go(Config) + // expect — plan only: the expected argv command lines (space-joined). When set, the + // plan method compares the built argv against it (FAIL on mismatch); empty ⇒ the + // step passes as long as dispatch + round-trip succeed, echoing the built argv. + expect?: [...string] @go(Expect) +} diff --git a/schemaparams/testdata/twin/schema/tunnel.cue b/schemaparams/testdata/twin/schema/tunnel.cue new file mode 100644 index 0000000..1de0eba --- /dev/null +++ b/schemaparams/testdata/twin/schema/tunnel.cue @@ -0,0 +1,72 @@ +// The OUT-OF-TREE plugin-tunnel's OWN CUE schema — the typed params for the +// `tunnel` VERB (verb:tunnel), the externalized tailscale/cloudflare TUNNEL +// EXECUTION LEG. It is the SINGLE SOURCE for this plugin's params, used two ways +// (the same contract core `spec` + the reference examplerunverb use): +// +// 1. GENERATE the Go param struct — the schema→Go pipeline concatenates this file under a +// `package params` + `@go(params)` header and generates the Go struct from it, +// emits +// ../params/cue_types_gen.go, so the provider decodes plugin_input into a TYPED +// struct (params.TunnelInput / TunnelConfig / TunnelPort), never a hand-parsed map. +// 2. VALIDATE authored input AT RUNTIME — the plugin SERVES this source over the +// Describe channel (sdk.BuildCapabilities → schema_cue); the host splices it onto +// its base (base ++ plugin) and validates every authored `plugin: tunnel` step's +// plugin_input against #TunnelInput. The host NEVER reads this file from disk — the +// schema travels with the plugin. +// +// verb:tunnel is NOT a check verb in the usual sense — it is the externalized TUNNEL +// EXECUTION backend. The pod-lifecycle plugins (candy/plugin-deploy-pod, candy/plugin-pod) +// drive its start/stop/setup methods directly over this verb's Invoke envelope +// ({method,config}) via InvokeProvider. It ALSO +// carries a benign `plan` (dry-run) method — a `plugin: tunnel` check step returns the +// EXACT tailscale/cloudflared argv it WOULD run WITHOUT exec, so a disposable bed proves +// the registry dispatch + the TunnelConfig wire round-trip + the moved command-building +// logic with ZERO tailscale/cloudflare credentials. +// +// SELF-CONTAINED: #TunnelInput references only its sibling defs (#TunnelConfig / +// #TunnelPort) WITHIN this schema, NO base def, so it compiles standalone (gengotypes + +// the SDK serve-side check) AND splices onto the base — the splice exists to detect a +// def-name collision with the base, not to resolve base references. + +// #TunnelInput — the plugin_input shape for the `tunnel` verb. +#TunnelInput: { + // method — the tunnel operation: start | stop | setup | plan. `plan` is the + // creds-free dry-run that returns the argv the operation WOULD run (no exec). + method: string & !="" @go(Method) + // config — the resolved tunnel configuration to act on (byte-compatible with the + // core's TunnelConfig, sent by the calling pod-lifecycle plugin over the Invoke envelope). + config?: #TunnelConfig @go(Config) + // expect — plan only: the expected argv command lines (space-joined). When set, the + // plan method compares the built argv against it (FAIL on mismatch); empty ⇒ the + // step passes as long as dispatch + round-trip succeed, echoing the built argv. + expect?: [...string] @go(Expect) +} + +// #TunnelConfig — the resolved, ready-to-execute tunnel configuration. +#TunnelConfig: { + // provider — "tailscale" or "cloudflare". + provider?: string @go(Provider) + // tunnel_name — cloudflare: tunnel name. + tunnel_name?: string @go(TunnelName) + // hostname — cloudflare: default hostname (from the image dns field). + hostname?: string @go(Hostname) + // box_name — for PID file naming / cloudflare tunnel-name default. + box_name?: string @go(BoxName) + // ports — all tunneled ports with their access scope. + ports?: [...#TunnelPort] @go(Ports) +} + +// #TunnelPort — a single port to tunnel with its protocol and access scope. +#TunnelPort: { + // port — the tailscale HTTPS listen port (must be a valid serve/funnel port). + port?: int @go(Port) + // backend_port — the localhost backend port (0 means same as port). + backend_port?: int @go(BackendPort) + // protocol — backend scheme: http | https | https+insecure | tcp | + // tls-terminated-tcp | ssh | rdp | smb (udp is skipped, never tunneled). + protocol?: string @go(Protocol) + // public — true = internet-accessible (funnel), false = private (tailnet-only serve). + public?: bool @go(Public) + // hostname — cloudflare: per-port hostname (from the map form). + hostname?: string @go(Hostname) +} diff --git a/schemaretag/schemaretag.go b/schemaretag/schemaretag.go index 8946949..c8667ff 100644 --- a/schemaretag/schemaretag.go +++ b/schemaretag/schemaretag.go @@ -1,21 +1,21 @@ // Package schemaretag is THE single struct-tag normalization contract for the // schema→Go pipeline (R3). // -// `cue exp gengotypes` emits json tags only, but every committed +// The params generator emits json tags only, but every committed // `cue_types_gen.go` in this org carries yaml tags too — so the generation -// pipeline has a THIRD step after concatenation (`schemaconcat`) and generation: -// normalize the struct tags, doubling every json tag with a yaml tag of the same -// name and giving a bare yaml key `,omitempty`. +// pipeline has a THIRD step after concatenation (`schemaconcat`) and generation +// (`schemaparams`): normalize the struct tags, doubling every json tag with a yaml +// tag of the same name and giving a bare yaml key `,omitempty`. // // That step used to live only inside `internal/schemagen`, which no other module // may import (Go's internal rule) — which is why a charly-native regeneration // verb could not reuse it, and why the recipe had to be published as -// checkout-relative shell. It lives here now: `internal/schemagen`'s retag mode -// and the `charly`-side generation verb both call THIS, so a generated file can -// never depend on which of them produced it. +// checkout-relative shell. It lives here now: `schemaparams` calls THIS, and the +// `charly`-side generation verb reaches it through `schemaparams`, so a generated +// file can never depend on which of them produced it. // -// STANDALONE: depends only on the stdlib `regexp`, so both callers can import it -// without pulling in the packages they regenerate. +// STANDALONE: depends only on the stdlib `regexp`, so its caller can import it +// without pulling in the packages it regenerates. package schemaretag import "regexp" diff --git a/spec/gen_repro_test.go b/spec/gen_repro_test.go index 50c9bb1..fe5f79c 100644 --- a/spec/gen_repro_test.go +++ b/spec/gen_repro_test.go @@ -2,14 +2,13 @@ package spec // Reproducibility gate: the committed cue_types_gen.go and vocab_gen.go MUST // equal a fresh `charly task cue-gen`. This re-runs the SAME tools the task runs (the -// internal/schemagen concat + the pinned cue exp gengotypes + the -// schemagen vocab emitter) into a temp dir and diffs the result against the -// committed files. It skips gracefully when the pinned cue CLI is unavailable -// (a dev box without ./bin/cue), but in CI — where `charly task cue-gen` has run — it -// catches any drift between schema/*.cue and the committed generated Go. +// internal/schemagen concat + the in-process schemaparams generator + the +// schemagen vocab emitter) into a temp dir and compares the result BYTE FOR BYTE +// against the committed files. Nothing here needs the `cue` CLI — generation is +// in-process — so the gate never skips, and it catches any drift between +// schema/*.cue and the committed generated Go. import ( - "go/format" "os" "os/exec" "path/filepath" @@ -17,34 +16,12 @@ import ( "testing" ) -const cueVersion = "v0.16.1" - -// findCue resolves the pinned cue CLI: ../bin/cue (repo bin) or a PATH `cue` — -// whichever reports cueVersion. Returns "" if none. -func findCue(t *testing.T) string { - t.Helper() - candidates := []string{ - filepath.Join("..", "bin", "cue"), // repoRoot/bin/cue (cwd = spec/) - } - if p, err := exec.LookPath("cue"); err == nil { - candidates = append(candidates, p) - } - for _, c := range candidates { - out, err := exec.Command(c, "version").CombinedOutput() - if err == nil && strings.Contains(string(out), cueVersion) { - abs, _ := filepath.Abs(c) - return abs - } - } - return "" -} - // freshTypesGen reproduces spec/cue_types_gen.go through the ONE pipeline: the same // `schemagen -mode=params` the `cue-gen` task runs, which delegates to // schemaparams.GenerateExcluding. The test does not re-run the individual steps — -// concatenation, `cue exp gengotypes` and the tag normalization all live in that one -// function now, and a test that reproduced them itself would be a second copy of the -// mechanism it is meant to gate (and would not notice the task's path breaking). +// concatenation, composition, generation and the tag normalization all live in that +// one function now, and a test that reproduced them itself would be a second copy of +// the mechanism it is meant to gate (and would not notice the task's path breaking). func freshTypesGen(t *testing.T) []byte { t.Helper() out := filepath.Join(t.TempDir(), "cue_types_gen.go") @@ -53,7 +30,7 @@ func freshTypesGen(t *testing.T) []byte { if err != nil { t.Fatalf("read fresh types: %v", err) } - return gofmt(t, raw) + return raw } // freshVocabGen reproduces spec/vocab_gen.go via schemagen -mode=vocab. @@ -65,7 +42,7 @@ func freshVocabGen(t *testing.T) []byte { if err != nil { t.Fatalf("read fresh vocab: %v", err) } - return gofmt(t, raw) + return raw } func runIn(t *testing.T, dir, name string, args ...string) { @@ -77,30 +54,16 @@ func runIn(t *testing.T, dir, name string, args ...string) { } } -func gofmt(t *testing.T, b []byte) []byte { - t.Helper() - f, err := format.Source(b) - if err != nil { - t.Fatalf("gofmt: %v", err) - } - return f -} - func committed(t *testing.T, name string) []byte { t.Helper() raw, err := os.ReadFile(name) if err != nil { t.Fatalf("read committed %s: %v", name, err) } - return gofmt(t, raw) + return raw } func TestGenReproducible(t *testing.T) { - cue := findCue(t) - if cue == "" { - t.Skipf("pinned cue %s not available (run `charly task cue-gen` to bootstrap ./bin/cue) — skipping reproducibility gate", cueVersion) - } - if got, want := freshTypesGen(t), committed(t, "cue_types_gen.go"); !equalBytes(got, want) { t.Errorf("cue_types_gen.go is STALE: a fresh `charly task cue-gen` differs from the committed file.\n"+ "Run `charly task cue-gen` and commit the result. (fresh=%d bytes, committed=%d bytes)", len(got), len(want)) From 20baf6439660df9f1d69e1625c4da71b11cac0e5 Mon Sep 17 00:00:00 2001 From: aitrawog Date: Mon, 9 Nov 2026 15:18:20 +0100 Subject: [PATCH 2/2] test(schemaparams): make the import compile arm hermetic in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compile arm built its temp module from a minimal `require spec` and `-mod=mod`, so `go build` expanded the module graph and asked the proxy for gopkg.in/check.v1's go.mod — which CI's cache did not hold, and GOPROXY=off refused the lookup: go: gopkg.in/yaml.v3@v3.0.1 requires gopkg.in/check.v1@v0.0.0-...: module lookup disabled by GOPROXY=off It passed locally only because this host's cache happened to hold that go.mod. The temp module now mirrors this module's own declaration (same go directive, same requirement set, module path rewritten, contract module added as a local replace) and builds in the default readonly mode, so no graph expansion is attempted. Reproduced with a cold cache minus check.v1: the old construction fails with the CI error, the new one passes. Assisted-by: DeepSeek Harness deepseek-v4.1-flash (fully tested and validated) --- schemaparams/schemaparams_test.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/schemaparams/schemaparams_test.go b/schemaparams/schemaparams_test.go index b677e11..03c4a25 100644 --- a/schemaparams/schemaparams_test.go +++ b/schemaparams/schemaparams_test.go @@ -80,6 +80,22 @@ func TestGenerateImportCompiles(t *testing.T) { if err != nil { t.Fatal(err) } + // The temp module mirrors THIS module's declaration: the same go directive and the + // same requirement set, with only the module path rewritten and the contract module + // added as a local replace. A minimal `require spec` makes `go build` EXPAND the + // module graph, which asks the proxy for go.mod files the repo's own build never + // needs — measured in CI, where gopkg.in/check.v1's go.mod was absent from the cache + // and GOPROXY=off refused the lookup. A complete declaration is also what a real + // consumer of this contract module has. + mod, err := os.ReadFile(filepath.Join(root, "go.mod")) + if err != nil { + t.Fatalf("read the module's go.mod: %v", err) + } + gomod := strings.Replace(string(mod), "module github.com/opencharly/spec", "module example.com/pluginparams", 1) + if gomod == string(mod) { + t.Fatalf("the contract module's go.mod does not start with its module path") + } + gomod += "\nrequire github.com/opencharly/spec v0.0.0\n\nreplace github.com/opencharly/spec => " + root + "\n" sum, err := os.ReadFile(filepath.Join(root, "go.sum")) if err != nil { t.Fatalf("read the module's go.sum (needed by the temp module's build): %v", err) @@ -91,16 +107,17 @@ func TestGenerateImportCompiles(t *testing.T) { if err := os.WriteFile(filepath.Join(tmp, "params", "cue_types_gen.go"), out, 0o644); err != nil { t.Fatal(err) } - gomod := "module example.com/pluginparams\n\ngo 1.26.0\n\nrequire github.com/opencharly/spec v0.0.0\n\nreplace github.com/opencharly/spec => " + root + "\n" if err := os.WriteFile(filepath.Join(tmp, "go.mod"), []byte(gomod), 0o644); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(tmp, "go.sum"), sum, 0o644); err != nil { t.Fatal(err) } + // No GOFLAGS: the default readonly mode uses the complete declaration above and + // never tries to rewrite it (a rewrite is what needed the proxy). cmd := exec.Command("go", "build", "./params") cmd.Dir = tmp - cmd.Env = append(os.Environ(), "GOWORK=off", "GOFLAGS=-mod=mod", "GOPROXY=off") + cmd.Env = append(os.Environ(), "GOWORK=off", "GOPROXY=off") if buildOut, err := cmd.CombinedOutput(); err != nil { t.Fatalf("the generated params do not compile against the contract module: %v\n%s\n--- generated ---\n%s", err, buildOut, out) }