From b1adedd29b9d8f586c34db4689858f40d17c3ed0 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 23 Jul 2026 13:04:45 -0400 Subject: [PATCH 1/9] feat(spector): add @surfaceDoc decorator and client surface checks Add the @surfaceDoc decorator to spector for describing expected properties of generated SDK surfaces (naming, access, hierarchy, paging, etc.) that wire tests cannot observe. Includes: - @surfaceDoc decorator definition and implementation in spector - Surface checks manifest generation (Markdown table format) - Shared language-agnostic verify-surface-checks runner - @surfaceDoc annotations on http-specs: pageable, special-words, encoded-name/json, single-discriminator, not-discriminated - Python client-criteria support (verifiers.json, context.md, verify runner) - AI orchestrator prompt for residual checks Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 54fb668c-64c3-41e5-8933-c4300c884974 --- .../.github/prompts/check-surface.prompt.md | 104 ++++ .../eng/scripts/client-criteria/README.md | 149 ++++++ .../eng/scripts/client-criteria/context.md | 76 +++ .../scripts/client-criteria/verifiers.json | 39 ++ packages/http-specs/package.json | 3 +- .../specs/payload/pageable/main.tsp | 493 ++++++++++++++++++ .../serialization/encoded-name/json/main.tsp | 6 + .../http-specs/specs/special-words/main.tsp | 146 ++++++ .../inheritance/not-discriminated/main.tsp | 6 + .../inheritance/single-discriminator/main.tsp | 38 ++ packages/http-specs/surface-checks.md | 18 + .../generated-defs/TypeSpec.Spector.ts | 70 ++- .../TypeSpec.Spector.ts-test.ts | 11 +- packages/spector/lib/main.tsp | 97 ++++ packages/spector/package.json | 1 + .../src/actions/generate-surface-checks.ts | 29 ++ .../src/actions/verify-surface-checks.ts | 489 +++++++++++++++++ packages/spector/src/cli/cli.ts | 103 +++- .../src/coverage/surface-checks-manifest.ts | 228 ++++++++ packages/spector/src/lib/decorators.ts | 279 ++++++++++ packages/spector/src/lib/index.ts | 8 + packages/spector/src/lib/tsp-index.ts | 3 +- packages/spector/test/surface-docs.test.ts | 323 ++++++++++++ .../test/verify-surface-checks.test.ts | 336 ++++++++++++ 24 files changed, 3036 insertions(+), 19 deletions(-) create mode 100644 packages/http-client-python/.github/prompts/check-surface.prompt.md create mode 100644 packages/http-client-python/eng/scripts/client-criteria/README.md create mode 100644 packages/http-client-python/eng/scripts/client-criteria/context.md create mode 100644 packages/http-client-python/eng/scripts/client-criteria/verifiers.json create mode 100644 packages/http-specs/surface-checks.md create mode 100644 packages/spector/src/actions/generate-surface-checks.ts create mode 100644 packages/spector/src/actions/verify-surface-checks.ts create mode 100644 packages/spector/src/coverage/surface-checks-manifest.ts create mode 100644 packages/spector/test/surface-docs.test.ts create mode 100644 packages/spector/test/verify-surface-checks.test.ts diff --git a/packages/http-client-python/.github/prompts/check-surface.prompt.md b/packages/http-client-python/.github/prompts/check-surface.prompt.md new file mode 100644 index 00000000000..27b7f63a809 --- /dev/null +++ b/packages/http-client-python/.github/prompts/check-surface.prompt.md @@ -0,0 +1,104 @@ +--- +description: "Orchestrate surface checks: one parallel deterministic batch, then AI for the rest" +mode: "agent" +--- + +You are the **orchestrator** for surface checks on an emitter's generated +SDK — properties (naming, visibility/access, shape) the wire tests can't see. +You verify every check in **two parallel phases**, not one-by-one: + +1. **Deterministic phase (parallel, no AI).** One batch call runs every check + whose category has a routine — concurrently. +2. **AI phase (only the leftovers).** You verify the few checks with no + deterministic routine; dispatch them in parallel if there are several. + +Read: + +- `packages/${input:emitter}/eng/scripts/client-criteria/context.md` — Emitter + facts (language, generated-root, flavors, checks-doc path) and concept prose. +- The **checks doc** — the `checks-doc` Markdown table named in `context.md` + (default `surface-checks.md`): the batch runner reads it for deterministic + checks and you read its rows' `doc` prose for the AI ones. Use this path unless the caller + explicitly overrides it. +- `packages/${input:emitter}/eng/scripts/client-criteria/verifiers.json` — the + routing table, **authored once per emitter (by you or a human) — never + generated by the emitter build.** A category listed here is decided without + AI, either by a `check` (a declarative pattern the runner matches) or by a `test` + (defer to an existing emitter test that already asserts this surface, e.g. a + Java reflection test). A category not listed is verified by AI. Even with **no** + verifiers.json at all you can still verify every check from `context.md` + + the checks doc — verifiers.json only moves work from AI to code. + +Inputs: + +- Emitter: ${input:emitter:package folder, e.g. http-client-python} +- Flavor: ${input:flavor:one of the flavors in the emitter's context.md} +- Scenario: ${input:scenario:optional substring filter on the check id} + +## Phase 1 — deterministic batch (parallel) + +Run the shared runner **once**. It is a single language-agnostic tool +(`tsp-spector verify-surface-checks`) that runs every check whose category has a +verifier — in parallel, no AI — and prints `{"results": [...], "needs_ai": [...]}`: + +``` +npx tsp-spector verify-surface-checks \ + --checks packages/${input:emitter}/eng/scripts/client-criteria/ \ + --verifiers packages/${input:emitter}/eng/scripts/client-criteria/verifiers.json \ + --generated-root \ + --flavor ${input:flavor} --language +``` + +Take `results` as-is — each is a verdict decided without AI: `how: deterministic` +(a declarative pattern matched the generated code), `how: test`, +`status: deferred` (a real emitter test already asserts this surface; the +`evidence` cites it), or `how: not-applicable`, `status: not-applicable` (the +emitter declared this category N/A for its language in `verifiers.json`). Do not +re-check or second-guess them. + +## Phase 2 — AI verification (the residual, in parallel) + +For each item in `needs_ai`, verify it yourself: find the target in the generated +package under `//` (folder matching the scenario), judge +against its `verify` text and context.md, and cite the exact file + line. Mark +`how: ai`. **If there are several, dispatch them as parallel subagents** (one per +item) rather than serially, then collect the verdicts. + +## Phase 3 — merge + +Combine `results` + your AI verdicts into one table, filtered by the scenario +input if given: + +`id | category | result (PASS/FAIL/N/A/DEFERRED) | how (deterministic/test/ai) | evidence` + +Render `status: deferred` rows as `DEFERRED` with `how: test` — they are covered +by the referenced test rather than checked here. Render `status: not-applicable` +rows as `N/A` with `how: not-applicable` — the category has no surface in this +language. + +End with a summary: pass/fail counts, and how many were code vs test vs AI. + +## Phase 4 — propose promotions (optional; only write when asked) + +For every item you had to verify with **AI** in Phase 2, consider whether it +could be decided without AI next time, and if so propose a `verifiers.json` +entry for its category. Two forms: + +- **`check`** — the emitter's `context.md` describes the concept concretely + enough to be a mechanical file check. Propose a declarative pattern entry + `{"": {"files": ["…"], "find": "…{target}…", "expect": "…"}}` + (placeholders/casing/expect as documented in `verifiers.json`). +- **`test`** — an existing test in the emitter repo already asserts this surface + (e.g. a reflection/`assert`-based test). Propose + `{"": {"test": "", "note": "…"}}` so the + check defers to that test instead of AI. + +Print the proposed JSON entries under a **"Suggested verifiers.json additions"** +heading with a one-line rationale each. verifiers.json is authored, rarely +changes, and is never generated by the emitter — so **only write the file if the +caller explicitly asks**; otherwise leave the suggestions for review. A verify +run is read-only by default. + +The point: the bulk is decided without AI — a py routine or an existing test — +and AI is spent only on the checks that actually need judgment, shrinking further +as verifiers.json is filled in. diff --git a/packages/http-client-python/eng/scripts/client-criteria/README.md b/packages/http-client-python/eng/scripts/client-criteria/README.md new file mode 100644 index 00000000000..dad5454d0b2 --- /dev/null +++ b/packages/http-client-python/eng/scripts/client-criteria/README.md @@ -0,0 +1,149 @@ +# Client surface checks for generated SDKs + +_Verify properties of generated SDKs that the wire tests can't observe — +described once in the spec, precomputed centrally, checked per emitter. Mostly +deterministic; AI only for the fuzzy residual._ + +Full design write-up: https://gist.github.com/iscai-msft/8be1b3ecf016bfd3bfbf3a7f6857d78c + +## What this is + +Spector verifies emitter parity on the **wire**. A large class of behavior never +reaches the wire — whether an internal type is actually hidden, a property was +renamed, a method relocated, a model flattened. Today those are checked by hand, +per language. + +**Client surface checks** close that gap. On a spec element that already carries +`@scenarioDoc`, you add `@surfaceDoc(category, subject, expected, doc?)` — an +explicit, language-agnostic assertion: the `category` of surface property, the +`subject` type/member it's about, and the `expected` client-facing output. +`expected` is normally a single canonical string that each emitter **recasts** +to its idiomatic form; when a name can't be derived by casing, it can instead be +a `scope → value` dict (e.g. `#{ python: "io_thing", csharp: "IOThing" }`) whose +values are matched **verbatim** for their language. +Nothing is inferred from other decorators, so a check is deterministic to read +and grounded in a real scenario. A precompute step collects every `@surfaceDoc` +into a shared **checks doc**. Each emitter supplies a **context.md** (what those +concepts look like in its output) and a declarative **verifiers.json** (file +globs + regex patterns per category). A single shared, language-agnostic runner +(`tsp-spector verify-surface-checks`) decides every check whose category has a +verifier; everything else goes to the `/check-surface` orchestrator prompt (AI). + +Whether a check is deterministic or AI is decided by whether the language +supplied a verifier for its category — not by the scenario. Add a verifier and +the same check is promoted from AI to deterministic; the spec assertion never +changes. A category may instead **defer to an existing emitter test** that +already asserts the surface (`{ "test": "…" }`) — no pattern, no AI. + +## The pieces + +| Piece | Lives in | AI? | +| ------------------------------------------------------ | -------------------------------------- | ------------------------ | +| `@surfaceDoc` decorator | spector (requires `@scenarioDoc`) | no | +| checks doc (precomputed Markdown table) | generated by spector, ships with specs | no | +| `context.md` + `verifiers.json` (concept descriptions) | each emitter repo (authored) | no (authored) | +| `verify-surface-checks` (shared runner) | spector `tsp-spector` CLI (all langs) | no | +| `/check-surface` orchestrator | monorepo `.github/prompts/` | AI only for the residual | + +Work is pushed left so AI is the exception: (1) the `@surfaceDoc` manifest +walk is deterministic; (2) concept descriptions are authored once per language, +~a dozen entries reused across all scenarios; (3) the shared runner decides every +item with a verifier (a declarative pattern, or a deferral to an existing test); +(4) AI handles only the residual, cached on +`(item id + generated-file hash + model version)` and scoped to changed packages. + +## Categories + +A check's `category` says what kind of surface property is asserted; `subject` +and `expected` say about-what and to-what. `SurfaceCategory` is an **extensible** +union — the named members are the canonical, tight set, but any string is +accepted. + +| category | `subject` is… | `expected` is… | +| ---------------- | ---------------------------- | --------------------------------------------------------------------------------------------- | +| `naming` | renamed type/member/op/param | canonical client identifier (recast per language, or a `scope → value` dict matched verbatim) | +| `access` | type/operation | `"internal"` or `"public"` | +| `clientLocation` | operation | client/group name it should be surfaced on | +| `hierarchy` | derived model | base type's client name | +| `flatten` | the flattened property | promoted property name(s); blank = just inlined | +| `paging` | operation | item/element type the pager yields | +| `other` | anything | free-form; verified against the prose (AI) | + +The precompute step derives one fixed, category-agnostic set of placeholders for +**every** category — so the core never needs to know what a category means: + +| placeholder | derived from | +| ------------ | ----------------------------------- | +| `{target}` | the `subject`'s name | +| `{expected}` | the `expected` argument | +| `{kind}` | the `subject`'s symbol kind | +| `{origin}` | the `subject`'s declaring container | + +### Adding a category + +The category is just a routing key, so a new one is expandable without touching +core spector: + +1. Optionally add a member to the `SurfaceCategory` union in spector for + autocomplete/discoverability (a one-line, backward-compatible change). +2. In each emitter that wants it deterministic, add a `verifiers.json` entry + (using `{target}`/`{expected}`/`{kind}`/`{origin}`) and a `context.md` blurb. + +An emitter that adds **nothing** for a category still works: it falls back to +AI-verifying the prose — exactly how `other` behaves. So promoting a check from +`other` to a specific category (a central spec edit) never requires any emitter +to change; each opts into determinism at its own pace. + +One call (Copilot Chat agent mode or CLI): + +``` +/check-surface → emitter: http-client-python flavor: azure scenario: (blank = all) +``` + +The prompt is an **orchestrator**: in one agent session it runs the shared +`tsp-spector verify-surface-checks` once (every category with a verifier, decided +without AI), then verifies only the residual itself (AI). It prints one merged +table marking each row `deterministic`, `test`, `not-applicable`, or `ai`. + +## Onboarding a language + +One required file: **context.md** under +`packages//eng/scripts/client-criteria/` — Emitter facts (language, +generated-root, flavors, checks-doc path) plus a **concept description** per +category, written concretely enough to execute as a file check. Optionally add +**verifiers.json** — a declarative table mapping each category to file globs + +regex patterns (or a deferral to an existing test) for a zero-AI path in CI, +consumed by the shared `tsp-spector verify-surface-checks` runner. A new emitter +writes no assertions, no per-scenario criteria, and **no runner code** — the +decorator, checks doc, runner, and orchestrator are all shared; only config +differs per language. + +### Marking a category not-applicable + +When a surface concept has no equivalent in a language (e.g. a language with no +class inheritance can't have a `hierarchy` surface), the emitter marks that +**category** N/A in its `verifiers.json` — mirroring how wire scenarios are +marked N/A via `unsupportedScenarios`. It's emitter-owned, needs no pattern, and +never reaches AI; every check in the category resolves to `not-applicable`: + +```json +{ + "hierarchy": { "na": true, "note": "no class inheritance in this language" } +} +``` + +## Resolved + +- **Where the precomputed checks doc ships:** alongside the specs, exactly like + the scenario summary. `packages/http-specs` runs + `tsp-spector generate-surface-checks ./specs` (wired into `regen-docs`) to + produce a committed `surface-checks.md`, the same way + `generate-scenarios-summary` produces `spec-summary.md`. +- **How items mark N/A for a language:** per **category** in the emitter's + `verifiers.json` (`{ "na": true }`), mirroring wire `unsupportedScenarios`; + the runner reports `not-applicable` (see above). + +## Open questions + +- Should the manifest record the **resolved per-language symbol name** (so + renamed-and-access-controlled types stay mechanically checkable)? diff --git a/packages/http-client-python/eng/scripts/client-criteria/context.md b/packages/http-client-python/eng/scripts/client-criteria/context.md new file mode 100644 index 00000000000..0022df8e582 --- /dev/null +++ b/packages/http-client-python/eng/scripts/client-criteria/context.md @@ -0,0 +1,76 @@ +# Client surface checks — context: http-client-python + +## Emitter facts + +_The shared `/check-surface` prompt and `tsp-spector verify-surface-checks` read these._ + +- **language:** python _(selects which per-language client name applies when a check is language-scoped)_ +- **generated-root:** `packages/http-client-python/tests/generated/` +- **flavors:** `azure`, `unbranded` +- **checks-doc:** `packages/http-specs/surface-checks.md` _(generated from `@surfaceDoc`, ships with the specs)_ + + +### What differs by flavor (branded = azure, unbranded) + +Only the **generated-root** subfolder and the **import namespace** (`azure.core` +vs `corehttp`). Symbol locations and every concept description below are +flavor-invariant. + +## What each check looks like + +What each verifiable concept looks like in generated Python. Authored once, +reused across every scenario that uses the concept. Prefer concrete, checkable +verifiers so the shared deterministic runner can decide **without calling AI** — +see `verifiers.json` for the machine-readable declarative form (file globs + +regex patterns + casing) consumed by `tsp-spector verify-surface-checks`. + +### access: internal + +- **enum** → the class is NOT re-exported from `models/__init__.py`; it may also + be defined with a leading `_`. +- **model** → absent from the package's public `__init__.py` exports. +- **operation / method** → prefixed `_`, or not present on the public client. + +### access: public + +- present in the public `__init__.py` exports, no leading underscore. + +### naming + +- the generated identifier equals the `expected` client name **recast to + Python's idiomatic casing for the subject's kind** (case-sensitive): + `enum`/`model`/`type` → `PascalCase`, `property`/`parameter`/`operation` → + `snake_case`, enum value → `UPPER_SNAKE`. So an enum whose `expected` is + `ClientExtensibleEnum` must appear exactly as `ClientExtensibleEnum` (not + `client_extensible_enum`). The per-kind casing map lives in `verifiers.json` + (`{expected:byKind}`); the manifest carries the subject's `kind` under + `details`. +- **exact per-language names:** when a name can't be derived by casing (a + reserved-word escape, an acronym, a hard rename), the spec author supplies a + `scope → value` dict on `@surfaceDoc` (e.g. `#{ python: "io_thing" }`). Those + checks carry a `scope` column; the engine only runs the ones scoped to + `python` and matches `{expected}` **verbatim** (the `:byKind` recast is + skipped). The same `naming` routine handles both — no `verifiers.json` change. + +### flatten + +- the nested model's fields appear directly as attributes on the parent class; + `expected` names the promoted property (blank = assert inlined). + +### client-location + +- an operation group is a class under `operations/_operations.py`; a method is a + public `def` on it; the **root client** is the main `*Client` class. An + operation is "on" a client/group when its snake_case name is a method of the + matching class. Deterministic: assert the method is present on the `expected` + client/group and absent from its `origin` (the one it moved from). + +### hierarchy + +- a client subtype is expressed as the base in `class (, …)` + in `models/_models.py`; inherited members are attributes present on the + subclass. + +> When an assertion has no concrete verifier here (and none in verifiers.json), +> `/check-surface` may fall back to judgment — but add a verifier when +> you can, to keep AI calls rare. diff --git a/packages/http-client-python/eng/scripts/client-criteria/verifiers.json b/packages/http-client-python/eng/scripts/client-criteria/verifiers.json new file mode 100644 index 00000000000..7702f1d5b05 --- /dev/null +++ b/packages/http-client-python/eng/scripts/client-criteria/verifiers.json @@ -0,0 +1,39 @@ +{ + "_comment": "Declarative routing table for the shared `tsp-spector verify-surface-checks` engine. AUTHORED once per emitter (by an agent or a human) — NOT generated by the emitter build, and NOT language-specific code: the engine is language-agnostic, all the Python specifics live here as file globs + regex templates + a casing map. A category listed here is decided without AI; one not listed is verified by AI against the prose. The engine substitutes a fixed, category-agnostic set of placeholders into `find`/`scope`: {target}=the subject symbol's name, {expected}=the author's expected client-surface output, {kind}=the subject's symbol kind, {origin}=the subject's declaring container. Add `:casing` to recase, e.g. {expected:byKind} (per-kind map below), {target:snake}. `expect`: present (default), absent, {absentWhen:}, or {whenExpected:{:present|absent}} (branch on {expected}). SCOPE: when a check is scoped to a language (authored as a `scope → value` dict on `@surfaceDoc`), the engine (1) only runs it for the matching language and (2) matches {expected} VERBATIM — casing modifiers like `:byKind` are ignored — so a per-language exact name (a reserved-word escape, an acronym, a hard rename) needs NO verifier change here; the same `naming` routine below handles both idiomatic and exact.", + "_promotable": "A category with a routine here is deterministic; leaving it OUT makes it AI-verified (against the prose) until added. Adding an entry (a pattern like `hierarchy` below) flips its checks from AI to deterministic with no change to the spec — and no other emitter has to do anything. An agent can author these entries from context.md.", + "_test_verifier_example": "Instead of a pattern, a category may defer to an existing test that already asserts this surface: { \"exactName\": { \"test\": \"emitter-tests/src/test/java/.../ExactNameTests.java#L34-L35\", \"note\": \"reflection asserts the exact generated name\" } }. Reported status 'deferred', how 'test' (no pattern, no AI).", + "_not_applicable_example": "If a surface concept has no equivalent in this language, mark the whole category N/A (mirrors wire `unsupportedScenarios`): { \"hierarchy\": { \"na\": true, \"note\": \"no class inheritance in this language\" } }. Every check in the category is reported status 'not-applicable', how 'not-applicable' (no pattern, no AI).", + "naming": { + "files": ["**/*.py"], + "find": "\\b{expected:byKind}\\b", + "casing": { + "type": "pascal", + "model": "pascal", + "enum": "pascal", + "property": "snake", + "parameter": "snake", + "operation": "snake", + "enumvalue": "upper_snake" + } + }, + "access": { + "files": ["**/models/__init__.py"], + "find": "\\b{target}\\b", + "expect": { "whenExpected": { "internal": "absent", "public": "present" } } + }, + "hierarchy": { + "files": ["**/models/_models.py"], + "find": "class\\s+{target}\\s*\\([^)]*\\b{expected}\\b" + }, + "client-location": { + "files": ["**/operations/_operations.py"], + "scope": "class\\s+\\w*{expected}\\w*[\\s\\S]*?(?=\\nclass |$)", + "find": "def\\s+{target:snake}\\b", + "also": { + "files": ["**/operations/_operations.py"], + "scope": "class\\s+\\w*{origin}\\w*[\\s\\S]*?(?=\\nclass |$)", + "find": "def\\s+{target:snake}\\b", + "expect": "absent" + } + } +} diff --git a/packages/http-specs/package.json b/packages/http-specs/package.json index 70902b16ac2..7efba3be297 100644 --- a/packages/http-specs/package.json +++ b/packages/http-specs/package.json @@ -13,7 +13,8 @@ "ci": "prettier specs --write && pnpm validate:all", "validate-scenarios": "tsp-spector validate-scenarios ./specs", "generate-scenarios-summary": "tsp-spector generate-scenarios-summary ./specs", - "regen-docs": "pnpm generate-scenarios-summary", + "generate-surface-checks": "tsp-spector generate-surface-checks ./specs", + "regen-docs": "pnpm generate-scenarios-summary && pnpm generate-surface-checks", "upload-manifest": "tsp-spector upload-manifest ./specs --setName @typespec/http-specs --containerName manifests-typespec --storageAccountName typespec", "upload-coverage": "tsp-spector upload-coverage --generatorName @typespec/http-specs --generatorVersion 0.1.0-alpha.4 --containerName coverages --generatorMode standard --storageAccountName typespec", "validate-mock-apis": "tsp-spector validate-mock-apis ./specs", diff --git a/packages/http-specs/specs/payload/pageable/main.tsp b/packages/http-specs/specs/payload/pageable/main.tsp index f123c65954c..6230df7b5b6 100644 --- a/packages/http-specs/specs/payload/pageable/main.tsp +++ b/packages/http-specs/specs/payload/pageable/main.tsp @@ -1,8 +1,10 @@ import "@typespec/http"; import "@typespec/spector"; +import "@typespec/xml"; using Http; using Spector; +using Xml; /** * Test for pageable payload. @@ -49,6 +51,12 @@ namespace ServerDrivenPagination { } ``` """) + @surfaceDoc( + "paging", + link, + "Pet", + "The client surfaces this operation as a pageable collection that yields `Pet` items across pages." + ) @route("/link") @list op link(): { @@ -58,6 +66,146 @@ namespace ServerDrivenPagination { @nextLink next?: url; }; + @scenario + @scenarioDoc(""" + Test case for using link as pagination with string nextLink. + + Two requests need to be tested. + 1. Initial request: + Expected route: /payload/pageable/server-driven-pagination/link-string + Expected response body: + ```json + { "pets": [ + { "id": "1", "name": "dog" }, + { "id": "2", "name": "cat" } + ], + "next": "http://[host]:[port]/payload/pageable/server-driven-pagination/link-string/nextPage" + } + ``` + 2. Next page request: + Expected route: /payload/pageable/server-driven-pagination/link-string/nextPage + Expected response body: + ```json + { "pets": [ + { "id": "3", "name": "bird" }, + { "id": "4", "name": "fish" } + ] + } + ``` + """) + @surfaceDoc( + "paging", + linkString, + "Pet", + "The client surfaces this operation as a pageable collection that yields `Pet` items, following a string next-link." + ) + @route("/link-string") + @list + op linkString(): { + @pageItems + pets: Pet[]; + + @nextLink next?: string; + }; + + @scenario + @scenarioDoc(""" + Test case for using link as pagination with nested structure. + + Two requests need to be tested. + 1. Initial request: + Expected route: /payload/pageable/server-driven-pagination/nested-link + Expected response body: + ```json + { "nestedItems": { + "pets": [ + { "id": "1", "name": "dog" }, + { "id": "2", "name": "cat" } + ] + }, + "nestedNext": { + "next": "http://[host]:[port]/payload/pageable/server-driven-pagination/nested-link/nextPage" + } + } + ``` + 2. Next page request: + Expected route: /payload/pageable/server-driven-pagination/nested-link/nextPage + Expected response body: + ```json + { "nestedItems": { + "pets": [ + { "id": "3", "name": "bird" }, + { "id": "4", "name": "fish" } + ] + } + } + ``` + """) + @surfaceDoc( + "paging", + nestedLink, + "Pet", + "The client surfaces this operation as a pageable collection of `Pet`, flattening the nested items/next-link envelope." + ) + @route("/nested-link") + @list + op nestedLink(): { + nestedItems: { + @pageItems + pets: Pet[]; + }; + nestedNext: { + @nextLink next?: url; + }; + }; + + /** Scenario where the initial request is not a GET request. However following the next link always result in a GET request. */ + namespace AlternateInitialVerb { + model Filter { + filter: string; + } + + @scenario + @scenarioDoc(""" + Test case for initial POST request followed by GET for next pages using link pagination. + The initial request is a POST with a filter body, and the next page is fetched using a GET request on the next link. + + Two requests need to be tested. + 1. Initial request (POST): + Expected route: /payload/pageable/server-driven-pagination/link/initial-post + Expected request body: + ```json + { "filter": "foo eq bar" } + ``` + Expected response body: + ```json + { "pets": [ + { "id": "1", "name": "dog" }, + { "id": "2", "name": "cat" } + ], + "next": "http://[host]:[port]/payload/pageable/server-driven-pagination/link/initial-post/nextPage?token=abc" + } + ``` + 2. Next page request (GET): + Expected route: /payload/pageable/server-driven-pagination/link/initial-post/nextPage?token=abc + Expected response body: + ```json + { "pets": [ + { "id": "3", "name": "bird" }, + { "id": "4", "name": "fish" } + ] + } + ``` + """) + @route("/link/initial-post") + @list + @post + op post(@body body: Filter): { + @pageItems pets: Pet[]; + @nextLink next?: url; + }; + } + @route("/continuationtoken") namespace ContinuationToken { @scenario @@ -97,6 +245,12 @@ namespace ServerDrivenPagination { } ``` """) + @surfaceDoc( + "paging", + requestQueryResponseBody, + "Pet", + "The client surfaces this operation as a pageable collection of `Pet`, transparently threading the continuation token across pages." + ) @route("/request-query-response-body") @list op requestQueryResponseBody(@continuationToken @query token?: string, ...HeaderAndQuery): { @@ -248,5 +402,344 @@ namespace ServerDrivenPagination { @continuationToken @header nextToken?: string; }; + + @scenario + @scenarioDoc(""" + Test case for using continuation token as pagination with nested response structure. Continuation token is passed in the request query and nested within response body. + + Two requests need to be tested. + + 1. Initial request: + Expected route: /payload/pageable/server-driven-pagination/continuationtoken/request-query-nested-response-body?bar=bar + + Expected request header: + foo=foo + + Expected response body: + ```json + { "nestedItems": { + "pets": [ + { "id": "1", "name": "dog" }, + { "id": "2", "name": "cat" } + ] + }, + "nestedNext": { + "nextToken": "page2" + } + } + ``` + + 2. Next page request: + Expected route: /payload/pageable/server-driven-pagination/continuationtoken/request-query-nested-response-body?bar=bar&token=page2 + + Expected request header: + foo=foo + + Expected response body: + ```json + { "nestedItems": { + "pets": [ + { "id": "3", "name": "bird" }, + { "id": "4", "name": "fish" } + ] + } + } + ``` + """) + @route("/request-query-nested-response-body") + @list + op requestQueryNestedResponseBody( + @continuationToken @query token?: string, + ...HeaderAndQuery, + ): { + nestedItems: { + @pageItems + pets: Pet[]; + }; + nestedNext?: { + @continuationToken nextToken?: string; + }; + }; + + @scenario + @scenarioDoc(""" + Test case for using continuation token as pagination with nested response structure. Continuation token is passed in the request header and nested within response body. + + Two requests need to be tested. + + 1. Initial request: + Expected route: /payload/pageable/server-driven-pagination/continuationtoken/request-header-nested-response-body?bar=bar + + Expected request header: + foo=foo + + Expected response body: + ```json + { "nestedItems": { + "pets": [ + { "id": "1", "name": "dog" }, + { "id": "2", "name": "cat" } + ] + }, + "next": { + "nextToken": "page2" + } + } + ``` + + 2. Next page request: + Expected route: /payload/pageable/server-driven-pagination/continuationtoken/request-header-nested-response-body?bar=bar + + Expected request header: + token=page2 + foo=foo + + Expected response body: + ```json + { "nestedItems": { + "pets": [ + { "id": "3", "name": "bird" }, + { "id": "4", "name": "fish" } + ] + } + } + ``` + """) + @route("/request-header-nested-response-body") + @list + op requestHeaderNestedResponseBody( + @continuationToken @header token?: string, + ...HeaderAndQuery, + ): { + nestedItems: { + @pageItems + pets: Pet[]; + }; + nestedNext?: { + @continuationToken nextToken?: string; + }; + }; } } + +@route("/pagesize") +namespace PageSize { + @scenario + @scenarioDoc(""" + Test case for simple pagination without nextlink or continuationToken. + + Single request: + Expected route: /payload/pageable/pagesize/without-continuation + + Expected response body: + ```json + { "pets": [ + { "id": "1", "name": "dog" }, + { "id": "2", "name": "cat" }, + { "id": "3", "name": "bird" }, + { "id": "4", "name": "fish" } + ] + } + ``` + """) + @surfaceDoc( + "paging", + listWithoutContinuation, + "Pet", + "The client surfaces this single-page operation as a pageable collection of `Pet`." + ) + @route("/without-continuation") + @list + op listWithoutContinuation(): { + @pageItems + pets: Pet[]; + }; + + @scenario + @scenarioDoc(""" + Test case for pagination with a regular @pageSize parameter. + + Two requests need to be tested: + 1. Request with pageSize=2: + Expected route: /payload/pageable/pagesize/list?pageSize=2 + + Expected response body: + ```json + { "pets": [ + { "id": "1", "name": "dog" }, + { "id": "2", "name": "cat" } + ] + } + ``` + + 2. Request with pageSize=4: + Expected route: /payload/pageable/pagesize/list?pageSize=4 + + Expected response body: + ```json + { "pets": [ + { "id": "1", "name": "dog" }, + { "id": "2", "name": "cat" }, + { "id": "3", "name": "bird" }, + { "id": "4", "name": "fish" } + ] + } + ``` + """) + @surfaceDoc( + "paging", + listWithPageSize, + "Pet", + "The client surfaces this operation as a pageable collection of `Pet` with a page-size parameter." + ) + @route("/list") + @list + op listWithPageSize(@pageSize @query pageSize?: int32): { + @pageItems + pets: Pet[]; + }; +} + +@doc("An XML pet item.") +@name("Pet") +model XmlPet { + @name("Id") id: string; + @name("Name") name: string; +} + +@route("/xml") +namespace XmlPagination { + @scenario + @scenarioDoc(""" + Test case for XML pagination with continuation token. Continuation token is passed in the request query and response body. + + Two requests need to be tested. + + 1. Initial request: + Expected route: /payload/pageable/xml/list + + Expected response body: + ```xml + + + + 1 + dog + + + 2 + cat + + + page2 + + ``` + + 2. Next page request: + Expected route: /payload/pageable/xml/list?marker=page2 + + Expected response body: + ```xml + + + + 3 + bird + + + 4 + fish + + + + ``` + """) + @route("/list-with-continuation") + @list + op listWithContinuation(@continuationToken @query marker?: string): { + @header("content-type") contentType: "application/xml"; + @body body: XmlPetListResult; + }; + + @scenario + @scenarioDoc(""" + Test case for XML pagination with next link. + + Two requests need to be tested. + + 1. Initial request: + Expected route: /payload/pageable/xml/list-with-next-link + + Expected response body: + ```xml + + + + 1 + dog + + + 2 + cat + + + http://[host]:[port]/payload/pageable/xml/list-with-next-link/nextPage + + ``` + + 2. Next page request: + Expected route: /payload/pageable/xml/list-with-next-link/nextPage + + Expected response body: + ```xml + + + + 3 + bird + + + 4 + fish + + + + ``` + """) + @surfaceDoc( + "paging", + listWithNextLink, + "XmlPet", + "The client surfaces this XML operation as a pageable collection that yields `XmlPet` items across pages." + ) + @route("/list-with-next-link") + @list + op listWithNextLink(): { + @header("content-type") contentType: "application/xml"; + @body body: XmlPetListResultWithNextLink; + }; +} + +@doc("The XML response for listing pets.") +@name("PetListResult") +model XmlPetListResult { + @pageItems + @name("Pets") + pets: XmlPet[]; + + @continuationToken + @name("NextMarker") + nextMarker?: string; +} + +@doc("The XML response for listing pets with next link.") +@name("PetListResult") +model XmlPetListResultWithNextLink { + @pageItems + @name("Pets") + pets: XmlPet[]; + + @nextLink + @name("NextLink") + nextLink?: url; +} diff --git a/packages/http-specs/specs/serialization/encoded-name/json/main.tsp b/packages/http-specs/specs/serialization/encoded-name/json/main.tsp index 8d1d0e006f9..349842738ce 100644 --- a/packages/http-specs/specs/serialization/encoded-name/json/main.tsp +++ b/packages/http-specs/specs/serialization/encoded-name/json/main.tsp @@ -26,6 +26,12 @@ namespace Property { {"wireName": true} ``` """) + @surfaceDoc( + "naming", + JsonEncodedNameModel.defaultName, + "defaultName", + "The client property is named `defaultName` even though the wire name is `wireName`." + ) @post op send(@bodyRoot body: JsonEncodedNameModel): NoContentResponse; diff --git a/packages/http-specs/specs/special-words/main.tsp b/packages/http-specs/specs/special-words/main.tsp index 7517d04be5a..dd1c39ce7ad 100644 --- a/packages/http-specs/specs/special-words/main.tsp +++ b/packages/http-specs/specs/special-words/main.tsp @@ -34,6 +34,7 @@ using Spector; * in * is * lambda + * list * not * or * pass @@ -231,4 +232,149 @@ namespace ModelProperties { """) @route("same-as-model") op sameAsModel(@body body: SameAsModel): void; + + // Python dict method names that could conflict + model DictMethods { + keys: string; + items: string; + values: string; + popitem: string; + clear: string; + update: string; + setdefault: string; + pop: string; + get: string; + copy: string; + } + + @scenario + @scenarioDoc(""" + Verify that model properties can use names that are Python dict methods. These names (keys, items, values, etc.) may conflict with Python's dict class methods. + + Send + + ```json + { + "keys": "ok", + "items": "ok", + "values": "ok", + "popitem": "ok", + "clear": "ok", + "update": "ok", + "setdefault": "ok", + "pop": "ok", + "get": "ok", + "copy": "ok" + } + ``` + """) + @route("dict-methods") + op dictMethods(@body body: DictMethods): void; + + // Test for model property named "list" which is a reserved word in many languages + model ModelWithList { + list: string; + } + + @scenario + @scenarioDoc(""" + Verify that a property can be named "list", which is a reserved word in many languages like Python. + + Send + + ```json + {"list": "ok"} + ``` + """) + @route("list") + op withList(@body body: ModelWithList): void; +} + +/** + * Verify that operation parameters whose names match Python Mapping protocol methods + * (e.g., items, keys, values) are sent with the original name on the wire, not mangled. + */ +@route("/operations") +namespace ReservedOperationBodyParams { + @scenario + @scenarioDoc(""" + Verify that an operation parameter named "items" is sent with the key "items" on the wire, + not mangled to "items_property" or similar. + + Send + + ```json + {"items": ["item"]} + ``` + """) + @post + @route("/body-param-reserved") + op withItems(items: string[]): void; +} + +/** + * Verify enum member names that are special words. + */ +@route("/extensible-strings") +namespace ExtensibleStrings { + /** + * Verify enum member names that are special words using extensible enum (union). + */ + union ExtensibleString { + string, + and: "and", + as: "as", + assert: "assert", + async: "async", + await: "await", + break: "break", + class: "class", + constructor: "constructor", + continue: "continue", + def: "def", + del: "del", + elif: "elif", + `else`: "else", + except: "except", + exec: "exec", + finally: "finally", + for: "for", + from: "from", + global: "global", + `if`: "if", + `import`: "import", + in: "in", + `is`: "is", + lambda: "lambda", + not: "not", + or: "or", + pass: "pass", + raise: "raise", + `return`: "return", + try: "try", + while: "while", + with: "with", + yield: "yield", + } + + @scenario + @scenarioDoc(""" + Verify that enum members with special word names can be sent and received properly. + Send 'class' and expect the same value back. + """) + @surfaceDoc( + "naming", + ExtensibleString.and, + "and", + "The reserved-word enum member surfaces on the client with an idiomatic, non-conflicting identifier." + ) + @put + @route("/string") + op putExtensibleStringValue( + @header contentType: "application/json", + @body body: ExtensibleString, + ): { + @header contentType: "application/json"; + @body body: ExtensibleString; + }; } diff --git a/packages/http-specs/specs/type/model/inheritance/not-discriminated/main.tsp b/packages/http-specs/specs/type/model/inheritance/not-discriminated/main.tsp index 69c186445be..7ae7d905556 100644 --- a/packages/http-specs/specs/type/model/inheritance/not-discriminated/main.tsp +++ b/packages/http-specs/specs/type/model/inheritance/not-discriminated/main.tsp @@ -43,6 +43,12 @@ op postValid(@body input: Siamese): NoContentResponse; {"name": "abc", "age": 32, "smart": true} ``` """) +@surfaceDoc( + "hierarchy", + Siamese, + "Cat", + "The `Siamese` model extends `Cat` as its base type on the client surface." +) @route("/valid") @get op getValid(): Siamese; diff --git a/packages/http-specs/specs/type/model/inheritance/single-discriminator/main.tsp b/packages/http-specs/specs/type/model/inheritance/single-discriminator/main.tsp index 0a396d731ad..ac2f0f0f36c 100644 --- a/packages/http-specs/specs/type/model/inheritance/single-discriminator/main.tsp +++ b/packages/http-specs/specs/type/model/inheritance/single-discriminator/main.tsp @@ -49,6 +49,13 @@ model TRex extends Dinosaur { kind: "t-rex"; } +@doc("A discriminated model with no defined subtypes. The discriminator is declared but no models extend it.") +@discriminator("kind") +model Fish { + kind: string; + size: int32; +} + @scenario @route("/model") @scenarioDoc(""" @@ -58,6 +65,12 @@ model TRex extends Dinosaur { {"wingspan": 1, "kind": "sparrow"} ``` """) +@surfaceDoc( + "hierarchy", + Sparrow, + "Bird", + "The `Sparrow` model extends `Bird` as its base type on the client surface." +) @get op getModel(): Bird; @@ -170,3 +183,28 @@ op getWrongDiscriminator(): Bird; """) @get op getLegacyModel(): Dinosaur; + +@scenario +@route("/no-subtypes/model") +@scenarioDoc(""" + Generate and receive a discriminated model that has no defined subtypes. + The base model declares a discriminator but no models extend it. + Expected response body: + ```json + {"kind": "salmon", "size": 10} + ``` + """) +@get +op getNoSubtypesModel(): Fish; + +@scenario +@route("/no-subtypes/model") +@scenarioDoc(""" + Send a discriminated model that has no defined subtypes. + Expected input body: + ```json + {"kind": "salmon", "size": 10} + ``` + """) +@put +op putNoSubtypesModel(@body input: Fish): NoContentResponse; diff --git a/packages/http-specs/surface-checks.md b/packages/http-specs/surface-checks.md new file mode 100644 index 00000000000..f64af13c792 --- /dev/null +++ b/packages/http-specs/surface-checks.md @@ -0,0 +1,18 @@ +# Surface checks + +Generated from `@surfaceDoc` annotations. This table is both the human summary +and the machine-readable checks doc parsed by the shared `verify-surface-checks` runner. + +| id | scenario | category | target | scope | details | doc | +| ----------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | --------- | ------------------------ | ----- | ---------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| Payload_Pageable_PageSize_listWithoutContinuation_paging | Payload_Pageable_PageSize_listWithoutContinuation | paging | listWithoutContinuation | | expected=Pet; kind=operation; origin=PageSize | The client surfaces this single-page operation as a pageable collection of `Pet`. | +| Payload_Pageable_PageSize_listWithPageSize_paging | Payload_Pageable_PageSize_listWithPageSize | paging | listWithPageSize | | expected=Pet; kind=operation; origin=PageSize | The client surfaces this operation as a pageable collection of `Pet` with a page-size parameter. | +| Payload_Pageable_ServerDrivenPagination_ContinuationToken_requestQueryResponseBody_paging | Payload_Pageable_ServerDrivenPagination_ContinuationToken_requestQueryResponseBody | paging | requestQueryResponseBody | | expected=Pet; kind=operation; origin=ContinuationToken | The client surfaces this operation as a pageable collection of `Pet`, transparently threading the continuation token across pages. | +| Payload_Pageable_ServerDrivenPagination_link_paging | Payload_Pageable_ServerDrivenPagination_link | paging | link | | expected=Pet; kind=operation; origin=ServerDrivenPagination | The client surfaces this operation as a pageable collection that yields `Pet` items across pages. | +| Payload_Pageable_ServerDrivenPagination_linkString_paging | Payload_Pageable_ServerDrivenPagination_linkString | paging | linkString | | expected=Pet; kind=operation; origin=ServerDrivenPagination | The client surfaces this operation as a pageable collection that yields `Pet` items, following a string next-link. | +| Payload_Pageable_ServerDrivenPagination_nestedLink_paging | Payload_Pageable_ServerDrivenPagination_nestedLink | paging | nestedLink | | expected=Pet; kind=operation; origin=ServerDrivenPagination | The client surfaces this operation as a pageable collection of `Pet`, flattening the nested items/next-link envelope. | +| Payload_Pageable_XmlPagination_listWithNextLink_paging | Payload_Pageable_XmlPagination_listWithNextLink | paging | listWithNextLink | | expected=XmlPet; kind=operation; origin=XmlPagination | The client surfaces this XML operation as a pageable collection that yields `XmlPet` items across pages. | +| Serialization_EncodedName_Json_Property_send_naming | Serialization_EncodedName_Json_Property_send | naming | defaultName | | expected=defaultName; kind=property; origin=JsonEncodedNameModel | The client property is named `defaultName` even though the wire name is `wireName`. | +| SpecialWords_ExtensibleStrings_putExtensibleStringValue_naming | SpecialWords_ExtensibleStrings_putExtensibleStringValue | naming | and | | expected=and; kind=enumvalue; origin=ExtensibleString | The reserved-word enum member surfaces on the client with an idiomatic, non-conflicting identifier. | +| Type_Model_Inheritance_NotDiscriminated_getValid_hierarchy | Type_Model_Inheritance_NotDiscriminated_getValid | hierarchy | Siamese | | expected=Cat; kind=model; origin=NotDiscriminated | The `Siamese` model extends `Cat` as its base type on the client surface. | +| Type_Model_Inheritance_SingleDiscriminator_getModel_hierarchy | Type_Model_Inheritance_SingleDiscriminator_getModel | hierarchy | Sparrow | | expected=Bird; kind=model; origin=SingleDiscriminator | The `Sparrow` model extends `Bird` as its base type on the client surface. | diff --git a/packages/spector/generated-defs/TypeSpec.Spector.ts b/packages/spector/generated-defs/TypeSpec.Spector.ts index 6003f5b7864..fdfd500b6fc 100644 --- a/packages/spector/generated-defs/TypeSpec.Spector.ts +++ b/packages/spector/generated-defs/TypeSpec.Spector.ts @@ -1,5 +1,6 @@ import type { DecoratorContext, + DecoratorValidatorCallbacks, Interface, Model, Namespace, @@ -15,7 +16,7 @@ export type ScenarioServiceDecorator = ( target: Namespace, route: string, options?: Type, -) => void; +) => DecoratorValidatorCallbacks | void; /** * Mark an operation, interface or namespace as a scenario. All containing operations will be part of the same scenario. @@ -24,7 +25,7 @@ export type ScenarioDecorator = ( context: DecoratorContext, target: Namespace | Interface | Operation, name?: string, -) => void; +) => DecoratorValidatorCallbacks | void; /** * Specify documentation on how to implement this scenario. @@ -37,10 +38,73 @@ export type ScenarioDocDecorator = ( target: Namespace | Interface | Operation, doc: string, formatArgs?: Model, -) => void; +) => DecoratorValidatorCallbacks | void; + +/** + * Describe one expected property of the **generated SDK surface** — something a + * wire test can't see, such as a client rename, an access change, an operation + * relocated to another client, or a reshaped inheritance hierarchy. Mirrors + * `@scenarioDoc`, but for the surface instead of the wire. + * + * To keep surface checks deterministic and grounded in a real scenario, + * `@surfaceDoc` may only be applied to an element that also carries + * `@scenarioDoc` (a `@scenario` namespace/interface/operation). The check itself + * is fully explicit: the author states the `category`, the `subject` the check + * is about (a type/member reference — which may differ from the annotated + * operation), and the `expected` client-surface output. Nothing is inferred from + * other decorators, so adding or reading a check needs no knowledge of the + * client-generator vocabulary. + * + * The precompute step (`listSurfaceDocs`) records these language-agnostically; + * each emitter's `verifiers.json` decides how to check a category using the + * generic `{target}` (subject name), `{expected}`, `{kind}` (subject kind) and + * `{origin}` (subject's declaring container) placeholders. A category with no + * verifier for an emitter is verified against the prose by AI — so introducing a + * new category never requires any emitter to change. + * + * By default `expected` is a single canonical, language-agnostic string and each + * emitter recasts it to its idiomatic form (e.g. `{expected:byKind}`). When a + * name can't be derived by casing (a reserved-word escape, an acronym, a hard + * rename), pass a **dict mapping scope → exact value** instead — keyed the same + * way the client-generator decorators are scoped (`"python"`, `"python,csharp"`, + * `"!java"`). A dict value is matched **verbatim** for its scope; a language not + * listed in the dict is not checked. + * + * ```tsp + * // idiomatic: one canonical name, every emitter recasts it + * @surfaceDoc("naming", widgetName, "widgetName") + * + * // exact per language: each scope matched verbatim + * @surfaceDoc("naming", ioThing, #{ python: "io_thing", csharp: "IOThing" }) + * ``` + * + * @param category The kind of surface assertion. See { + * @link SurfaceCategory}. + * @param subject The type or member the check is about (may differ from `target`). + * @param expected The expected client-surface output: a single string (recast per + * language) or a `scope → value` dict (each value matched verbatim for its scope). + * @param doc Optional extra natural-language description. Markdown. + */ +export type SurfaceDocDecorator = ( + context: DecoratorContext, + target: Namespace | Interface | Operation, + category: + | string + | "naming" + | "access" + | "client-location" + | "hierarchy" + | "flatten" + | "paging" + | "other", + subject: Type, + expected: string | Record, + doc?: string, +) => DecoratorValidatorCallbacks | void; export type TypeSpecSpectorDecorators = { scenarioService: ScenarioServiceDecorator; scenario: ScenarioDecorator; scenarioDoc: ScenarioDocDecorator; + surfaceDoc: SurfaceDocDecorator; }; diff --git a/packages/spector/generated-defs/TypeSpec.Spector.ts-test.ts b/packages/spector/generated-defs/TypeSpec.Spector.ts-test.ts index 08dc02b6da3..cc404423ef3 100644 --- a/packages/spector/generated-defs/TypeSpec.Spector.ts-test.ts +++ b/packages/spector/generated-defs/TypeSpec.Spector.ts-test.ts @@ -1,5 +1,10 @@ -/** An error here would mean that the decorator is not exported or doesn't have the right name. */ +// An error in the imports would mean that the decorator is not exported or +// doesn't have the right name. + import { $decorators } from "@typespec/spector"; import type { TypeSpecSpectorDecorators } from "./TypeSpec.Spector.js"; -/** An error here would mean that the exported decorator is not using the same signature. Make sure to have export const $decName: DecNameDecorator = (...) => ... */ -const _: TypeSpecSpectorDecorators = $decorators["TypeSpec.Spector"]; + +/** + * An error here would mean that the exported decorator is not using the same signature. Make sure to have export const $decName: DecNameDecorator = (...) => ... + */ +const _decs: TypeSpecSpectorDecorators = $decorators["TypeSpec.Spector"]; diff --git a/packages/spector/lib/main.tsp b/packages/spector/lib/main.tsp index f22ab2d0aa4..a58a33a9b10 100644 --- a/packages/spector/lib/main.tsp +++ b/packages/spector/lib/main.tsp @@ -35,3 +35,100 @@ extern dec scenarioDoc( doc: valueof string, formatArgs?: Model ); + +/** + * The category of a client-surface check: what kind of generated-surface + * property is being asserted. This is an **extensible** union — the named + * members are the canonical, tight set every emitter understands, but any + * string is accepted so a new category can be introduced without changing this + * core library. A category an emitter hasn't onboarded a verifier for simply + * falls back to being verified against the prose (AI), exactly like `other`. + * + * Each category fixes what `subject` and `expected` mean: + * + * | category | `subject` is… | `expected` is… | + * | ---------------- | ---------------------------- | ------------------------------------------------ | + * | `naming` | renamed type/member/op/param | canonical client identifier (recast per language) | + * | `access` | type/operation | `"internal"` or `"public"` | + * | `clientLocation` | operation | client/group name it should be surfaced on | + * | `hierarchy` | derived model | base type's client name | + * | `flatten` | the flattened property | promoted property name(s); blank = just inlined | + * | `paging` | operation | item/element type the pager yields | + * | `other` | anything | free-form; verified against the prose (AI) | + */ +union SurfaceCategory { + string, + + /** Client-facing identifier of `subject` (recast to the language's idiomatic casing). */ + naming: "naming", + + /** Public vs internal visibility of `subject` on the client surface. */ + access: "access", + + /** Which client or operation group an operation is surfaced on. */ + clientLocation: "client-location", + + /** Base type `subject` extends on the client surface. */ + hierarchy: "hierarchy", + + /** Nested model properties inlined onto the parent model. */ + flatten: "flatten", + + /** `subject` surfaces as a pageable collection of an item type. */ + paging: "paging", + + /** No mechanical rule — verified against the prose (AI residual). */ + other: "other", +} + +/** + * Describe one expected property of the **generated SDK surface** — something a + * wire test can't see, such as a client rename, an access change, an operation + * relocated to another client, or a reshaped inheritance hierarchy. Mirrors + * `@scenarioDoc`, but for the surface instead of the wire. + * + * To keep surface checks deterministic and grounded in a real scenario, + * `@surfaceDoc` may only be applied to an element that also carries + * `@scenarioDoc` (a `@scenario` namespace/interface/operation). The check itself + * is fully explicit: the author states the `category`, the `subject` the check + * is about (a type/member reference — which may differ from the annotated + * operation), and the `expected` client-surface output. Nothing is inferred from + * other decorators, so adding or reading a check needs no knowledge of the + * client-generator vocabulary. + * + * The precompute step (`listSurfaceDocs`) records these language-agnostically; + * each emitter's `verifiers.json` decides how to check a category using the + * generic `{target}` (subject name), `{expected}`, `{kind}` (subject kind) and + * `{origin}` (subject's declaring container) placeholders. A category with no + * verifier for an emitter is verified against the prose by AI — so introducing a + * new category never requires any emitter to change. + * + * By default `expected` is a single canonical, language-agnostic string and each + * emitter recasts it to its idiomatic form (e.g. `{expected:byKind}`). When a + * name can't be derived by casing (a reserved-word escape, an acronym, a hard + * rename), pass a **dict mapping scope → exact value** instead — keyed the same + * way the client-generator decorators are scoped (`"python"`, `"python,csharp"`, + * `"!java"`). A dict value is matched **verbatim** for its scope; a language not + * listed in the dict is not checked. + * + * ```tsp + * // idiomatic: one canonical name, every emitter recasts it + * @surfaceDoc("naming", widgetName, "widgetName") + * + * // exact per language: each scope matched verbatim + * @surfaceDoc("naming", ioThing, #{ python: "io_thing", csharp: "IOThing" }) + * ``` + * + * @param category The kind of surface assertion. See {@link SurfaceCategory}. + * @param subject The type or member the check is about (may differ from `target`). + * @param expected The expected client-surface output: a single string (recast per + * language) or a `scope → value` dict (each value matched verbatim for its scope). + * @param doc Optional extra natural-language description. Markdown. + */ +extern dec surfaceDoc( + target: Namespace | Interface | Operation, + category: valueof SurfaceCategory, + subject: unknown, + expected: valueof string | Record, + doc?: valueof string +); diff --git a/packages/spector/package.json b/packages/spector/package.json index d03d190e164..61ce5f392e2 100644 --- a/packages/spector/package.json +++ b/packages/spector/package.json @@ -53,6 +53,7 @@ "morgan": "^1.10.0", "multer": "1.4.5-lts.2", "picocolors": "~1.1.1", + "prettier": "~3.5.3", "source-map-support": "~0.5.21", "xml2js": "^0.6.2", "yargs": "~17.7.2" diff --git a/packages/spector/src/actions/generate-surface-checks.ts b/packages/spector/src/actions/generate-surface-checks.ts new file mode 100644 index 00000000000..e1b7292f1d6 --- /dev/null +++ b/packages/spector/src/actions/generate-surface-checks.ts @@ -0,0 +1,29 @@ +import { writeFile } from "fs/promises"; +import pc from "picocolors"; +import { + computeSurfaceChecksManifest, + createSurfaceChecksSummary, +} from "../coverage/surface-checks-manifest.js"; +import { logger } from "../logger.js"; + +export interface GenerateSurfaceChecksConfig { + scenariosPath: string; + outputFile: string; +} + +export async function generateSurfaceChecks({ + scenariosPath, + outputFile, +}: GenerateSurfaceChecksConfig) { + logger.info(`Computing surface checks manifest for ${scenariosPath}`); + const [manifest, diagnostics] = await computeSurfaceChecksManifest(scenariosPath); + if (manifest === undefined || diagnostics.length > 0) { + process.exit(-1); + } + + const summary = await createSurfaceChecksSummary(manifest); + await writeFile(outputFile, summary); + logger.info( + `${pc.green("✓")} Surface checks doc generated at ${outputFile} (${manifest.items.length} checks).`, + ); +} diff --git a/packages/spector/src/actions/verify-surface-checks.ts b/packages/spector/src/actions/verify-surface-checks.ts new file mode 100644 index 00000000000..9c1133e8551 --- /dev/null +++ b/packages/spector/src/actions/verify-surface-checks.ts @@ -0,0 +1,489 @@ +import { glob, readFile, readdir } from "fs/promises"; +import { join } from "path"; + +/** + * Shared, language-agnostic runner for client-surface checks. + * + * The wire-format Spector tests can't observe surface properties (naming, + * visibility, shape, hierarchy). `@surfaceDoc` describes them once in the spec; + * `generate-surface-checks` precomputes a checks doc; each emitter authors a + * **declarative** `verifiers.json` that tells this engine, per category, which + * files to look in and what pattern proves the surface is correct. Nothing here + * is language-specific: Python/Java/C#/... all share this engine and differ only + * in their `verifiers.json` (globs + regex templates + casing) and generated-root. + * + * A category may instead defer to an existing emitter test + * (`{ "test": "…" }`) — reported `deferred`/`test`, no pattern and no AI. A + * category with no verifier at all is left for the AI orchestrator (`needs_ai`). + */ + +// --------------------------------------------------------------------------- +// Checks doc +// --------------------------------------------------------------------------- + +export interface CheckItem { + id: string; + scenario?: string; + category: string; + target: string; + /** + * Language scope, e.g. `python` or `!java`. Set only for verbatim per-language + * checks; empty means all languages (idiomatic recast). + */ + scope?: string; + details: Record; + doc: string; + /** Optional per-language resolved names, e.g. `{ "python": "…" }`. */ + client_names?: Record; +} + +/** Parse the checks doc — a Markdown table (the shipped format) or JSON. */ +export function parseChecksDoc(text: string): CheckItem[] { + const trimmed = text.trimStart(); + if (trimmed.startsWith("{") || trimmed.startsWith("[")) { + const data = JSON.parse(text); + return Array.isArray(data) ? data : data.items; + } + const rows: CheckItem[] = []; + let headerSeen = false; + for (const raw of text.split(/\r?\n/)) { + const line = raw.trim(); + if (!line.startsWith("|")) continue; + const cells = line + .replace(/^\|/, "") + .replace(/\|$/, "") + .split(/(? c.trim().replace(/\\\|/g, "|")); + if (!headerSeen) { + if (cells[0] === "id") headerSeen = true; + continue; + } + if (cells.every((c) => /^[- ]*$/.test(c))) continue; // separator row + if (cells.length < 7) continue; + const [id, scenario, category, target, scope, details, doc] = cells; + rows.push({ + id, + scenario: scenario || undefined, + category, + target, + scope: scope || undefined, + details: parseDetails(details), + doc, + }); + } + return rows; +} + +/** `key=value; key=value` → object; `true`/`false` become booleans. */ +function parseDetails(cell: string): Record { + const out: Record = {}; + for (const pair of cell.split(";")) { + const p = pair.trim(); + if (!p || !p.includes("=")) continue; + const i = p.indexOf("="); + const k = p.slice(0, i).trim(); + const v = p + .slice(i + 1) + .trim() + .replace(/\\\|/g, "|"); + out[k] = v === "true" ? true : v === "false" ? false : v; + } + return out; +} + +/** + * Decide whether a scoped check applies to the active language, using the same + * scope grammar as the client-generator decorators: comma-separated language + * names, `!name` to exclude, and `!(a,b)` to exclude a group. An empty/undefined + * scope applies to every language. + */ +export function scopeApplies(scope: string | undefined, language: string): boolean { + const s = scope?.trim(); + if (!s) return true; + const group = s.match(/^!\((.*)\)$/); + if (group) { + const excluded = group[1].split(",").map((x) => x.trim()); + return !excluded.includes(language); + } + const positives: string[] = []; + const negatives: string[] = []; + for (const part of s + .split(",") + .map((x) => x.trim()) + .filter(Boolean)) { + if (part.startsWith("!")) { + negatives.push(part.slice(1).trim()); + } else { + positives.push(part); + } + } + if (positives.length > 0) return positives.includes(language); + if (negatives.length > 0) return !negatives.includes(language); + return true; +} + +type Expect = + | "present" + | "absent" + | { absentWhen: string } + | { whenExpected: Record }; + +/** A pattern rule: search `files` for `find`; PASS per `expect`. */ +export interface Rule { + /** Glob(s) relative to the package dir. Default `**\/*`. */ + files?: string | string[]; + /** Optional regex template isolating blocks (e.g. a class body); `find` runs within each. */ + scope?: string; + /** Regex template with `{var[:casing]}` placeholders. */ + find: string; + /** `present` (default), `absent`, `{ absentWhen: }`, or `{ whenExpected: { : present|absent } }`. */ + expect?: Expect; + /** Per-kind casing map for the `{name:byKind}` modifier. */ + casing?: Record; + /** A second rule AND-ed with this one (e.g. present here AND absent there). */ + also?: Rule; +} + +/** Defer to an existing emitter test that already asserts this surface. */ +export interface TestVerifier { + test: string; + note?: string; +} + +/** + * Declare a whole category not-applicable for this language — the surface concept + * has no equivalent in the emitter's output (e.g. a language with no method + * overloading). Mirrors how wire scenarios are marked N/A via + * `unsupportedScenarios`: emitter-owned, no pattern, no AI. Every check in the + * category resolves to `not-applicable`. + */ +export interface NaVerifier { + na: true; + note?: string; +} + +export type Verifier = Rule | TestVerifier | NaVerifier; + +function isTestVerifier(v: Verifier): v is TestVerifier { + return "test" in v && !!v.test && !("find" in v); +} + +function isNaVerifier(v: Verifier): v is NaVerifier { + return "na" in v && v.na === true; +} + +// --------------------------------------------------------------------------- +// Casing (language-idiomatic recasting, driven by verifiers.json) +// --------------------------------------------------------------------------- + +function splitWords(name: string): string[] { + let s = name.replace(/[_\-\s]+/g, " "); + s = s.replace(/([a-z0-9])([A-Z])/g, "$1 $2"); + s = s.replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2"); + return s.split(/\s+/).filter(Boolean); +} + +function applyCasing(name: string, casing: string): string { + const w = splitWords(name); + const cap = (x: string) => x.charAt(0).toUpperCase() + x.slice(1).toLowerCase(); + switch (casing) { + case "pascal": + return w.map(cap).join(""); + case "camel": + return w.length ? w[0].toLowerCase() + w.slice(1).map(cap).join("") : name; + case "snake": + return w.map((x) => x.toLowerCase()).join("_"); + case "upper_snake": + return w.map((x) => x.toUpperCase()).join("_"); + default: + return name; + } +} + +// --------------------------------------------------------------------------- +// Template + rule engine +// --------------------------------------------------------------------------- + +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Resolve a `{var[:casing]}` placeholder to a concrete value. + * `target` → item.target; `name` → resolved client name; anything else → a + * `details` field. `:byKind` looks up the casing map at `details.kind`. + */ +function resolveVar( + name: string, + mod: string | undefined, + item: CheckItem, + language: string, + casingMap: Record | undefined, +): string | undefined { + let val: string | undefined; + if (name === "target") { + val = item.target; + } else if (name === "name") { + val = item.client_names?.[language] ?? (item.details?.name as string | undefined); + } else { + const d = item.details?.[name]; + val = d == null ? undefined : String(d); + } + if (val == null) return undefined; + // A scoped check carries a verbatim per-language value — never recase it. + if (mod && !item.scope) { + const casing = mod === "byKind" ? casingMap?.[String(item.details?.kind)] : mod; + if (!casing) return undefined; // no casing rule for this kind → N/A + val = applyCasing(val, casing); + } + return val; +} + +/** Compile a regex template; returns null if any placeholder is unresolvable (N/A). */ +function buildRegex( + template: string, + item: CheckItem, + language: string, + casingMap: Record | undefined, +): RegExp | null { + let missing = false; + const source = template.replace(/\{(\w+)(?::(\w+))?\}/g, (_m, v: string, mod?: string) => { + const resolved = resolveVar(v, mod, item, language, casingMap); + if (resolved == null) { + missing = true; + return ""; + } + return escapeRegExp(resolved); + }); + return missing ? null : new RegExp(source); +} + +async function readGlobbed(pkgDir: string, files: string | string[] | undefined): Promise { + const globs = Array.isArray(files) ? files : [files ?? "**/*"]; + const seen = new Set(); + for (const g of globs) { + for await (const f of glob(g, { cwd: pkgDir })) seen.add(f as string); + } + const texts: string[] = []; + for (const rel of [...seen].sort()) { + try { + texts.push(await readFile(join(pkgDir, rel), "utf8")); + } catch { + // directories / unreadable entries are skipped + } + } + return texts.join("\n"); +} + +function resolveExpect(expect: Expect | undefined, item: CheckItem): "present" | "absent" { + if (!expect) return "present"; + if (typeof expect === "string") return expect; + if ("whenExpected" in expect) { + return expect.whenExpected[String(item.details?.expected)] ?? "present"; + } + return item.details?.[expect.absentWhen] ? "absent" : "present"; +} + +interface RuleOutcome { + status: "pass" | "fail"; + evidence: string; +} + +/** Run one pattern rule. Returns null when it doesn't apply (missing placeholder → N/A). */ +async function runRule( + rule: Rule, + item: CheckItem, + pkgDir: string, + language: string, +): Promise { + const findRe = buildRegex(rule.find, item, language, rule.casing); + if (findRe === null) return null; + + const text = await readGlobbed(pkgDir, rule.files); + let haystacks: string[] = [text]; + if (rule.scope) { + const scopeRe = buildRegex(rule.scope, item, language, rule.casing); + if (scopeRe === null) return null; + haystacks = [...text.matchAll(new RegExp(scopeRe.source, "g"))].map((m) => m[0]); + } + + const found = haystacks.some((h) => findRe.test(h)); + const expect = resolveExpect(rule.expect, item); + const ok = expect === "present" ? found : !found; + const evidence = + `/${findRe.source}/ ${found ? "found" : "not found"}` + + (rule.scope ? " in scope" : "") + + ` (wanted ${expect})`; + return { status: ok ? "pass" : "fail", evidence }; +} + +export interface VerifyResult { + id: string; + category: string; + status: string; + evidence: string; + how: string; +} + +async function evalVerifier( + v: Verifier, + item: CheckItem, + pkgDir: string, + language: string, +): Promise<{ status: string; evidence: string; how: string }> { + const primary = await runRule(v as Rule, item, pkgDir, language); + if (primary === null) { + return { + status: "na", + how: "deterministic", + evidence: `no applicable rule (missing detail) for '${item.category}'`, + }; + } + let { status, evidence } = primary; + const also = (v as Rule).also; + if (also) { + const second = await runRule(also, item, pkgDir, language); + if (second) { + if (second.status === "fail") status = "fail"; + evidence += ` | also: ${second.evidence}`; + } + } + return { status, evidence, how: "deterministic" }; +} + +// --------------------------------------------------------------------------- +// Package resolution + orchestration +// --------------------------------------------------------------------------- + +/** Find the generated package dir under / whose name contains the scenario. */ +async function findPackage( + root: string, + flavor: string, + scenario: string, +): Promise { + const base = join(root, flavor); + let entries; + try { + entries = await readdir(base, { withFileTypes: true }); + } catch { + return undefined; + } + const hit = entries + .filter((e) => e.isDirectory()) + .map((e) => e.name) + .sort() + .find((n) => scenario && n.toLowerCase().includes(scenario.toLowerCase())); + return hit ? join(base, hit) : undefined; +} + +export interface VerifySurfaceChecksConfig { + checksPath: string; + verifiersPath: string; + generatedRoot: string; + flavor: string; + language: string; + /** Optional case-insensitive substring filter on the check id. */ + scenario?: string; +} + +export interface VerifySurfaceChecksOutput { + results: VerifyResult[]; + needs_ai: Array<{ + id: string; + category: string; + target: string; + doc: string; + details: Record; + }>; +} + +/** + * Run every check whose category has a verifier (in parallel, no AI) and collect + * the residual (no verifier) into `needs_ai` for the orchestrator. + */ +export async function verifySurfaceChecks( + config: VerifySurfaceChecksConfig, +): Promise { + const { checksPath, verifiersPath, generatedRoot, flavor, language, scenario } = config; + const items = parseChecksDoc(await readFile(checksPath, "utf8")).filter( + (it) => + (!scenario || it.id.toLowerCase().includes(scenario.toLowerCase())) && + scopeApplies(it.scope, language), + ); + const rawVerifiers = JSON.parse(await readFile(verifiersPath, "utf8")) as Record; + const verifiers: Record = {}; + for (const [k, val] of Object.entries(rawVerifiers)) { + if (!k.startsWith("_")) verifiers[k] = val as Verifier; + } + + const results: VerifyResult[] = []; + const needsAi: VerifySurfaceChecksOutput["needs_ai"] = []; + + await Promise.all( + items.map(async (item) => { + const v = verifiers[item.category]; + if (!v) { + needsAi.push({ + id: item.id, + category: item.category, + target: item.target, + doc: item.doc, + details: item.details, + }); + return; + } + // Emitter-declared not-applicable: the concept has no surface in this + // language. Mirrors wire `unsupportedScenarios`; no package or pattern. + if (isNaVerifier(v)) { + const note = v.note ? ` — ${v.note}` : ""; + results.push({ + id: item.id, + category: item.category, + status: "not-applicable", + how: "not-applicable", + evidence: `category '${item.category}' not applicable for ${language}${note}`, + }); + return; + } + // Defer to an existing test: no package or pattern needed. + if (isTestVerifier(v)) { + const note = v.note ? ` — ${v.note}` : ""; + results.push({ + id: item.id, + category: item.category, + status: "deferred", + how: "test", + evidence: `covered by existing test: ${v.test}${note}`, + }); + return; + } + const pkg = await findPackage(generatedRoot, flavor, item.scenario ?? ""); + if (!pkg) { + results.push({ + id: item.id, + category: item.category, + status: "error", + how: "deterministic", + evidence: `package for scenario '${item.scenario}' not found`, + }); + return; + } + try { + const r = await evalVerifier(v, item, pkg, language); + results.push({ id: item.id, category: item.category, ...r }); + } catch (e) { + results.push({ + id: item.id, + category: item.category, + status: "error", + how: "deterministic", + evidence: `${(e as Error).name}: ${(e as Error).message}`, + }); + } + }), + ); + + results.sort((a, b) => a.id.localeCompare(b.id)); + needsAi.sort((a, b) => a.id.localeCompare(b.id)); + return { results, needs_ai: needsAi }; +} diff --git a/packages/spector/src/cli/cli.ts b/packages/spector/src/cli/cli.ts index 219d7ef745a..eb05eb362b3 100644 --- a/packages/spector/src/cli/cli.ts +++ b/packages/spector/src/cli/cli.ts @@ -3,12 +3,14 @@ import "source-map-support/register.js"; import yargs from "yargs"; import { checkCoverage } from "../actions/check-coverage.js"; import { generateScenarioSummary } from "../actions/generate-scenario-summary.js"; +import { generateSurfaceChecks } from "../actions/generate-surface-checks.js"; import { validateScenarios } from "../actions/index.js"; import { serve, startInBackground, stop } from "../actions/serve.js"; import { serverTest } from "../actions/server-test.js"; import { uploadCoverageReport } from "../actions/upload-coverage-report.js"; import { uploadScenarioManifest } from "../actions/upload-scenario-manifest.js"; import { validateMockApis } from "../actions/validate-mock-apis.js"; +import { verifySurfaceChecks } from "../actions/verify-surface-checks.js"; import { logger } from "../logger.js"; import { getCommit } from "../utils/misc-utils.js"; @@ -89,6 +91,80 @@ async function main() { } }, ) + .command( + "generate-surface-checks ", + "Precompute the surface-checks doc (Markdown, from @surfaceDoc) for all specs.", + (cmd) => { + return cmd + .positional("scenariosPaths", { + description: "Path(s) to the scenarios", + type: "string", + array: true, + demandOption: true, + }) + .option("outputFile", { + type: "string", + description: "Path to the generated surface-checks doc (Markdown).", + default: join(process.cwd(), "surface-checks.md"), + }); + }, + async (args) => { + for (const scenariosPath of args.scenariosPaths) { + await generateSurfaceChecks({ + scenariosPath: resolve(process.cwd(), scenariosPath), + outputFile: resolve(process.cwd(), args.outputFile), + }); + } + }, + ) + .command( + "verify-surface-checks", + "Run the deterministic client-surface checks (shared, language-agnostic engine).", + (cmd) => { + return cmd + .option("checks", { + type: "string", + demandOption: true, + description: "Path to the checks doc (Markdown table from @surfaceDoc, or JSON).", + }) + .option("verifiers", { + type: "string", + demandOption: true, + description: "Path to the emitter's declarative verifiers.json routing table.", + }) + .option("generated-root", { + type: "string", + demandOption: true, + description: "Root of the generated SDKs; packages live under //.", + }) + .option("flavor", { + type: "string", + default: "azure", + description: "Flavor subfolder under generated-root (e.g. azure, unbranded).", + }) + .option("language", { + type: "string", + demandOption: true, + description: "Emitter language (selects per-language client names).", + }) + .option("scenario", { + type: "string", + description: "Optional case-insensitive substring filter on the check id.", + }); + }, + async (args) => { + const output = await verifySurfaceChecks({ + checksPath: resolve(process.cwd(), args.checks), + verifiersPath: resolve(process.cwd(), args.verifiers), + generatedRoot: resolve(process.cwd(), args["generated-root"]), + flavor: args.flavor, + language: args.language, + scenario: args.scenario, + }); + // eslint-disable-next-line no-console + console.log(JSON.stringify(output, null, 2)); + }, + ) .command("server", "Server management", (cmd) => { cmd .command( @@ -273,20 +349,13 @@ async function main() { }, ) .command( - "upload-manifest ", + "upload-manifest ", "Upload the scenario manifest. DO NOT CALL in generator.", (cmd) => { return cmd - .positional("scenariosPaths", { + .positional("scenariosPath", { description: "Path to the scenarios and mock apis", type: "string", - array: true, - demandOption: true, - }) - .option("setName", { - type: "string", - description: "Set used to generate the manifest.", - array: true, demandOption: true, }) .option("storageAccountName", { @@ -298,14 +367,26 @@ async function main() { description: "Name of the Container", demandOption: true, }) + .option("manifestName", { + type: "string", + description: + "Name of the manifest(will be located at manifests/.json in the container).", + demandOption: true, + }) + .option("override", { + type: "boolean", + description: "Override existing manifest with the same version.", + default: false, + }) .demandOption("storageAccountName"); }, async (args) => { await uploadScenarioManifest({ - scenariosPaths: args.scenariosPaths, + scenariosPath: args.scenariosPath, storageAccountName: args.storageAccountName, - setNames: args.setName, containerName: args.containerName, + manifestName: args.manifestName, + override: args.override, }); }, ) diff --git a/packages/spector/src/coverage/surface-checks-manifest.ts b/packages/spector/src/coverage/surface-checks-manifest.ts new file mode 100644 index 00000000000..fbd9b1a50af --- /dev/null +++ b/packages/spector/src/coverage/surface-checks-manifest.ts @@ -0,0 +1,228 @@ +import { getSourceLocation, normalizePath } from "@typespec/compiler"; +import { relative } from "path"; +import pc from "picocolors"; +import prettier from "prettier"; +import type { SurfaceDetails, SurfaceDoc, SurfaceSubject } from "../lib/decorators.js"; +import { buildSurfaceDetails } from "../lib/decorators.js"; +import { logger } from "../logger.js"; +import { findScenarioSpecFiles } from "../scenarios-resolver.js"; +import { importSpecExpect, importTypeSpec } from "../spec-utils/index.js"; +import { createDiagnosticReporter, Diagnostic, getSourceLocationStr } from "../utils/index.js"; +import { getCommit, getPackageJson } from "../utils/misc-utils.js"; + +/** One language-neutral entry of the surface-checks doc — what to check, for every emitter. */ +export interface SurfaceCheckItem { + /** + * Stable id, `_` (with a `_n` suffix to disambiguate + * several same-category checks on one scenario), e.g. `listWithPageSize_paging`. + */ + id: string; + /** The enclosing `@scenario` name this surface check belongs to. */ + scenario: string | undefined; + /** The kind of surface assertion (routes the check to a verifier). */ + category: string; + /** The subject symbol's own name, matched against the generated code. */ + target: string; + /** + * Language scope, e.g. `python` or `!java`. Set only for verbatim per-language + * checks (from a `scope → value` dict); empty means all languages (recast). + */ + scope?: string; + /** Language-agnostic description of the expected surface. */ + doc: string; + /** Where the `@surfaceDoc` lives in the spec. */ + location: SurfaceCheckLocation; + /** Generic detail fields (`expected`, `kind`, `origin`); absent for prose-only checks. */ + details?: SurfaceDetails; +} + +export interface SurfaceCheckLocation { + path: string; + start: { line: number; character: number }; + end: { line: number; character: number }; +} + +export interface SurfaceChecksManifest { + version: string; + commit: string; + items: SurfaceCheckItem[]; +} + +export async function computeSurfaceChecksManifest( + scenariosPath: string, +): Promise<[SurfaceChecksManifest | undefined, readonly Diagnostic[]]> { + const [surfaceDocs, diagnostics] = await loadSurfaceDocs(scenariosPath); + if (diagnostics.length > 0) { + return [undefined, diagnostics]; + } + + const commit = getCommit(scenariosPath); + const pkg = await getPackageJson(scenariosPath); + return [createSurfaceChecksManifest(scenariosPath, pkg?.version, commit, surfaceDocs), []]; +} + +export function createSurfaceChecksManifest( + scenariosPath: string, + version: string | undefined, + commit: string, + surfaceDocs: SurfaceDoc[], +): SurfaceChecksManifest { + const items: SurfaceCheckItem[] = []; + const usedIds = new Map(); + for (const surfaceDoc of surfaceDocs) { + const location = getCheckLocation(scenariosPath, surfaceDoc.subject); + const target = getSubjectName(surfaceDoc.subject); + const details = buildSurfaceDetails(surfaceDoc); + const scopeSuffix = surfaceDoc.scope ? `_${surfaceDoc.scope.replace(/[^\w]+/g, "-")}` : ""; + // Every check is identified by its resolved scenario name, so all checks on + // one surface scenario share a stable prefix; `uniqueId` disambiguates when + // a scenario carries several checks of the same category. + const idBase = surfaceDoc.scenario ?? target; + items.push({ + id: uniqueId(usedIds, `${idBase}_${surfaceDoc.category}${scopeSuffix}`), + scenario: surfaceDoc.scenario, + category: surfaceDoc.category, + target, + ...(surfaceDoc.scope ? { scope: surfaceDoc.scope } : {}), + doc: surfaceDoc.doc, + location, + ...(Object.keys(details).length > 0 ? { details } : {}), + }); + } + items.sort((a, b) => a.id.localeCompare(b.id)); + return { version: version ?? "?", commit, items }; +} + +/** + * Render a {@link SurfaceChecksManifest} as a single Markdown document that is + * both human-readable and machine-readable: the table below is the source of + * truth consumed by the shared `verify-surface-checks` runner, and is also easy + * to read in review. The + * `details` column encodes category-specific expectations as `key=value` pairs + * separated by `; ` (booleans as `true`/`false`). + */ +export function createSurfaceChecksSummary(manifest: SurfaceChecksManifest): Promise { + const lines = [ + `# Surface checks`, + ``, + `Generated from \`@surfaceDoc\` annotations. This table is both the human summary`, + `and the machine-readable checks doc parsed by the shared \`verify-surface-checks\` runner.`, + ``, + `| id | scenario | category | target | scope | details | doc |`, + `| --- | --- | --- | --- | --- | --- | --- |`, + ]; + for (const item of manifest.items) { + const cells = [ + item.id, + item.scenario ?? "", + item.category, + item.target, + item.scope ?? "", + renderDetails(item.details), + escapeCell(item.doc), + ]; + lines.push(`| ${cells.join(" | ")} |`); + } + return prettier.format(lines.join("\n"), { parser: "markdown" }); +} + +/** Encode `details` as `key=value; key=value` for the Markdown table cell. */ +function renderDetails(details: SurfaceDetails | undefined): string { + if (!details) { + return ""; + } + return escapeCell( + Object.entries(details) + .map(([k, v]) => `${k}=${v}`) + .join("; "), + ); +} + +/** Escape characters that would break a Markdown table cell. */ +function escapeCell(text: string): string { + return text.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\n/g, " "); +} + +function uniqueId(used: Map, base: string): string { + const count = used.get(base) ?? 0; + used.set(base, count + 1); + return count === 0 ? base : `${base}_${count + 1}`; +} + +function getSubjectName(subject: SurfaceSubject): string { + return typeof subject.name === "string" ? subject.name : ""; +} + +function getCheckLocation(scenariosPath: string, subject: SurfaceSubject): SurfaceCheckLocation { + const tspLocation = getSourceLocation(subject); + return { + path: normalizePath(relative(scenariosPath, tspLocation.file.path)), + start: tspLocation.file.getLineAndCharacterOfPosition(tspLocation.pos), + end: tspLocation.file.getLineAndCharacterOfPosition(tspLocation.end), + }; +} + +export async function loadSurfaceDocs( + scenariosPath: string, +): Promise<[SurfaceDoc[], readonly Diagnostic[]]> { + const scenarioFiles = await findScenarioSpecFiles(scenariosPath); + const typespecCompiler = await importTypeSpec(scenariosPath); + const specExpect = await importSpecExpect(scenariosPath); + const diagnostics = createDiagnosticReporter(); + + const surfaceDocs: SurfaceDoc[] = []; + for (const { name, specFilePath } of scenarioFiles) { + logger.debug(`Collecting surface docs from "${specFilePath}"`); + const program = await typespecCompiler.compile(typespecCompiler.NodeHost, specFilePath, { + additionalImports: ["@typespec/spector"], + noEmit: true, + warningAsError: true, + }); + + const programDiagnostics = program.diagnostics.filter( + (d) => + !( + d.code === "@azure-tools/typespec-azure-core/casing-style" && + typeof d.target === "object" && + "kind" in d.target && + d.target.kind === "Namespace" && + d.target.name === "DPG" + ), + ); + + if (programDiagnostics.length > 0) { + for (const item of programDiagnostics) { + const sourceLocation = typespecCompiler.getSourceLocation(item.target); + diagnostics.reportDiagnostic({ + message: `${item.message}: ${sourceLocation && getSourceLocationStr(sourceLocation)}`, + }); + } + diagnostics.reportDiagnostic({ + message: `${pc.red("✘")} Scenario ${name} is invalid.`, + }); + continue; + } + + // `@surfaceDoc` must sit on an element that also carries `@scenarioDoc`, so + // every surface check is grounded in a documented scenario. Enforced here + // (not via a compiler `$onValidate` hook) so plain spec compilation does not + // trigger spector's other, currently-dormant scenario validations. + const missingScenarioDoc = specExpect.listSurfaceDocsMissingScenarioDoc(program); + if (missingScenarioDoc.length > 0) { + for (const target of missingScenarioDoc) { + const sourceLocation = typespecCompiler.getSourceLocation(target); + diagnostics.reportDiagnostic({ + message: `@surfaceDoc may only be applied to an element that also has @scenarioDoc, so every surface check is grounded in a documented scenario.: ${sourceLocation && getSourceLocationStr(sourceLocation)}`, + }); + } + diagnostics.reportDiagnostic({ + message: `${pc.red("✘")} Scenario ${name} is invalid.`, + }); + continue; + } + + surfaceDocs.push(...specExpect.listSurfaceDocs(program)); + } + + return [surfaceDocs, diagnostics.diagnostics]; +} diff --git a/packages/spector/src/lib/decorators.ts b/packages/spector/src/lib/decorators.ts index 4b61d2cd261..0f5e2a2e454 100644 --- a/packages/spector/src/lib/decorators.ts +++ b/packages/spector/src/lib/decorators.ts @@ -1,14 +1,18 @@ import { $service, Enum, + EnumMember, getNamespaceFullName, getTypeName, Interface, listServices, Model, + ModelProperty, Namespace, Operation, Program, + Union, + UnionVariant, } from "@typespec/compiler"; import { $route, @@ -23,6 +27,7 @@ import { ScenarioDecorator, ScenarioDocDecorator, ScenarioServiceDecorator, + SurfaceDocDecorator, } from "../../generated-defs/TypeSpec.Spector.js"; import { SpectorStateKeys } from "./lib.js"; @@ -35,6 +40,39 @@ export const $scenarioDoc: ScenarioDocDecorator = (context, target, doc, formatA context.program.stateMap(SpectorStateKeys.ScenarioDoc).set(target, formattedDoc); }; +export const $surfaceDoc: SurfaceDocDecorator = ( + context, + target, + category, + subject, + expected, + doc?, +) => { + const map = context.program.stateMap(SpectorStateKeys.SurfaceDoc); + const existing: StoredSurfaceDoc[] = map.get(target) ?? []; + existing.push({ + category, + subject: subject as SurfaceSubject, + expected: expected as string | Record, + doc, + }); + // A target can carry several `@surfaceDoc`s (different categories), so store a + // list — never overwrite a previously recorded check. + map.set(target, existing); +}; + +/** + * What a single `@surfaceDoc` application records. `expected` is either one + * canonical string (recast per language) or a `scope → value` dict (each value + * matched verbatim for its scope). + */ +interface StoredSurfaceDoc { + category: string; + subject: SurfaceSubject; + expected: string | Record; + doc?: string; +} + export const $scenarioService: ScenarioServiceDecorator = (context, target, route, options?) => { const properties = new Map().set("title", { type: { kind: "String", value: getNamespaceFullName(target).replace(/\./g, "") }, @@ -221,3 +259,244 @@ export function getScenarioName( } return resolveScenarioName(target, name); } + +/** + * The element `@surfaceDoc` is applied to. To keep surface checks grounded in a + * real scenario, this must be a `@scenario`/`@scenarioDoc` element (enforced in + * `loadSurfaceDocs` while the manifest is built), so the union matches + * `@scenarioDoc`'s target. + */ +export type SurfaceDocTarget = Namespace | Interface | Operation; + +/** + * The `subject` of a surface check — the type or member whose generated surface + * is being asserted. May differ from the annotated {@link SurfaceDocTarget} + * (e.g. a check on an operation asserts something about a model it returns). + */ +export type SurfaceSubject = + | Namespace + | Interface + | Operation + | Model + | Enum + | Union + | ModelProperty + | EnumMember + | UnionVariant; + +/** + * The generic, category-agnostic fields the shared runner substitutes into an + * emitter's `verifiers.json` (as `{expected}`, `{kind}`, `{origin}`). They are + * derived the same way for every category, so a new category needs no core + * change — only a `verifiers.json` entry (or the AI fallback). + */ +export interface SurfaceDetails { + /** The author's `expected` client-surface output for this check. */ + expected?: string; + /** The subject's language-agnostic symbol kind (for casing-aware checks). */ + kind?: string; + /** The subject's declaring container (e.g. the client an operation moved from). */ + origin?: string; +} + +/** A resolved `@surfaceDoc` annotation. */ +export interface SurfaceDoc { + /** + * The resolved name of the enclosing `@scenario` this check belongs to (named + * the way `@scenario`s are, e.g. `Payload_Pageable_PageSize_listWithPageSize`). + * Every surface check is grounded in — and identified by — its scenario, so + * multiple checks on one scenario share this name. `undefined` only if the + * annotated element has no enclosing `@scenario` (which `loadSurfaceDocs` + * rejects, since `@surfaceDoc` requires `@scenarioDoc`). + */ + scenario: string | undefined; + /** The annotated element (a scenario namespace/interface/operation). */ + target: SurfaceDocTarget; + /** The type/member the check is about. */ + subject: SurfaceSubject; + /** The kind of surface assertion (routes the check to a verifier). */ + category: string; + /** The expected client-surface output for this category. */ + expected: string; + /** + * The language scope this check applies to, e.g. `"python"`, `"python,csharp"`, + * or `"!java"`. Set only when `expected` came from a `scope → value` dict; in + * that case the value is matched **verbatim**. Unset = all languages (recast). + */ + scope?: string; + /** Natural-language description (author-supplied, or synthesized for fallback). */ + doc: string; +} + +/** A short prose fallback so the AI path always has something to verify against. */ +function synthesizeDoc(category: string, subject: SurfaceSubject, expected: string): string { + const subjectName = typeof subject.name === "string" ? subject.name : "the subject"; + return `${category}: ${subjectName} → ${expected}`; +} + +function getSurfaceParent(target: SurfaceSubject): SurfaceSubject | undefined { + switch (target.kind) { + case "Namespace": + case "Interface": + case "Model": + case "Enum": + case "Union": + return target.namespace; + case "Operation": + return target.interface ?? target.namespace; + case "ModelProperty": + return target.model; + case "EnumMember": + return target.enum; + case "UnionVariant": + return target.union; + default: + return undefined; + } +} + +/** + * Build a scenario-style name by walking up an element's containers and joining + * their names, stopping at the (unnamed) global or the `_Specs_` root — the same + * convention `@scenario` uses. + */ +function getEnclosingScenarioName(program: Program, target: SurfaceDocTarget): string | undefined { + let current: SurfaceSubject | undefined = target; + while (current) { + if ( + current.kind === "Namespace" || + current.kind === "Interface" || + current.kind === "Operation" + ) { + const name = getScenarioName(program, current); + if (name) { + return name; + } + } + current = getSurfaceParent(current); + } + return undefined; +} + +/** + * Collect every `@surfaceDoc` in the program into a list of language-agnostic + * surface docs. Analogous to {@link listScenarios}, but for the generated + * surface instead of the wire. Each entry records the author-supplied category, + * subject, and expected output verbatim — nothing is inferred from other + * decorators. Feeds the `surface-checks.md` checks doc. + */ +export function listSurfaceDocs(program: Program): SurfaceDoc[] { + const map = program.stateMap(SpectorStateKeys.SurfaceDoc); + const result: SurfaceDoc[] = []; + for (const [target, storedList] of map as Map) { + for (const stored of storedList) { + const scenario = getEnclosingScenarioName(program, target); + for (const { expected, scope } of expandExpected(stored.expected)) { + result.push({ + scenario, + target, + subject: stored.subject, + category: stored.category, + expected, + scope, + doc: stored.doc ?? synthesizeDoc(stored.category, stored.subject, expected), + }); + } + } + } + return result.sort( + (a, b) => + (a.scenario ?? "").localeCompare(b.scenario ?? "") || + a.category.localeCompare(b.category) || + (a.scope ?? "").localeCompare(b.scope ?? ""), + ); +} + +/** + * Surface docs whose annotated target does not also carry `@scenarioDoc`. + * + * `@surfaceDoc` must sit on an element that also has `@scenarioDoc` so every + * surface check is grounded in a documented scenario. This is enforced while + * the surface-checks manifest is built (see `loadSurfaceDocs`) rather than as a + * compiler `$onValidate` hook, so it never activates spector's other, + * currently-dormant scenario validations for consumers that only compile specs. + */ +export function listSurfaceDocsMissingScenarioDoc(program: Program): SurfaceDocTarget[] { + const map = program.stateMap(SpectorStateKeys.SurfaceDoc); + const result: SurfaceDocTarget[] = []; + for (const target of map.keys() as Iterable) { + if (getScenarioDoc(program, target) === undefined) { + result.push(target); + } + } + return result; +} + +/** + * Normalize an author's `expected` into one entry per check. A bare string is a + * single, unscoped (idiomatically recast) check; a `scope → value` dict yields + * one verbatim check per scope key. + */ +function expandExpected( + expected: string | Record, +): { expected: string; scope?: string }[] { + if (typeof expected === "string") { + return [{ expected }]; + } + return Object.entries(expected).map(([scope, value]) => ({ expected: value, scope })); +} + +/** + * Build the generic, category-agnostic detail fields the shared runner + * substitutes into `verifiers.json`. Derived the same way for every category so + * a new category needs no core change: `expected` verbatim, plus the subject's + * `kind` (for casing) and `origin` (its declaring container). + */ +export function buildSurfaceDetails(doc: SurfaceDoc): SurfaceDetails { + const details: SurfaceDetails = {}; + if (doc.expected !== "") { + details.expected = doc.expected; + } + const kind = getSurfaceKind(doc.subject); + if (kind !== undefined) { + details.kind = kind; + } + const origin = getDeclaringContainerName(doc.subject); + if (origin !== undefined) { + details.origin = origin; + } + return details; +} + +/** + * Map a surface subject to the language-agnostic symbol kind used by + * casing-aware `naming` checks so each emitter can recast the expected + * identifier into its idiomatic casing. + */ +function getSurfaceKind(target: SurfaceSubject): string | undefined { + switch (target.kind) { + case "Enum": + case "Union": + return "enum"; + case "EnumMember": + case "UnionVariant": + return "enumvalue"; + case "Model": + return "model"; + case "ModelProperty": + return "property"; + case "Operation": + return "operation"; + case "Namespace": + case "Interface": + return "client"; + default: + return undefined; + } +} + +/** The name of the client/group an element is declared in — its surface origin. */ +function getDeclaringContainerName(target: SurfaceSubject): string | undefined { + const parent = getSurfaceParent(target); + return parent && typeof parent.name === "string" && parent.name ? parent.name : undefined; +} diff --git a/packages/spector/src/lib/index.ts b/packages/spector/src/lib/index.ts index 0e7f1ce808c..041f35bb9d6 100644 --- a/packages/spector/src/lib/index.ts +++ b/packages/spector/src/lib/index.ts @@ -2,14 +2,22 @@ export { $scenario, $scenarioDoc, $scenarioService, + $surfaceDoc, Scenario, ScenarioEndpoint, + SurfaceDetails, + SurfaceDoc, + SurfaceDocTarget, + SurfaceSubject, + buildSurfaceDetails, getScenarioDoc, getScenarioEndpoints, getScenarioName, isScenario, listScenarioIn, listScenarios, + listSurfaceDocs, + listSurfaceDocsMissingScenarioDoc, } from "./decorators.js"; export { $lib, reportDiagnostic } from "./lib.js"; export { $onValidate } from "./validate.js"; diff --git a/packages/spector/src/lib/tsp-index.ts b/packages/spector/src/lib/tsp-index.ts index bf29ae1369e..d4be5ba8252 100644 --- a/packages/spector/src/lib/tsp-index.ts +++ b/packages/spector/src/lib/tsp-index.ts @@ -1,5 +1,5 @@ import type { TypeSpecSpectorDecorators } from "../../generated-defs/TypeSpec.Spector.js"; -import { $scenario, $scenarioDoc, $scenarioService } from "./decorators.js"; +import { $scenario, $scenarioDoc, $scenarioService, $surfaceDoc } from "./decorators.js"; export { $lib } from "./lib.js"; /** @internal */ @@ -8,5 +8,6 @@ export const $decorators = { scenario: $scenario, scenarioDoc: $scenarioDoc, scenarioService: $scenarioService, + surfaceDoc: $surfaceDoc, } satisfies TypeSpecSpectorDecorators, }; diff --git a/packages/spector/test/surface-docs.test.ts b/packages/spector/test/surface-docs.test.ts new file mode 100644 index 00000000000..63a1b2312d4 --- /dev/null +++ b/packages/spector/test/surface-docs.test.ts @@ -0,0 +1,323 @@ +import { resolvePath } from "@typespec/compiler"; +import { createTester } from "@typespec/compiler/testing"; +import { describe, expect, it } from "vitest"; +import { + createSurfaceChecksManifest, + createSurfaceChecksSummary, + type SurfaceCheckItem, +} from "../src/coverage/surface-checks-manifest.js"; +import { + listSurfaceDocs, + listSurfaceDocsMissingScenarioDoc, + type SurfaceDoc, +} from "../src/lib/decorators.js"; + +const Tester = createTester(resolvePath(import.meta.dirname, ".."), { + libraries: ["@typespec/spector"], +}) + .importLibraries() + .using("Spector"); + +/** Compile `code` and return its single resolved surface doc. */ +async function docOf(code: string): Promise { + const { program } = await Tester.compile(code); + const docs = listSurfaceDocs(program); + expect(docs).toHaveLength(1); + return docs[0]; +} + +/** Compile `code` and return the surface-checks items keyed by id. */ +async function manifestItems(code: string): Promise> { + const { program } = await Tester.compile(code); + const manifest = createSurfaceChecksManifest(".", "1.0.0", "abc123", listSurfaceDocs(program)); + return Object.fromEntries(manifest.items.map((i) => [i.id, i])); +} + +describe("@surfaceDoc", () => { + // --- authoring ---------------------------------------------------------- + + it("records the explicit category, subject, expected and prose", async () => { + const doc = await docOf(` + enum ServerExtensibleEnum { + value1, + value2, + } + + @scenario + @scenarioDoc("Return the kind.") + @surfaceDoc("naming", ServerExtensibleEnum, "ClientExtensibleEnum", "Exposed to clients as ClientExtensibleEnum.") + op getKind(): ServerExtensibleEnum; + `); + expect(doc.category).toBe("naming"); + expect(doc.subject.kind).toBe("Enum"); + expect(doc.expected).toBe("ClientExtensibleEnum"); + expect(doc.doc).toBe("Exposed to clients as ClientExtensibleEnum."); + }); + + it("resolves the scenario name from the annotated element, shared by all its checks", async () => { + const doc = await docOf(` + enum ServerExtensibleEnum { + value1, + value2, + } + + @scenario + @scenarioDoc("Return the kind.") + @surfaceDoc("naming", ServerExtensibleEnum, "ClientExtensibleEnum") + op getKind(): ServerExtensibleEnum; + `); + expect(doc.scenario).toBe("getKind"); + }); + + it("synthesizes prose when `doc` is omitted (so the AI fallback always has one)", async () => { + const doc = await docOf(` + model Widget { + id: string; + } + + @scenario + @scenarioDoc("Get a widget.") + @surfaceDoc("access", Widget, "internal") + op get(): Widget; + `); + expect(doc.doc).toContain("access"); + expect(doc.doc).toContain("Widget"); + expect(doc.doc).toContain("internal"); + }); + + it("accepts any category string (extensible), so new categories need no core change", async () => { + const doc = await docOf(` + model Widget { + id: string; + } + + @scenario + @scenarioDoc("Get a widget.") + @surfaceDoc("brand-new-category", Widget, "whatever") + op get(): Widget; + `); + expect(doc.category).toBe("brand-new-category"); + }); + + // --- validation: must be grounded in a scenario doc --------------------- + + it("flags a surface doc whose target has no @scenarioDoc", async () => { + const { program } = await Tester.compile(` + model Widget { + id: string; + } + + @scenario + @surfaceDoc("access", Widget, "internal") + op get(): Widget; + `); + const missing = listSurfaceDocsMissingScenarioDoc(program); + expect(missing).toHaveLength(1); + }); + + it("does not flag a surface doc whose target also has @scenarioDoc", async () => { + const { program } = await Tester.compile(` + model Widget { + id: string; + } + + @scenario + @scenarioDoc("Get a widget.") + @surfaceDoc("access", Widget, "internal") + op get(): Widget; + `); + expect(listSurfaceDocsMissingScenarioDoc(program)).toHaveLength(0); + }); + + // --- manifest: generic, category-agnostic details ----------------------- + + it("derives {expected, kind} for a naming check", async () => { + const items = await manifestItems(` + enum ServerExtensibleEnum { + value1, + value2, + } + + @scenario + @scenarioDoc("Return the kind.") + @surfaceDoc("naming", ServerExtensibleEnum, "ClientExtensibleEnum", "Exposed as ClientExtensibleEnum.") + op getKind(): ServerExtensibleEnum; + `); + const item = items["getKind_naming"]; + expect(item.id).toBe("getKind_naming"); + expect(item.category).toBe("naming"); + expect(item.target).toBe("ServerExtensibleEnum"); + expect(item.doc).toContain("Exposed as ClientExtensibleEnum"); + expect(item.details).toEqual({ expected: "ClientExtensibleEnum", kind: "enum" }); + // A location the verifier can point back to for reporting. + expect(typeof item.location.path).toBe("string"); + expect(item.location.start.line).toBeGreaterThanOrEqual(0); + expect(item.location.end.line).toBeGreaterThanOrEqual(item.location.start.line); + }); + + it("derives {expected, kind, origin} for a subject inside a container", async () => { + const items = await manifestItems(` + model Widget { + id: string; + } + + @scenario + @scenarioDoc("Get a widget.") + @surfaceDoc("naming", Widget.id, "identifier", "Renamed to identifier.") + op get(): Widget; + `); + const item = items["get_naming"]; + expect(item.category).toBe("naming"); + expect(item.target).toBe("id"); + expect(item.details).toEqual({ expected: "identifier", kind: "property", origin: "Widget" }); + }); + + it("omits `expected` from details when it is blank", async () => { + const items = await manifestItems(` + @scenario + @scenarioDoc("List items.") + @surfaceDoc("paging", listItems, "") + op listItems(): string[]; + `); + const item = items["listItems_paging"]; + expect(item.category).toBe("paging"); + expect(item.details?.expected).toBeUndefined(); + }); + + it("keeps the prose for a prose-only `other` check", async () => { + const items = await manifestItems(` + model Widget { + id: string; + } + + @scenario + @scenarioDoc("Get a widget.") + @surfaceDoc("other", Widget, "strongly typed", "The response body is a strongly typed model.") + op get(): Widget; + `); + const item = items["get_other"]; + expect(item.category).toBe("other"); + expect(item.doc).toContain("strongly typed model"); + }); + + // --- rendering ---------------------------------------------------------- + + it("renders a single Markdown table that carries every routable field", async () => { + const { program } = await Tester.compile(` + model Widget { + id: string; + } + + @scenario + @scenarioDoc("Get a widget.") + @surfaceDoc("naming", Widget, "WidgetInternal", "Hidden | renamed to WidgetInternal.") + op get(): Widget; + `); + const manifest = createSurfaceChecksManifest(".", "1.0.0", "abc123", listSurfaceDocs(program)); + const md = await createSurfaceChecksSummary(manifest); + + // The rendered doc is idempotent: no volatile version/commit provenance. + expect(md).not.toContain("commit:"); + expect(md).toContain("Generated from `@surfaceDoc` annotations."); + // Table header with the routable columns (prettier pads cell widths). + const header = md.split("\n").find((l) => l.includes("| id")); + expect(header).toBeDefined(); + for (const col of ["id", "scenario", "category", "target", "scope", "details", "doc"]) { + expect(header).toContain(col); + } + // details encoded as key=value. + expect(md).toContain("expected=WidgetInternal; kind=model"); + // Pipes inside prose are escaped so they don't break the table. + expect(md).toContain("Hidden \\| renamed to WidgetInternal."); + }); + + // --- per-language exact names (scope → value dict) ---------------------- + + it("expands a `scope → value` dict into one verbatim check per scope", async () => { + const { program } = await Tester.compile(` + model IOThing { + id: string; + } + + @scenario + @scenarioDoc("Get the thing.") + @surfaceDoc("naming", IOThing, #{ python: "io_thing", csharp: "IOThing" }) + op get(): IOThing; + `); + const docs = listSurfaceDocs(program); + expect(docs).toHaveLength(2); + const byScope = Object.fromEntries(docs.map((d) => [d.scope, d])); + expect(byScope["python"].expected).toBe("io_thing"); + expect(byScope["csharp"].expected).toBe("IOThing"); + expect(byScope["python"].category).toBe("naming"); + }); + + it("leaves scope unset for a single canonical (recast) value", async () => { + const doc = await docOf(` + model Widget { + id: string; + } + + @scenario + @scenarioDoc("Get a widget.") + @surfaceDoc("naming", Widget, "Widget") + op get(): Widget; + `); + expect(doc.scope).toBeUndefined(); + }); + + it("includes the scope in the manifest id and renders a scope column", async () => { + const { program } = await Tester.compile(` + model IOThing { + id: string; + } + + @scenario + @scenarioDoc("Get the thing.") + @surfaceDoc("naming", IOThing, #{ python: "io_thing" }) + op get(): IOThing; + `); + const manifest = createSurfaceChecksManifest(".", "1.0.0", "abc123", listSurfaceDocs(program)); + const item = manifest.items.find((i) => i.id === "get_naming_python"); + expect(item).toBeDefined(); + expect(item!.scope).toBe("python"); + expect(item!.details).toEqual({ expected: "io_thing", kind: "model" }); + const md = await createSurfaceChecksSummary(manifest); + expect(md).toContain("io_thing"); + expect(md).toContain("python"); + }); + + it("records multiple @surfaceDocs on one scenario, sharing its name with distinct ids", async () => { + const { program } = await Tester.compile(` + model Widget { + id: string; + } + + @scenario + @scenarioDoc("Get a widget.") + @surfaceDoc("naming", Widget, "WidgetClient") + @surfaceDoc("access", Widget, "internal") + op get(): Widget; + `); + const docs = listSurfaceDocs(program); + expect(docs).toHaveLength(2); + expect(docs.map((d) => d.category).sort()).toEqual(["access", "naming"]); + // Every check on the scenario shares its resolved scenario name... + expect(docs.map((d) => d.scenario)).toEqual(["get", "get"]); + + // ...and each surfaces as its own check, keyed `_`. + const items = await manifestItems(` + model Widget { + id: string; + } + + @scenario + @scenarioDoc("Get a widget.") + @surfaceDoc("naming", Widget, "WidgetClient") + @surfaceDoc("access", Widget, "internal") + op get(): Widget; + `); + expect(items["get_naming"].scenario).toBe("get"); + expect(items["get_access"].scenario).toBe("get"); + }); +}); diff --git a/packages/spector/test/verify-surface-checks.test.ts b/packages/spector/test/verify-surface-checks.test.ts new file mode 100644 index 00000000000..9affc4997d3 --- /dev/null +++ b/packages/spector/test/verify-surface-checks.test.ts @@ -0,0 +1,336 @@ +import { mkdir, mkdtemp, writeFile } from "fs/promises"; +import { tmpdir } from "os"; +import { join } from "path"; +import { beforeAll, describe, expect, it } from "vitest"; +import { + parseChecksDoc, + verifySurfaceChecks, + type VerifySurfaceChecksOutput, +} from "../src/actions/verify-surface-checks.js"; + +// A tiny generated "Python SDK" fixture under /azure/surfacedemo/, plus a +// declarative verifiers.json, exercising every routine kind of the shared engine. +let root: string; +let verifiersPath: string; + +const verifiers = { + access: { + files: ["**/models/__init__.py"], + find: "\\b{target}\\b", + expect: { absentWhen: "internal" }, + }, + naming: { + files: ["**/*.py"], + find: "\\b{name:byKind}\\b", + casing: { enum: "pascal", property: "snake" }, + }, + hierarchy: { + files: ["**/models/_models.py"], + find: "class\\s+{target}\\s*\\([^)]*\\b{base}\\b[^)]*\\)", + }, + "client-location": { + files: ["**/operations/_operations.py"], + scope: "class\\s+\\w*{client}\\w*[\\s\\S]*?(?=\\nclass |$)", + find: "def\\s+{target:snake}\\b", + also: { + files: ["**/operations/_operations.py"], + scope: "class\\s+\\w*{absentFrom}\\w*[\\s\\S]*?(?=\\nclass |$)", + find: "def\\s+{target:snake}\\b", + expect: "absent", + }, + }, +}; + +beforeAll(async () => { + root = await mkdtemp(join(tmpdir(), "surface-verify-")); + const pkg = join(root, "azure", "surfacedemo"); + await mkdir(join(pkg, "models"), { recursive: true }); + await mkdir(join(pkg, "operations"), { recursive: true }); + + await writeFile( + join(pkg, "models", "__init__.py"), + `from ._enums import ClientExtensibleEnum\n__all__ = ["ClientExtensibleEnum"]\n`, + ); + await writeFile( + join(pkg, "models", "_enums.py"), + `class ClientExtensibleEnum(str, Enum):\n ONE = "one"\n`, + ); + await writeFile( + join(pkg, "models", "_models.py"), + `class Animal:\n pass\n\nclass Pet(Animal):\n pass\n\nclass Dog(Pet, discriminator="dog"):\n pass\n`, + ); + await writeFile( + join(pkg, "operations", "_operations.py"), + `class WidgetsOperations:\n def get_widget(self):\n pass\n\nclass GadgetsOperations:\n def list_gadgets(self):\n pass\n`, + ); + await writeFile(join(pkg, "models", "_scoped.py"), `class IOThing:\n pass\n`); + + verifiersPath = join(root, "verifiers.json"); + await writeFile(verifiersPath, JSON.stringify(verifiers)); +}); + +async function run(items: object[], scenario?: string): Promise { + const checksPath = join(root, "checks.json"); + await writeFile(checksPath, JSON.stringify({ items })); + return verifySurfaceChecks({ + checksPath, + verifiersPath, + generatedRoot: root, + flavor: "azure", + language: "python", + scenario, + }); +} + +describe("verify-surface-checks (shared engine)", () => { + it("naming: idiomatic casing via {name:byKind} passes on the recast identifier", async () => { + const out = await run([ + { + id: "n1", + scenario: "surfacedemo", + category: "naming", + target: "ServerExtensibleEnum", + details: { name: "ClientExtensibleEnum", kind: "enum" }, + doc: "", + }, + ]); + expect(out.results[0]).toMatchObject({ status: "pass", how: "deterministic" }); + }); + + it("naming: wrong expected name fails", async () => { + const out = await run([ + { + id: "n2", + scenario: "surfacedemo", + category: "naming", + target: "X", + details: { name: "TotallyMadeUp", kind: "enum" }, + doc: "", + }, + ]); + expect(out.results[0].status).toBe("fail"); + }); + + it("access: public export present passes; internal expectation flips it", async () => { + const pass = await run([ + { + id: "a1", + scenario: "surfacedemo", + category: "access", + target: "ClientExtensibleEnum", + details: {}, + doc: "", + }, + ]); + expect(pass.results[0].status).toBe("pass"); + const fail = await run([ + { + id: "a2", + scenario: "surfacedemo", + category: "access", + target: "ClientExtensibleEnum", + details: { internal: true }, + doc: "", + }, + ]); + expect(fail.results[0].status).toBe("fail"); // present but wanted internal (absent) + }); + + it("hierarchy: subtype base pattern passes for Dog(Pet), fails for Dog(Animal)", async () => { + const good = await run([ + { + id: "h1", + scenario: "surfacedemo", + category: "hierarchy", + target: "Dog", + details: { base: "Pet" }, + doc: "", + }, + ]); + expect(good.results[0].status).toBe("pass"); + const bad = await run([ + { + id: "h2", + scenario: "surfacedemo", + category: "hierarchy", + target: "Dog", + details: { base: "Animal" }, + doc: "", + }, + ]); + expect(bad.results[0].status).toBe("fail"); + }); + + it("client-location: present on expected client AND absent from the other", async () => { + const good = await run([ + { + id: "c1", + scenario: "surfacedemo", + category: "client-location", + target: "getWidget", + details: { client: "Widgets", absentFrom: "Gadgets" }, + doc: "", + }, + ]); + expect(good.results[0].status).toBe("pass"); + const bad = await run([ + { + id: "c2", + scenario: "surfacedemo", + category: "client-location", + target: "getWidget", + details: { client: "Gadgets", absentFrom: "Widgets" }, + doc: "", + }, + ]); + expect(bad.results[0].status).toBe("fail"); // getWidget is on Widgets, not Gadgets + }); + + it("scope: a check scoped to another language is skipped entirely", async () => { + const out = await run([ + { + id: "s1", + scenario: "surfacedemo", + category: "naming", + target: "IOThing", + scope: "csharp", + details: { name: "IOThing", kind: "enum" }, + doc: "", + }, + ]); + expect(out.results).toHaveLength(0); + expect(out.needs_ai).toHaveLength(0); + }); + + it("scope: a scoped value is matched verbatim, bypassing idiomatic casing", async () => { + const out = await run([ + { + id: "s2", + scenario: "surfacedemo", + category: "naming", + target: "IOThing", + scope: "python", + details: { name: "IOThing", kind: "enum" }, + doc: "", + }, + ]); + expect(out.results[0]).toMatchObject({ status: "pass", how: "deterministic" }); + }); + + it("scope: an unscoped value is still recast (pascal mangles IOThing → no match)", async () => { + const out = await run([ + { + id: "s3", + scenario: "surfacedemo", + category: "naming", + target: "IOThing", + details: { name: "IOThing", kind: "enum" }, + doc: "", + }, + ]); + expect(out.results[0].status).toBe("fail"); + }); + + it("no verifier for a category → needs_ai", async () => { + const out = await run([ + { + id: "u1", + scenario: "surfacedemo", + category: "flatten", + target: "Foo", + details: {}, + doc: "prose", + }, + ]); + expect(out.results).toHaveLength(0); + expect(out.needs_ai[0]).toMatchObject({ id: "u1", category: "flatten", doc: "prose" }); + }); + + it("test-deferral verifier → deferred/test without touching the package", async () => { + const p = join(root, "verifiers.test.json"); + await writeFile( + p, + JSON.stringify({ + hierarchy: { test: "path/To/Test.java#L1-L2", note: "reflection asserts base" }, + }), + ); + const checksPath = join(root, "checks.json"); + await writeFile( + checksPath, + JSON.stringify({ + items: [ + { + id: "t1", + scenario: "surfacedemo", + category: "hierarchy", + target: "Dog", + details: { base: "Pet" }, + doc: "", + }, + ], + }), + ); + const out = await verifySurfaceChecks({ + checksPath, + verifiersPath: p, + generatedRoot: root, + flavor: "azure", + language: "python", + }); + expect(out.results[0]).toMatchObject({ status: "deferred", how: "test" }); + expect(out.results[0].evidence).toContain("path/To/Test.java#L1-L2"); + }); + + it("na verifier → not-applicable without touching the package", async () => { + const p = join(root, "verifiers.na.json"); + await writeFile( + p, + JSON.stringify({ + hierarchy: { na: true, note: "no class inheritance in this language" }, + }), + ); + const checksPath = join(root, "checks.json"); + await writeFile( + checksPath, + JSON.stringify({ + items: [ + { + id: "na1", + scenario: "surfacedemo", + category: "hierarchy", + target: "Dog", + details: { base: "Pet" }, + doc: "", + }, + ], + }), + ); + const out = await verifySurfaceChecks({ + checksPath, + verifiersPath: p, + generatedRoot: root, + flavor: "azure", + language: "python", + }); + expect(out.results[0]).toMatchObject({ status: "not-applicable", how: "not-applicable" }); + expect(out.results[0].evidence).toContain("no class inheritance in this language"); + expect(out.needs_ai).toHaveLength(0); + }); + + it("parseChecksDoc reads a Markdown table with details + escaped pipes", () => { + const md = [ + "| id | scenario | category | target | scope | details | doc |", + "| --- | --- | --- | --- | --- | --- | --- |", + "| x1 | S | access | Widget | | internal=true | Hidden \\| renamed. |", + ].join("\n"); + const items = parseChecksDoc(md); + expect(items[0]).toMatchObject({ + id: "x1", + scenario: "S", + category: "access", + target: "Widget", + details: { internal: true }, + doc: "Hidden | renamed.", + }); + }); +}); From 7deaeed6070f9b7f2c41c08c14e0dd10e04375a3 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 27 Jul 2026 12:59:51 -0400 Subject: [PATCH 2/9] fix(spector): add missing SurfaceDoc state key to lib.ts The @surfaceDoc decorator uses SpectorStateKeys.SurfaceDoc to store state, but the key was never declared in the state map, which would cause a runtime error when the decorator is used. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 54fb668c-64c3-41e5-8933-c4300c884974 --- packages/spector/src/lib/lib.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/spector/src/lib/lib.ts b/packages/spector/src/lib/lib.ts index b4450552ec1..de2db227390 100644 --- a/packages/spector/src/lib/lib.ts +++ b/packages/spector/src/lib/lib.ts @@ -27,6 +27,7 @@ export const $lib = createTypeSpecLibrary({ Scenario: { description: "Mark a scenario to be executed" }, ScenarioDoc: { description: "Mark a scenario documentation" }, ScenarioService: { description: "Mark a scenario service to be executed" }, + SurfaceDoc: { description: "Mark a surface documentation check" }, }, }); From b7d228d7a66c3e50d218b337f77c18b422fd629c Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 29 Jul 2026 13:08:08 -0400 Subject: [PATCH 3/9] feat(spector): migrate @surfaceDoc to auto dec with positional params Convert @surfaceDoc from extern dec to auto dec: - Use positional params (category, subject, expected, doc?) - Add tspconfig.yaml with kind:project and auto-decorators feature - tspd gen-extern-signature now generates getSurfaceDoc/setSurfaceDoc accessors - Remove surfaceDoc from TypeSpecSpectorDecorators and decorator map - Update all spec call sites from keyword #{} syntax to positional - Clean up tests: remove autoDecOptions workaround Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 54fb668c-64c3-41e5-8933-c4300c884974 --- .../specs/payload/pageable/main.tsp | 18 ++++ .../generated-defs/TypeSpec.Spector.ts | 62 +++++++++++-- packages/spector/lib/main.tsp | 49 ++++------ packages/spector/src/lib/decorators.ts | 92 ++++++++----------- packages/spector/src/lib/index.ts | 1 - packages/spector/src/lib/lib.ts | 1 - packages/spector/src/lib/tsp-index.ts | 3 +- packages/spector/test/surface-docs.test.ts | 66 +++++-------- packages/spector/tspconfig.yaml | 3 + 9 files changed, 153 insertions(+), 142 deletions(-) create mode 100644 packages/spector/tspconfig.yaml diff --git a/packages/http-specs/specs/payload/pageable/main.tsp b/packages/http-specs/specs/payload/pageable/main.tsp index db89029037c..57c8dcd3884 100644 --- a/packages/http-specs/specs/payload/pageable/main.tsp +++ b/packages/http-specs/specs/payload/pageable/main.tsp @@ -51,6 +51,12 @@ namespace ServerDrivenPagination { } ``` """) + @surfaceDoc( + "paging", + Pageable.ServerDrivenPagination.link, + "Pet", + "The paginated operation yields `Pet` elements." + ) @route("/link") @list op link(): { @@ -518,6 +524,12 @@ namespace PageSize { } ``` """) + @surfaceDoc( + "paging", + Pageable.PageSize.listWithoutContinuation, + "Pet", + "Single-page pageable operation yielding `Pet` elements, no continuation." + ) @route("/without-continuation") @list op listWithoutContinuation(): { @@ -556,6 +568,12 @@ namespace PageSize { } ``` """) + @surfaceDoc( + "paging", + Pageable.PageSize.listWithPageSize, + "Pet", + "Page-size-driven pageable operation yielding `Pet` elements." + ) @route("/list") @list op listWithPageSize(@pageSize @query pageSize?: int32): { diff --git a/packages/spector/generated-defs/TypeSpec.Spector.ts b/packages/spector/generated-defs/TypeSpec.Spector.ts index 62c13be7be7..fdc4334ef99 100644 --- a/packages/spector/generated-defs/TypeSpec.Spector.ts +++ b/packages/spector/generated-defs/TypeSpec.Spector.ts @@ -1,11 +1,14 @@ -import type { - DecoratorContext, - DecoratorValidatorCallbacks, - Interface, - Model, - Namespace, - Operation, - Type, +import { + type DecoratorContext, + type DecoratorValidatorCallbacks, + getAutoDecoratorValue, + type Interface, + type Model, + type Namespace, + type Operation, + type Program, + setAutoDecorator, + type Type, } from "@typespec/compiler"; /** @@ -45,3 +48,46 @@ export type TypeSpecSpectorDecorators = { scenario: ScenarioDecorator; scenarioDoc: ScenarioDocDecorator; }; + +export function getSurfaceDoc( + program: Program, + target: Namespace | Interface | Operation, +): + | { + category: + | string + | "naming" + | "access" + | "client-location" + | "hierarchy" + | "flatten" + | "paging" + | "other"; + subject: Type; + expected: string | Record; + doc: string; + } + | undefined { + return getAutoDecoratorValue(program, "TypeSpec.Spector.surfaceDoc", target) as any; +} + +export function setSurfaceDoc( + program: Program, + target: Namespace | Interface | Operation, + value: { + category: + | string + | "naming" + | "access" + | "client-location" + | "hierarchy" + | "flatten" + | "paging" + | "other"; + subject: Type; + expected: string | Record; + doc: string; + }, +): void { + setAutoDecorator(program, "TypeSpec.Spector.surfaceDoc", target, value); +} diff --git a/packages/spector/lib/main.tsp b/packages/spector/lib/main.tsp index a58a33a9b10..3dac1ce0cb7 100644 --- a/packages/spector/lib/main.tsp +++ b/packages/spector/lib/main.tsp @@ -87,45 +87,30 @@ union SurfaceCategory { * relocated to another client, or a reshaped inheritance hierarchy. Mirrors * `@scenarioDoc`, but for the surface instead of the wire. * - * To keep surface checks deterministic and grounded in a real scenario, * `@surfaceDoc` may only be applied to an element that also carries - * `@scenarioDoc` (a `@scenario` namespace/interface/operation). The check itself - * is fully explicit: the author states the `category`, the `subject` the check - * is about (a type/member reference — which may differ from the annotated - * operation), and the `expected` client-surface output. Nothing is inferred from - * other decorators, so adding or reading a check needs no knowledge of the - * client-generator vocabulary. + * `@scenarioDoc`, so every surface check is grounded in a documented scenario. + * Like `@scenarioDoc`, each target gets exactly one `@surfaceDoc`. * - * The precompute step (`listSurfaceDocs`) records these language-agnostically; - * each emitter's `verifiers.json` decides how to check a category using the - * generic `{target}` (subject name), `{expected}`, `{kind}` (subject kind) and - * `{origin}` (subject's declaring container) placeholders. A category with no - * verifier for an emitter is verified against the prose by AI — so introducing a - * new category never requires any emitter to change. + * ```tsp + * @surfaceDoc(#{ + * category: "naming", + * subject: widgetName, + * expected: "widgetName", + * }) + * ``` * - * By default `expected` is a single canonical, language-agnostic string and each - * emitter recasts it to its idiomatic form (e.g. `{expected:byKind}`). When a - * name can't be derived by casing (a reserved-word escape, an acronym, a hard - * rename), pass a **dict mapping scope → exact value** instead — keyed the same - * way the client-generator decorators are scoped (`"python"`, `"python,csharp"`, - * `"!java"`). A dict value is matched **verbatim** for its scope; a language not - * listed in the dict is not checked. + * `expected` is a single canonical, language-agnostic string by default, recast + * by each emitter. For exact per-language names, pass a `scope → value` dict: * * ```tsp - * // idiomatic: one canonical name, every emitter recasts it - * @surfaceDoc("naming", widgetName, "widgetName") - * - * // exact per language: each scope matched verbatim - * @surfaceDoc("naming", ioThing, #{ python: "io_thing", csharp: "IOThing" }) + * @surfaceDoc(#{ + * category: "naming", + * subject: ioThing, + * expected: #{ python: "io_thing", csharp: "IOThing" }, + * }) * ``` - * - * @param category The kind of surface assertion. See {@link SurfaceCategory}. - * @param subject The type or member the check is about (may differ from `target`). - * @param expected The expected client-surface output: a single string (recast per - * language) or a `scope → value` dict (each value matched verbatim for its scope). - * @param doc Optional extra natural-language description. Markdown. */ -extern dec surfaceDoc( +auto dec surfaceDoc( target: Namespace | Interface | Operation, category: valueof SurfaceCategory, subject: unknown, diff --git a/packages/spector/src/lib/decorators.ts b/packages/spector/src/lib/decorators.ts index 0f5e2a2e454..bf8a829933f 100644 --- a/packages/spector/src/lib/decorators.ts +++ b/packages/spector/src/lib/decorators.ts @@ -2,6 +2,8 @@ import { $service, Enum, EnumMember, + getAutoDecoratorTargets, + getAutoDecoratorValue, getNamespaceFullName, getTypeName, Interface, @@ -27,7 +29,6 @@ import { ScenarioDecorator, ScenarioDocDecorator, ScenarioServiceDecorator, - SurfaceDocDecorator, } from "../../generated-defs/TypeSpec.Spector.js"; import { SpectorStateKeys } from "./lib.js"; @@ -40,39 +41,6 @@ export const $scenarioDoc: ScenarioDocDecorator = (context, target, doc, formatA context.program.stateMap(SpectorStateKeys.ScenarioDoc).set(target, formattedDoc); }; -export const $surfaceDoc: SurfaceDocDecorator = ( - context, - target, - category, - subject, - expected, - doc?, -) => { - const map = context.program.stateMap(SpectorStateKeys.SurfaceDoc); - const existing: StoredSurfaceDoc[] = map.get(target) ?? []; - existing.push({ - category, - subject: subject as SurfaceSubject, - expected: expected as string | Record, - doc, - }); - // A target can carry several `@surfaceDoc`s (different categories), so store a - // list — never overwrite a previously recorded check. - map.set(target, existing); -}; - -/** - * What a single `@surfaceDoc` application records. `expected` is either one - * canonical string (recast per language) or a `scope → value` dict (each value - * matched verbatim for its scope). - */ -interface StoredSurfaceDoc { - category: string; - subject: SurfaceSubject; - expected: string | Record; - doc?: string; -} - export const $scenarioService: ScenarioServiceDecorator = (context, target, route, options?) => { const properties = new Map().set("title", { type: { kind: "String", value: getNamespaceFullName(target).replace(/\./g, "") }, @@ -378,6 +346,21 @@ function getEnclosingScenarioName(program: Program, target: SurfaceDocTarget): s return undefined; } +/** The FQN used by the auto dec for `@surfaceDoc`. */ +const SURFACE_DOC_FQN = "TypeSpec.Spector.surfaceDoc"; + +/** + * What the auto dec stores for `@surfaceDoc`. With positional params + * `category`, `subject`, `expected`, `doc?`, the auto dec stores: + * `{ category, subject, expected, doc }`. + */ +interface StoredSurfaceDoc { + category: string; + subject: SurfaceSubject; + expected: string | Record; + doc?: string; +} + /** * Collect every `@surfaceDoc` in the program into a list of language-agnostic * surface docs. Analogous to {@link listScenarios}, but for the generated @@ -386,22 +369,24 @@ function getEnclosingScenarioName(program: Program, target: SurfaceDocTarget): s * decorators. Feeds the `surface-checks.md` checks doc. */ export function listSurfaceDocs(program: Program): SurfaceDoc[] { - const map = program.stateMap(SpectorStateKeys.SurfaceDoc); + const targets = getAutoDecoratorTargets(program, SURFACE_DOC_FQN); const result: SurfaceDoc[] = []; - for (const [target, storedList] of map as Map) { - for (const stored of storedList) { - const scenario = getEnclosingScenarioName(program, target); - for (const { expected, scope } of expandExpected(stored.expected)) { - result.push({ - scenario, - target, - subject: stored.subject, - category: stored.category, - expected, - scope, - doc: stored.doc ?? synthesizeDoc(stored.category, stored.subject, expected), - }); - } + for (const [target] of targets) { + const stored = getAutoDecoratorValue(program, SURFACE_DOC_FQN, target) as + StoredSurfaceDoc | undefined; + if (!stored) continue; + const docTarget = target as SurfaceDocTarget; + const scenario = getEnclosingScenarioName(program, docTarget); + for (const { expected, scope } of expandExpected(stored.expected)) { + result.push({ + scenario, + target: docTarget, + subject: stored.subject, + category: stored.category, + expected, + scope, + doc: stored.doc ?? synthesizeDoc(stored.category, stored.subject, expected), + }); } } return result.sort( @@ -422,11 +407,12 @@ export function listSurfaceDocs(program: Program): SurfaceDoc[] { * currently-dormant scenario validations for consumers that only compile specs. */ export function listSurfaceDocsMissingScenarioDoc(program: Program): SurfaceDocTarget[] { - const map = program.stateMap(SpectorStateKeys.SurfaceDoc); + const targets = getAutoDecoratorTargets(program, SURFACE_DOC_FQN); const result: SurfaceDocTarget[] = []; - for (const target of map.keys() as Iterable) { - if (getScenarioDoc(program, target) === undefined) { - result.push(target); + for (const [target] of targets) { + const docTarget = target as SurfaceDocTarget; + if (getScenarioDoc(program, docTarget) === undefined) { + result.push(docTarget); } } return result; diff --git a/packages/spector/src/lib/index.ts b/packages/spector/src/lib/index.ts index 041f35bb9d6..efa3408a1a0 100644 --- a/packages/spector/src/lib/index.ts +++ b/packages/spector/src/lib/index.ts @@ -2,7 +2,6 @@ export { $scenario, $scenarioDoc, $scenarioService, - $surfaceDoc, Scenario, ScenarioEndpoint, SurfaceDetails, diff --git a/packages/spector/src/lib/lib.ts b/packages/spector/src/lib/lib.ts index de2db227390..b4450552ec1 100644 --- a/packages/spector/src/lib/lib.ts +++ b/packages/spector/src/lib/lib.ts @@ -27,7 +27,6 @@ export const $lib = createTypeSpecLibrary({ Scenario: { description: "Mark a scenario to be executed" }, ScenarioDoc: { description: "Mark a scenario documentation" }, ScenarioService: { description: "Mark a scenario service to be executed" }, - SurfaceDoc: { description: "Mark a surface documentation check" }, }, }); diff --git a/packages/spector/src/lib/tsp-index.ts b/packages/spector/src/lib/tsp-index.ts index d4be5ba8252..bf29ae1369e 100644 --- a/packages/spector/src/lib/tsp-index.ts +++ b/packages/spector/src/lib/tsp-index.ts @@ -1,5 +1,5 @@ import type { TypeSpecSpectorDecorators } from "../../generated-defs/TypeSpec.Spector.js"; -import { $scenario, $scenarioDoc, $scenarioService, $surfaceDoc } from "./decorators.js"; +import { $scenario, $scenarioDoc, $scenarioService } from "./decorators.js"; export { $lib } from "./lib.js"; /** @internal */ @@ -8,6 +8,5 @@ export const $decorators = { scenario: $scenario, scenarioDoc: $scenarioDoc, scenarioService: $scenarioService, - surfaceDoc: $surfaceDoc, } satisfies TypeSpecSpectorDecorators, }; diff --git a/packages/spector/test/surface-docs.test.ts b/packages/spector/test/surface-docs.test.ts index 63a1b2312d4..b8b0b49cee1 100644 --- a/packages/spector/test/surface-docs.test.ts +++ b/packages/spector/test/surface-docs.test.ts @@ -54,7 +54,7 @@ describe("@surfaceDoc", () => { expect(doc.doc).toBe("Exposed to clients as ClientExtensibleEnum."); }); - it("resolves the scenario name from the annotated element, shared by all its checks", async () => { + it("resolves the scenario name from the annotated element", async () => { const doc = await docOf(` enum ServerExtensibleEnum { value1, @@ -102,7 +102,8 @@ describe("@surfaceDoc", () => { // --- validation: must be grounded in a scenario doc --------------------- it("flags a surface doc whose target has no @scenarioDoc", async () => { - const { program } = await Tester.compile(` + const { program } = await Tester.compile( + ` model Widget { id: string; } @@ -110,13 +111,15 @@ describe("@surfaceDoc", () => { @scenario @surfaceDoc("access", Widget, "internal") op get(): Widget; - `); + `, + ); const missing = listSurfaceDocsMissingScenarioDoc(program); expect(missing).toHaveLength(1); }); it("does not flag a surface doc whose target also has @scenarioDoc", async () => { - const { program } = await Tester.compile(` + const { program } = await Tester.compile( + ` model Widget { id: string; } @@ -125,7 +128,8 @@ describe("@surfaceDoc", () => { @scenarioDoc("Get a widget.") @surfaceDoc("access", Widget, "internal") op get(): Widget; - `); + `, + ); expect(listSurfaceDocsMissingScenarioDoc(program)).toHaveLength(0); }); @@ -203,7 +207,8 @@ describe("@surfaceDoc", () => { // --- rendering ---------------------------------------------------------- it("renders a single Markdown table that carries every routable field", async () => { - const { program } = await Tester.compile(` + const { program } = await Tester.compile( + ` model Widget { id: string; } @@ -212,7 +217,8 @@ describe("@surfaceDoc", () => { @scenarioDoc("Get a widget.") @surfaceDoc("naming", Widget, "WidgetInternal", "Hidden | renamed to WidgetInternal.") op get(): Widget; - `); + `, + ); const manifest = createSurfaceChecksManifest(".", "1.0.0", "abc123", listSurfaceDocs(program)); const md = await createSurfaceChecksSummary(manifest); @@ -234,7 +240,8 @@ describe("@surfaceDoc", () => { // --- per-language exact names (scope → value dict) ---------------------- it("expands a `scope → value` dict into one verbatim check per scope", async () => { - const { program } = await Tester.compile(` + const { program } = await Tester.compile( + ` model IOThing { id: string; } @@ -243,7 +250,8 @@ describe("@surfaceDoc", () => { @scenarioDoc("Get the thing.") @surfaceDoc("naming", IOThing, #{ python: "io_thing", csharp: "IOThing" }) op get(): IOThing; - `); + `, + ); const docs = listSurfaceDocs(program); expect(docs).toHaveLength(2); const byScope = Object.fromEntries(docs.map((d) => [d.scope, d])); @@ -267,7 +275,8 @@ describe("@surfaceDoc", () => { }); it("includes the scope in the manifest id and renders a scope column", async () => { - const { program } = await Tester.compile(` + const { program } = await Tester.compile( + ` model IOThing { id: string; } @@ -276,7 +285,8 @@ describe("@surfaceDoc", () => { @scenarioDoc("Get the thing.") @surfaceDoc("naming", IOThing, #{ python: "io_thing" }) op get(): IOThing; - `); + `, + ); const manifest = createSurfaceChecksManifest(".", "1.0.0", "abc123", listSurfaceDocs(program)); const item = manifest.items.find((i) => i.id === "get_naming_python"); expect(item).toBeDefined(); @@ -286,38 +296,4 @@ describe("@surfaceDoc", () => { expect(md).toContain("io_thing"); expect(md).toContain("python"); }); - - it("records multiple @surfaceDocs on one scenario, sharing its name with distinct ids", async () => { - const { program } = await Tester.compile(` - model Widget { - id: string; - } - - @scenario - @scenarioDoc("Get a widget.") - @surfaceDoc("naming", Widget, "WidgetClient") - @surfaceDoc("access", Widget, "internal") - op get(): Widget; - `); - const docs = listSurfaceDocs(program); - expect(docs).toHaveLength(2); - expect(docs.map((d) => d.category).sort()).toEqual(["access", "naming"]); - // Every check on the scenario shares its resolved scenario name... - expect(docs.map((d) => d.scenario)).toEqual(["get", "get"]); - - // ...and each surfaces as its own check, keyed `_`. - const items = await manifestItems(` - model Widget { - id: string; - } - - @scenario - @scenarioDoc("Get a widget.") - @surfaceDoc("naming", Widget, "WidgetClient") - @surfaceDoc("access", Widget, "internal") - op get(): Widget; - `); - expect(items["get_naming"].scenario).toBe("get"); - expect(items["get_access"].scenario).toBe("get"); - }); }); diff --git a/packages/spector/tspconfig.yaml b/packages/spector/tspconfig.yaml new file mode 100644 index 00000000000..d4912c747a4 --- /dev/null +++ b/packages/spector/tspconfig.yaml @@ -0,0 +1,3 @@ +kind: project +features: + - auto-decorators From 27ceafe76b64eaa6a949df2dc8fdb8fd50cdf941 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 29 Jul 2026 13:16:22 -0400 Subject: [PATCH 4/9] refactor(spector): use hybrid syntax for @surfaceDoc Change @surfaceDoc to hybrid call style: @surfaceDoc(Subject, #{ category: "...", expected: "...", doc: "..." }) subject stays positional (type reference), rest uses keyword object literal. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 54fb668c-64c3-41e5-8933-c4300c884974 --- .../specs/payload/pageable/main.tsp | 20 ++++---- .../serialization/encoded-name/json/main.tsp | 8 ++-- .../inheritance/not-discriminated/main.tsp | 8 ++-- .../inheritance/single-discriminator/main.tsp | 8 ++-- .../generated-defs/TypeSpec.Spector.ts | 48 ++++++++++--------- packages/spector/lib/main.tsp | 14 +++--- packages/spector/src/lib/decorators.ts | 23 +++++---- packages/spector/test/surface-docs.test.ts | 28 +++++------ 8 files changed, 86 insertions(+), 71 deletions(-) diff --git a/packages/http-specs/specs/payload/pageable/main.tsp b/packages/http-specs/specs/payload/pageable/main.tsp index 57c8dcd3884..d3001bfe776 100644 --- a/packages/http-specs/specs/payload/pageable/main.tsp +++ b/packages/http-specs/specs/payload/pageable/main.tsp @@ -52,10 +52,8 @@ namespace ServerDrivenPagination { ``` """) @surfaceDoc( - "paging", Pageable.ServerDrivenPagination.link, - "Pet", - "The paginated operation yields `Pet` elements." + #{ category: "paging", expected: "Pet", doc: "The paginated operation yields `Pet` elements." } ) @route("/link") @list @@ -525,10 +523,12 @@ namespace PageSize { ``` """) @surfaceDoc( - "paging", Pageable.PageSize.listWithoutContinuation, - "Pet", - "Single-page pageable operation yielding `Pet` elements, no continuation." + #{ + category: "paging", + expected: "Pet", + doc: "Single-page pageable operation yielding `Pet` elements, no continuation.", + } ) @route("/without-continuation") @list @@ -569,10 +569,12 @@ namespace PageSize { ``` """) @surfaceDoc( - "paging", Pageable.PageSize.listWithPageSize, - "Pet", - "Page-size-driven pageable operation yielding `Pet` elements." + #{ + category: "paging", + expected: "Pet", + doc: "Page-size-driven pageable operation yielding `Pet` elements.", + } ) @route("/list") @list diff --git a/packages/http-specs/specs/serialization/encoded-name/json/main.tsp b/packages/http-specs/specs/serialization/encoded-name/json/main.tsp index 349842738ce..1461a957057 100644 --- a/packages/http-specs/specs/serialization/encoded-name/json/main.tsp +++ b/packages/http-specs/specs/serialization/encoded-name/json/main.tsp @@ -27,10 +27,12 @@ namespace Property { ``` """) @surfaceDoc( - "naming", JsonEncodedNameModel.defaultName, - "defaultName", - "The client property is named `defaultName` even though the wire name is `wireName`." + #{ + category: "naming", + expected: "defaultName", + doc: "The client property is named `defaultName` even though the wire name is `wireName`.", + } ) @post op send(@bodyRoot body: JsonEncodedNameModel): NoContentResponse; diff --git a/packages/http-specs/specs/type/model/inheritance/not-discriminated/main.tsp b/packages/http-specs/specs/type/model/inheritance/not-discriminated/main.tsp index 7ae7d905556..66c0407fcbf 100644 --- a/packages/http-specs/specs/type/model/inheritance/not-discriminated/main.tsp +++ b/packages/http-specs/specs/type/model/inheritance/not-discriminated/main.tsp @@ -44,10 +44,12 @@ op postValid(@body input: Siamese): NoContentResponse; ``` """) @surfaceDoc( - "hierarchy", Siamese, - "Cat", - "The `Siamese` model extends `Cat` as its base type on the client surface." + #{ + category: "hierarchy", + expected: "Cat", + doc: "The `Siamese` model extends `Cat` as its base type on the client surface.", + } ) @route("/valid") @get diff --git a/packages/http-specs/specs/type/model/inheritance/single-discriminator/main.tsp b/packages/http-specs/specs/type/model/inheritance/single-discriminator/main.tsp index ac2f0f0f36c..3133a06a22a 100644 --- a/packages/http-specs/specs/type/model/inheritance/single-discriminator/main.tsp +++ b/packages/http-specs/specs/type/model/inheritance/single-discriminator/main.tsp @@ -66,10 +66,12 @@ model Fish { ``` """) @surfaceDoc( - "hierarchy", Sparrow, - "Bird", - "The `Sparrow` model extends `Bird` as its base type on the client surface." + #{ + category: "hierarchy", + expected: "Bird", + doc: "The `Sparrow` model extends `Bird` as its base type on the client surface.", + } ) @get op getModel(): Bird; diff --git a/packages/spector/generated-defs/TypeSpec.Spector.ts b/packages/spector/generated-defs/TypeSpec.Spector.ts index fdc4334ef99..5e0c3d05224 100644 --- a/packages/spector/generated-defs/TypeSpec.Spector.ts +++ b/packages/spector/generated-defs/TypeSpec.Spector.ts @@ -54,18 +54,20 @@ export function getSurfaceDoc( target: Namespace | Interface | Operation, ): | { - category: - | string - | "naming" - | "access" - | "client-location" - | "hierarchy" - | "flatten" - | "paging" - | "other"; subject: Type; - expected: string | Record; - doc: string; + check: { + readonly category: + | string + | "naming" + | "access" + | "client-location" + | "hierarchy" + | "flatten" + | "paging" + | "other"; + readonly expected: string | Record; + readonly doc?: string; + }; } | undefined { return getAutoDecoratorValue(program, "TypeSpec.Spector.surfaceDoc", target) as any; @@ -75,18 +77,20 @@ export function setSurfaceDoc( program: Program, target: Namespace | Interface | Operation, value: { - category: - | string - | "naming" - | "access" - | "client-location" - | "hierarchy" - | "flatten" - | "paging" - | "other"; subject: Type; - expected: string | Record; - doc: string; + check: { + readonly category: + | string + | "naming" + | "access" + | "client-location" + | "hierarchy" + | "flatten" + | "paging" + | "other"; + readonly expected: string | Record; + readonly doc?: string; + }; }, ): void { setAutoDecorator(program, "TypeSpec.Spector.surfaceDoc", target, value); diff --git a/packages/spector/lib/main.tsp b/packages/spector/lib/main.tsp index 3dac1ce0cb7..49b2f11ee08 100644 --- a/packages/spector/lib/main.tsp +++ b/packages/spector/lib/main.tsp @@ -92,9 +92,8 @@ union SurfaceCategory { * Like `@scenarioDoc`, each target gets exactly one `@surfaceDoc`. * * ```tsp - * @surfaceDoc(#{ + * @surfaceDoc(widgetName, #{ * category: "naming", - * subject: widgetName, * expected: "widgetName", * }) * ``` @@ -103,17 +102,18 @@ union SurfaceCategory { * by each emitter. For exact per-language names, pass a `scope → value` dict: * * ```tsp - * @surfaceDoc(#{ + * @surfaceDoc(ioThing, #{ * category: "naming", - * subject: ioThing, * expected: #{ python: "io_thing", csharp: "IOThing" }, * }) * ``` */ auto dec surfaceDoc( target: Namespace | Interface | Operation, - category: valueof SurfaceCategory, subject: unknown, - expected: valueof string | Record, - doc?: valueof string + check: valueof { + category: SurfaceCategory, + expected: string | Record, + doc?: string, + } ); diff --git a/packages/spector/src/lib/decorators.ts b/packages/spector/src/lib/decorators.ts index bf8a829933f..15fec46a47a 100644 --- a/packages/spector/src/lib/decorators.ts +++ b/packages/spector/src/lib/decorators.ts @@ -350,15 +350,17 @@ function getEnclosingScenarioName(program: Program, target: SurfaceDocTarget): s const SURFACE_DOC_FQN = "TypeSpec.Spector.surfaceDoc"; /** - * What the auto dec stores for `@surfaceDoc`. With positional params - * `category`, `subject`, `expected`, `doc?`, the auto dec stores: - * `{ category, subject, expected, doc }`. + * What the auto dec stores for `@surfaceDoc`. With params + * `subject: unknown, check: valueof { category, expected, doc? }`, + * the auto dec stores: `{ subject: Type, check: { category, expected, doc } }`. */ interface StoredSurfaceDoc { - category: string; subject: SurfaceSubject; - expected: string | Record; - doc?: string; + check: { + category: string; + expected: string | Record; + doc?: string; + }; } /** @@ -377,15 +379,16 @@ export function listSurfaceDocs(program: Program): SurfaceDoc[] { if (!stored) continue; const docTarget = target as SurfaceDocTarget; const scenario = getEnclosingScenarioName(program, docTarget); - for (const { expected, scope } of expandExpected(stored.expected)) { + const { category, expected, doc } = stored.check; + for (const { expected: exp, scope } of expandExpected(expected)) { result.push({ scenario, target: docTarget, subject: stored.subject, - category: stored.category, - expected, + category, + expected: exp, scope, - doc: stored.doc ?? synthesizeDoc(stored.category, stored.subject, expected), + doc: doc ?? synthesizeDoc(category, stored.subject, exp), }); } } diff --git a/packages/spector/test/surface-docs.test.ts b/packages/spector/test/surface-docs.test.ts index b8b0b49cee1..29f522ec12e 100644 --- a/packages/spector/test/surface-docs.test.ts +++ b/packages/spector/test/surface-docs.test.ts @@ -45,7 +45,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Return the kind.") - @surfaceDoc("naming", ServerExtensibleEnum, "ClientExtensibleEnum", "Exposed to clients as ClientExtensibleEnum.") + @surfaceDoc(ServerExtensibleEnum, #{ category: "naming", expected: "ClientExtensibleEnum", doc: "Exposed to clients as ClientExtensibleEnum." }) op getKind(): ServerExtensibleEnum; `); expect(doc.category).toBe("naming"); @@ -63,7 +63,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Return the kind.") - @surfaceDoc("naming", ServerExtensibleEnum, "ClientExtensibleEnum") + @surfaceDoc(ServerExtensibleEnum, #{ category: "naming", expected: "ClientExtensibleEnum" }) op getKind(): ServerExtensibleEnum; `); expect(doc.scenario).toBe("getKind"); @@ -77,7 +77,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get a widget.") - @surfaceDoc("access", Widget, "internal") + @surfaceDoc(Widget, #{ category: "access", expected: "internal" }) op get(): Widget; `); expect(doc.doc).toContain("access"); @@ -93,7 +93,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get a widget.") - @surfaceDoc("brand-new-category", Widget, "whatever") + @surfaceDoc(Widget, #{ category: "brand-new-category", expected: "whatever" }) op get(): Widget; `); expect(doc.category).toBe("brand-new-category"); @@ -109,7 +109,7 @@ describe("@surfaceDoc", () => { } @scenario - @surfaceDoc("access", Widget, "internal") + @surfaceDoc(Widget, #{ category: "access", expected: "internal" }) op get(): Widget; `, ); @@ -126,7 +126,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get a widget.") - @surfaceDoc("access", Widget, "internal") + @surfaceDoc(Widget, #{ category: "access", expected: "internal" }) op get(): Widget; `, ); @@ -144,7 +144,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Return the kind.") - @surfaceDoc("naming", ServerExtensibleEnum, "ClientExtensibleEnum", "Exposed as ClientExtensibleEnum.") + @surfaceDoc(ServerExtensibleEnum, #{ category: "naming", expected: "ClientExtensibleEnum", doc: "Exposed as ClientExtensibleEnum." }) op getKind(): ServerExtensibleEnum; `); const item = items["getKind_naming"]; @@ -167,7 +167,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get a widget.") - @surfaceDoc("naming", Widget.id, "identifier", "Renamed to identifier.") + @surfaceDoc(Widget.id, #{ category: "naming", expected: "identifier", doc: "Renamed to identifier." }) op get(): Widget; `); const item = items["get_naming"]; @@ -180,7 +180,7 @@ describe("@surfaceDoc", () => { const items = await manifestItems(` @scenario @scenarioDoc("List items.") - @surfaceDoc("paging", listItems, "") + @surfaceDoc(listItems, #{ category: "paging", expected: "" }) op listItems(): string[]; `); const item = items["listItems_paging"]; @@ -196,7 +196,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get a widget.") - @surfaceDoc("other", Widget, "strongly typed", "The response body is a strongly typed model.") + @surfaceDoc(Widget, #{ category: "other", expected: "strongly typed", doc: "The response body is a strongly typed model." }) op get(): Widget; `); const item = items["get_other"]; @@ -215,7 +215,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get a widget.") - @surfaceDoc("naming", Widget, "WidgetInternal", "Hidden | renamed to WidgetInternal.") + @surfaceDoc(Widget, #{ category: "naming", expected: "WidgetInternal", doc: "Hidden | renamed to WidgetInternal." }) op get(): Widget; `, ); @@ -248,7 +248,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get the thing.") - @surfaceDoc("naming", IOThing, #{ python: "io_thing", csharp: "IOThing" }) + @surfaceDoc(IOThing, #{ category: "naming", expected: #{ python: "io_thing", csharp: "IOThing" } }) op get(): IOThing; `, ); @@ -268,7 +268,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get a widget.") - @surfaceDoc("naming", Widget, "Widget") + @surfaceDoc(Widget, #{ category: "naming", expected: "Widget" }) op get(): Widget; `); expect(doc.scope).toBeUndefined(); @@ -283,7 +283,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get the thing.") - @surfaceDoc("naming", IOThing, #{ python: "io_thing" }) + @surfaceDoc(IOThing, #{ category: "naming", expected: #{ python: "io_thing" } }) op get(): IOThing; `, ); From 58b84f1326e9f92f18b1baf62a0d29a3060de4d5 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 29 Jul 2026 13:33:28 -0400 Subject: [PATCH 5/9] refactor(spector): all-keyword @surfaceDoc with subject as optional string @surfaceDoc now uses fully keyword syntax: @surfaceDoc(#{ category: "naming", expected: "W", subject: "Widget" }) - subject is optional string, defaults to target's name when omitted - Removed SurfaceSubject type (no longer holds Type references) - Removed kind/origin from SurfaceDetails (no longer derivable) - Simplified buildSurfaceDetails, removed getSurfaceParent/getSurfaceKind - Location now derived from target (decorated element) not subject Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 54fb668c-64c3-41e5-8933-c4300c884974 --- .../specs/payload/pageable/main.tsp | 35 +++--- .../serialization/encoded-name/json/main.tsp | 14 +-- .../inheritance/not-discriminated/main.tsp | 14 +-- .../inheritance/single-discriminator/main.tsp | 14 +-- .../generated-defs/TypeSpec.Spector.ts | 50 ++++---- packages/spector/lib/main.tsp | 9 +- .../src/coverage/surface-checks-manifest.ts | 23 ++-- packages/spector/src/lib/decorators.ts | 119 +++--------------- packages/spector/src/lib/index.ts | 1 - packages/spector/test/surface-docs.test.ts | 42 +++---- 10 files changed, 111 insertions(+), 210 deletions(-) diff --git a/packages/http-specs/specs/payload/pageable/main.tsp b/packages/http-specs/specs/payload/pageable/main.tsp index d3001bfe776..e25ba2c733b 100644 --- a/packages/http-specs/specs/payload/pageable/main.tsp +++ b/packages/http-specs/specs/payload/pageable/main.tsp @@ -51,10 +51,11 @@ namespace ServerDrivenPagination { } ``` """) - @surfaceDoc( - Pageable.ServerDrivenPagination.link, - #{ category: "paging", expected: "Pet", doc: "The paginated operation yields `Pet` elements." } - ) + @surfaceDoc(#{ + category: "paging", + expected: "Pet", + doc: "The paginated operation yields `Pet` elements.", + }) @route("/link") @list op link(): { @@ -522,14 +523,11 @@ namespace PageSize { } ``` """) - @surfaceDoc( - Pageable.PageSize.listWithoutContinuation, - #{ - category: "paging", - expected: "Pet", - doc: "Single-page pageable operation yielding `Pet` elements, no continuation.", - } - ) + @surfaceDoc(#{ + category: "paging", + expected: "Pet", + doc: "Single-page pageable operation yielding `Pet` elements, no continuation.", + }) @route("/without-continuation") @list op listWithoutContinuation(): { @@ -568,14 +566,11 @@ namespace PageSize { } ``` """) - @surfaceDoc( - Pageable.PageSize.listWithPageSize, - #{ - category: "paging", - expected: "Pet", - doc: "Page-size-driven pageable operation yielding `Pet` elements.", - } - ) + @surfaceDoc(#{ + category: "paging", + expected: "Pet", + doc: "Page-size-driven pageable operation yielding `Pet` elements.", + }) @route("/list") @list op listWithPageSize(@pageSize @query pageSize?: int32): { diff --git a/packages/http-specs/specs/serialization/encoded-name/json/main.tsp b/packages/http-specs/specs/serialization/encoded-name/json/main.tsp index 1461a957057..0a94c5edd24 100644 --- a/packages/http-specs/specs/serialization/encoded-name/json/main.tsp +++ b/packages/http-specs/specs/serialization/encoded-name/json/main.tsp @@ -26,14 +26,12 @@ namespace Property { {"wireName": true} ``` """) - @surfaceDoc( - JsonEncodedNameModel.defaultName, - #{ - category: "naming", - expected: "defaultName", - doc: "The client property is named `defaultName` even though the wire name is `wireName`.", - } - ) + @surfaceDoc(#{ + category: "naming", + subject: "JsonEncodedNameModel.defaultName", + expected: "defaultName", + doc: "The client property is named `defaultName` even though the wire name is `wireName`.", + }) @post op send(@bodyRoot body: JsonEncodedNameModel): NoContentResponse; diff --git a/packages/http-specs/specs/type/model/inheritance/not-discriminated/main.tsp b/packages/http-specs/specs/type/model/inheritance/not-discriminated/main.tsp index 66c0407fcbf..730b8bcf69f 100644 --- a/packages/http-specs/specs/type/model/inheritance/not-discriminated/main.tsp +++ b/packages/http-specs/specs/type/model/inheritance/not-discriminated/main.tsp @@ -43,14 +43,12 @@ op postValid(@body input: Siamese): NoContentResponse; {"name": "abc", "age": 32, "smart": true} ``` """) -@surfaceDoc( - Siamese, - #{ - category: "hierarchy", - expected: "Cat", - doc: "The `Siamese` model extends `Cat` as its base type on the client surface.", - } -) +@surfaceDoc(#{ + category: "hierarchy", + subject: "Siamese", + expected: "Cat", + doc: "The `Siamese` model extends `Cat` as its base type on the client surface.", +}) @route("/valid") @get op getValid(): Siamese; diff --git a/packages/http-specs/specs/type/model/inheritance/single-discriminator/main.tsp b/packages/http-specs/specs/type/model/inheritance/single-discriminator/main.tsp index 3133a06a22a..316bd3464a4 100644 --- a/packages/http-specs/specs/type/model/inheritance/single-discriminator/main.tsp +++ b/packages/http-specs/specs/type/model/inheritance/single-discriminator/main.tsp @@ -65,14 +65,12 @@ model Fish { {"wingspan": 1, "kind": "sparrow"} ``` """) -@surfaceDoc( - Sparrow, - #{ - category: "hierarchy", - expected: "Bird", - doc: "The `Sparrow` model extends `Bird` as its base type on the client surface.", - } -) +@surfaceDoc(#{ + category: "hierarchy", + subject: "Sparrow", + expected: "Bird", + doc: "The `Sparrow` model extends `Bird` as its base type on the client surface.", +}) @get op getModel(): Bird; diff --git a/packages/spector/generated-defs/TypeSpec.Spector.ts b/packages/spector/generated-defs/TypeSpec.Spector.ts index 5e0c3d05224..96f24d0c5b0 100644 --- a/packages/spector/generated-defs/TypeSpec.Spector.ts +++ b/packages/spector/generated-defs/TypeSpec.Spector.ts @@ -54,31 +54,6 @@ export function getSurfaceDoc( target: Namespace | Interface | Operation, ): | { - subject: Type; - check: { - readonly category: - | string - | "naming" - | "access" - | "client-location" - | "hierarchy" - | "flatten" - | "paging" - | "other"; - readonly expected: string | Record; - readonly doc?: string; - }; - } - | undefined { - return getAutoDecoratorValue(program, "TypeSpec.Spector.surfaceDoc", target) as any; -} - -export function setSurfaceDoc( - program: Program, - target: Namespace | Interface | Operation, - value: { - subject: Type; - check: { readonly category: | string | "naming" @@ -89,9 +64,30 @@ export function setSurfaceDoc( | "paging" | "other"; readonly expected: string | Record; + readonly subject?: string; readonly doc?: string; - }; + } + | undefined { + return getAutoDecoratorValue(program, "TypeSpec.Spector.surfaceDoc", target)?.["check"] as any; +} + +export function setSurfaceDoc( + program: Program, + target: Namespace | Interface | Operation, + check: { + readonly category: + | string + | "naming" + | "access" + | "client-location" + | "hierarchy" + | "flatten" + | "paging" + | "other"; + readonly expected: string | Record; + readonly subject?: string; + readonly doc?: string; }, ): void { - setAutoDecorator(program, "TypeSpec.Spector.surfaceDoc", target, value); + setAutoDecorator(program, "TypeSpec.Spector.surfaceDoc", target, { check: check }); } diff --git a/packages/spector/lib/main.tsp b/packages/spector/lib/main.tsp index 49b2f11ee08..be117ab84c6 100644 --- a/packages/spector/lib/main.tsp +++ b/packages/spector/lib/main.tsp @@ -92,17 +92,20 @@ union SurfaceCategory { * Like `@scenarioDoc`, each target gets exactly one `@surfaceDoc`. * * ```tsp - * @surfaceDoc(widgetName, #{ + * @surfaceDoc(#{ * category: "naming", * expected: "widgetName", + * subject: "Widget.defaultName", * }) * ``` * + * When `subject` is omitted it defaults to the decorated target's name. + * * `expected` is a single canonical, language-agnostic string by default, recast * by each emitter. For exact per-language names, pass a `scope → value` dict: * * ```tsp - * @surfaceDoc(ioThing, #{ + * @surfaceDoc(#{ * category: "naming", * expected: #{ python: "io_thing", csharp: "IOThing" }, * }) @@ -110,10 +113,10 @@ union SurfaceCategory { */ auto dec surfaceDoc( target: Namespace | Interface | Operation, - subject: unknown, check: valueof { category: SurfaceCategory, expected: string | Record, + subject?: string, doc?: string, } ); diff --git a/packages/spector/src/coverage/surface-checks-manifest.ts b/packages/spector/src/coverage/surface-checks-manifest.ts index fbd9b1a50af..7978d2ecb22 100644 --- a/packages/spector/src/coverage/surface-checks-manifest.ts +++ b/packages/spector/src/coverage/surface-checks-manifest.ts @@ -2,7 +2,7 @@ import { getSourceLocation, normalizePath } from "@typespec/compiler"; import { relative } from "path"; import pc from "picocolors"; import prettier from "prettier"; -import type { SurfaceDetails, SurfaceDoc, SurfaceSubject } from "../lib/decorators.js"; +import type { SurfaceDetails, SurfaceDoc, SurfaceDocTarget } from "../lib/decorators.js"; import { buildSurfaceDetails } from "../lib/decorators.js"; import { logger } from "../logger.js"; import { findScenarioSpecFiles } from "../scenarios-resolver.js"; @@ -70,19 +70,15 @@ export function createSurfaceChecksManifest( const items: SurfaceCheckItem[] = []; const usedIds = new Map(); for (const surfaceDoc of surfaceDocs) { - const location = getCheckLocation(scenariosPath, surfaceDoc.subject); - const target = getSubjectName(surfaceDoc.subject); + const location = getCheckLocation(scenariosPath, surfaceDoc.target); const details = buildSurfaceDetails(surfaceDoc); const scopeSuffix = surfaceDoc.scope ? `_${surfaceDoc.scope.replace(/[^\w]+/g, "-")}` : ""; - // Every check is identified by its resolved scenario name, so all checks on - // one surface scenario share a stable prefix; `uniqueId` disambiguates when - // a scenario carries several checks of the same category. - const idBase = surfaceDoc.scenario ?? target; + const idBase = surfaceDoc.scenario ?? surfaceDoc.subject; items.push({ id: uniqueId(usedIds, `${idBase}_${surfaceDoc.category}${scopeSuffix}`), scenario: surfaceDoc.scenario, category: surfaceDoc.category, - target, + target: surfaceDoc.subject, ...(surfaceDoc.scope ? { scope: surfaceDoc.scope } : {}), doc: surfaceDoc.doc, location, @@ -149,12 +145,11 @@ function uniqueId(used: Map, base: string): string { return count === 0 ? base : `${base}_${count + 1}`; } -function getSubjectName(subject: SurfaceSubject): string { - return typeof subject.name === "string" ? subject.name : ""; -} - -function getCheckLocation(scenariosPath: string, subject: SurfaceSubject): SurfaceCheckLocation { - const tspLocation = getSourceLocation(subject); +function getCheckLocation( + scenariosPath: string, + target: SurfaceDoc["target"], +): SurfaceCheckLocation { + const tspLocation = getSourceLocation(target); return { path: normalizePath(relative(scenariosPath, tspLocation.file.path)), start: tspLocation.file.getLineAndCharacterOfPosition(tspLocation.pos), diff --git a/packages/spector/src/lib/decorators.ts b/packages/spector/src/lib/decorators.ts index 15fec46a47a..a211cfe509d 100644 --- a/packages/spector/src/lib/decorators.ts +++ b/packages/spector/src/lib/decorators.ts @@ -1,7 +1,6 @@ import { $service, Enum, - EnumMember, getAutoDecoratorTargets, getAutoDecoratorValue, getNamespaceFullName, @@ -9,12 +8,9 @@ import { Interface, listServices, Model, - ModelProperty, Namespace, Operation, Program, - Union, - UnionVariant, } from "@typespec/compiler"; import { $route, @@ -236,22 +232,6 @@ export function getScenarioName( */ export type SurfaceDocTarget = Namespace | Interface | Operation; -/** - * The `subject` of a surface check — the type or member whose generated surface - * is being asserted. May differ from the annotated {@link SurfaceDocTarget} - * (e.g. a check on an operation asserts something about a model it returns). - */ -export type SurfaceSubject = - | Namespace - | Interface - | Operation - | Model - | Enum - | Union - | ModelProperty - | EnumMember - | UnionVariant; - /** * The generic, category-agnostic fields the shared runner substitutes into an * emitter's `verifiers.json` (as `{expected}`, `{kind}`, `{origin}`). They are @@ -261,10 +241,6 @@ export type SurfaceSubject = export interface SurfaceDetails { /** The author's `expected` client-surface output for this check. */ expected?: string; - /** The subject's language-agnostic symbol kind (for casing-aware checks). */ - kind?: string; - /** The subject's declaring container (e.g. the client an operation moved from). */ - origin?: string; } /** A resolved `@surfaceDoc` annotation. */ @@ -280,8 +256,8 @@ export interface SurfaceDoc { scenario: string | undefined; /** The annotated element (a scenario namespace/interface/operation). */ target: SurfaceDocTarget; - /** The type/member the check is about. */ - subject: SurfaceSubject; + /** The subject of the check — defaults to the target's name when omitted. */ + subject: string; /** The kind of surface assertion (routes the check to a verifier). */ category: string; /** The expected client-surface output for this category. */ @@ -297,30 +273,8 @@ export interface SurfaceDoc { } /** A short prose fallback so the AI path always has something to verify against. */ -function synthesizeDoc(category: string, subject: SurfaceSubject, expected: string): string { - const subjectName = typeof subject.name === "string" ? subject.name : "the subject"; - return `${category}: ${subjectName} → ${expected}`; -} - -function getSurfaceParent(target: SurfaceSubject): SurfaceSubject | undefined { - switch (target.kind) { - case "Namespace": - case "Interface": - case "Model": - case "Enum": - case "Union": - return target.namespace; - case "Operation": - return target.interface ?? target.namespace; - case "ModelProperty": - return target.model; - case "EnumMember": - return target.enum; - case "UnionVariant": - return target.union; - default: - return undefined; - } +function synthesizeDoc(category: string, subject: string, expected: string): string { + return `${category}: ${subject} → ${expected}`; } /** @@ -329,7 +283,7 @@ function getSurfaceParent(target: SurfaceSubject): SurfaceSubject | undefined { * convention `@scenario` uses. */ function getEnclosingScenarioName(program: Program, target: SurfaceDocTarget): string | undefined { - let current: SurfaceSubject | undefined = target; + let current: SurfaceDocTarget | Namespace | Interface | undefined = target; while (current) { if ( current.kind === "Namespace" || @@ -341,7 +295,13 @@ function getEnclosingScenarioName(program: Program, target: SurfaceDocTarget): s return name; } } - current = getSurfaceParent(current); + if (current.kind === "Namespace" || current.kind === "Interface") { + current = current.namespace; + } else if (current.kind === "Operation") { + current = current.interface ?? current.namespace; + } else { + break; + } } return undefined; } @@ -350,15 +310,14 @@ function getEnclosingScenarioName(program: Program, target: SurfaceDocTarget): s const SURFACE_DOC_FQN = "TypeSpec.Spector.surfaceDoc"; /** - * What the auto dec stores for `@surfaceDoc`. With params - * `subject: unknown, check: valueof { category, expected, doc? }`, - * the auto dec stores: `{ subject: Type, check: { category, expected, doc } }`. + * What the auto dec stores for `@surfaceDoc`. Single valueof model param `check`: + * `{ check: { category, expected, subject?, doc? } }`. */ interface StoredSurfaceDoc { - subject: SurfaceSubject; check: { category: string; expected: string | Record; + subject?: string; doc?: string; }; } @@ -379,16 +338,17 @@ export function listSurfaceDocs(program: Program): SurfaceDoc[] { if (!stored) continue; const docTarget = target as SurfaceDocTarget; const scenario = getEnclosingScenarioName(program, docTarget); - const { category, expected, doc } = stored.check; + const { category, expected, subject, doc } = stored.check; + const resolvedSubject = subject ?? docTarget.name ?? ""; for (const { expected: exp, scope } of expandExpected(expected)) { result.push({ scenario, target: docTarget, - subject: stored.subject, + subject: resolvedSubject, category, expected: exp, scope, - doc: doc ?? synthesizeDoc(category, stored.subject, exp), + doc: doc ?? synthesizeDoc(category, resolvedSubject, exp), }); } } @@ -446,46 +406,5 @@ export function buildSurfaceDetails(doc: SurfaceDoc): SurfaceDetails { if (doc.expected !== "") { details.expected = doc.expected; } - const kind = getSurfaceKind(doc.subject); - if (kind !== undefined) { - details.kind = kind; - } - const origin = getDeclaringContainerName(doc.subject); - if (origin !== undefined) { - details.origin = origin; - } return details; } - -/** - * Map a surface subject to the language-agnostic symbol kind used by - * casing-aware `naming` checks so each emitter can recast the expected - * identifier into its idiomatic casing. - */ -function getSurfaceKind(target: SurfaceSubject): string | undefined { - switch (target.kind) { - case "Enum": - case "Union": - return "enum"; - case "EnumMember": - case "UnionVariant": - return "enumvalue"; - case "Model": - return "model"; - case "ModelProperty": - return "property"; - case "Operation": - return "operation"; - case "Namespace": - case "Interface": - return "client"; - default: - return undefined; - } -} - -/** The name of the client/group an element is declared in — its surface origin. */ -function getDeclaringContainerName(target: SurfaceSubject): string | undefined { - const parent = getSurfaceParent(target); - return parent && typeof parent.name === "string" && parent.name ? parent.name : undefined; -} diff --git a/packages/spector/src/lib/index.ts b/packages/spector/src/lib/index.ts index efa3408a1a0..f9ff027404e 100644 --- a/packages/spector/src/lib/index.ts +++ b/packages/spector/src/lib/index.ts @@ -7,7 +7,6 @@ export { SurfaceDetails, SurfaceDoc, SurfaceDocTarget, - SurfaceSubject, buildSurfaceDetails, getScenarioDoc, getScenarioEndpoints, diff --git a/packages/spector/test/surface-docs.test.ts b/packages/spector/test/surface-docs.test.ts index 29f522ec12e..f7c0f22e803 100644 --- a/packages/spector/test/surface-docs.test.ts +++ b/packages/spector/test/surface-docs.test.ts @@ -45,11 +45,11 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Return the kind.") - @surfaceDoc(ServerExtensibleEnum, #{ category: "naming", expected: "ClientExtensibleEnum", doc: "Exposed to clients as ClientExtensibleEnum." }) + @surfaceDoc(#{ category: "naming", expected: "ClientExtensibleEnum", doc: "Exposed to clients as ClientExtensibleEnum.", subject: "ServerExtensibleEnum" }) op getKind(): ServerExtensibleEnum; `); expect(doc.category).toBe("naming"); - expect(doc.subject.kind).toBe("Enum"); + expect(doc.subject).toBe("ServerExtensibleEnum"); expect(doc.expected).toBe("ClientExtensibleEnum"); expect(doc.doc).toBe("Exposed to clients as ClientExtensibleEnum."); }); @@ -63,7 +63,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Return the kind.") - @surfaceDoc(ServerExtensibleEnum, #{ category: "naming", expected: "ClientExtensibleEnum" }) + @surfaceDoc(#{ category: "naming", expected: "ClientExtensibleEnum", subject: "ServerExtensibleEnum" }) op getKind(): ServerExtensibleEnum; `); expect(doc.scenario).toBe("getKind"); @@ -77,7 +77,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get a widget.") - @surfaceDoc(Widget, #{ category: "access", expected: "internal" }) + @surfaceDoc(#{ category: "access", expected: "internal", subject: "Widget" }) op get(): Widget; `); expect(doc.doc).toContain("access"); @@ -93,7 +93,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get a widget.") - @surfaceDoc(Widget, #{ category: "brand-new-category", expected: "whatever" }) + @surfaceDoc(#{ category: "brand-new-category", expected: "whatever", subject: "Widget" }) op get(): Widget; `); expect(doc.category).toBe("brand-new-category"); @@ -109,7 +109,7 @@ describe("@surfaceDoc", () => { } @scenario - @surfaceDoc(Widget, #{ category: "access", expected: "internal" }) + @surfaceDoc(#{ category: "access", expected: "internal", subject: "Widget" }) op get(): Widget; `, ); @@ -126,7 +126,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get a widget.") - @surfaceDoc(Widget, #{ category: "access", expected: "internal" }) + @surfaceDoc(#{ category: "access", expected: "internal", subject: "Widget" }) op get(): Widget; `, ); @@ -144,7 +144,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Return the kind.") - @surfaceDoc(ServerExtensibleEnum, #{ category: "naming", expected: "ClientExtensibleEnum", doc: "Exposed as ClientExtensibleEnum." }) + @surfaceDoc(#{ category: "naming", expected: "ClientExtensibleEnum", doc: "Exposed as ClientExtensibleEnum.", subject: "ServerExtensibleEnum" }) op getKind(): ServerExtensibleEnum; `); const item = items["getKind_naming"]; @@ -152,14 +152,14 @@ describe("@surfaceDoc", () => { expect(item.category).toBe("naming"); expect(item.target).toBe("ServerExtensibleEnum"); expect(item.doc).toContain("Exposed as ClientExtensibleEnum"); - expect(item.details).toEqual({ expected: "ClientExtensibleEnum", kind: "enum" }); + expect(item.details).toEqual({ expected: "ClientExtensibleEnum" }); // A location the verifier can point back to for reporting. expect(typeof item.location.path).toBe("string"); expect(item.location.start.line).toBeGreaterThanOrEqual(0); expect(item.location.end.line).toBeGreaterThanOrEqual(item.location.start.line); }); - it("derives {expected, kind, origin} for a subject inside a container", async () => { + it("derives {expected} for a subject inside a container", async () => { const items = await manifestItems(` model Widget { id: string; @@ -167,20 +167,20 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get a widget.") - @surfaceDoc(Widget.id, #{ category: "naming", expected: "identifier", doc: "Renamed to identifier." }) + @surfaceDoc(#{ category: "naming", expected: "identifier", doc: "Renamed to identifier.", subject: "Widget.id" }) op get(): Widget; `); const item = items["get_naming"]; expect(item.category).toBe("naming"); - expect(item.target).toBe("id"); - expect(item.details).toEqual({ expected: "identifier", kind: "property", origin: "Widget" }); + expect(item.target).toBe("Widget.id"); + expect(item.details).toEqual({ expected: "identifier" }); }); it("omits `expected` from details when it is blank", async () => { const items = await manifestItems(` @scenario @scenarioDoc("List items.") - @surfaceDoc(listItems, #{ category: "paging", expected: "" }) + @surfaceDoc(#{ category: "paging", expected: "", subject: "listItems" }) op listItems(): string[]; `); const item = items["listItems_paging"]; @@ -196,7 +196,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get a widget.") - @surfaceDoc(Widget, #{ category: "other", expected: "strongly typed", doc: "The response body is a strongly typed model." }) + @surfaceDoc(#{ category: "other", expected: "strongly typed", doc: "The response body is a strongly typed model.", subject: "Widget" }) op get(): Widget; `); const item = items["get_other"]; @@ -215,7 +215,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get a widget.") - @surfaceDoc(Widget, #{ category: "naming", expected: "WidgetInternal", doc: "Hidden | renamed to WidgetInternal." }) + @surfaceDoc(#{ category: "naming", expected: "WidgetInternal", doc: "Hidden | renamed to WidgetInternal.", subject: "Widget" }) op get(): Widget; `, ); @@ -232,7 +232,7 @@ describe("@surfaceDoc", () => { expect(header).toContain(col); } // details encoded as key=value. - expect(md).toContain("expected=WidgetInternal; kind=model"); + expect(md).toContain("expected=WidgetInternal"); // Pipes inside prose are escaped so they don't break the table. expect(md).toContain("Hidden \\| renamed to WidgetInternal."); }); @@ -248,7 +248,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get the thing.") - @surfaceDoc(IOThing, #{ category: "naming", expected: #{ python: "io_thing", csharp: "IOThing" } }) + @surfaceDoc(#{ category: "naming", expected: #{ python: "io_thing", csharp: "IOThing" }, subject: "IOThing" }) op get(): IOThing; `, ); @@ -268,7 +268,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get a widget.") - @surfaceDoc(Widget, #{ category: "naming", expected: "Widget" }) + @surfaceDoc(#{ category: "naming", expected: "Widget", subject: "Widget" }) op get(): Widget; `); expect(doc.scope).toBeUndefined(); @@ -283,7 +283,7 @@ describe("@surfaceDoc", () => { @scenario @scenarioDoc("Get the thing.") - @surfaceDoc(IOThing, #{ category: "naming", expected: #{ python: "io_thing" } }) + @surfaceDoc(#{ category: "naming", expected: #{ python: "io_thing" }, subject: "IOThing" }) op get(): IOThing; `, ); @@ -291,7 +291,7 @@ describe("@surfaceDoc", () => { const item = manifest.items.find((i) => i.id === "get_naming_python"); expect(item).toBeDefined(); expect(item!.scope).toBe("python"); - expect(item!.details).toEqual({ expected: "io_thing", kind: "model" }); + expect(item!.details).toEqual({ expected: "io_thing" }); const md = await createSurfaceChecksSummary(manifest); expect(md).toContain("io_thing"); expect(md).toContain("python"); From 4da00e57e4fbd77004f10c823b3d6f4ce38c86dc Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 30 Jul 2026 14:43:36 -0400 Subject: [PATCH 6/9] chore: align verify engine, tests, verifiers, and docs with all-keyword surfaceDoc - Remove `casing` from Rule interface and `byKind` modifier from resolveVar - Simplify verifiers.json: naming uses {expected} directly, remove client-location - Update verify-surface-checks tests to use new details shape (no kind/name) - Update README.md with new decorator syntax and placeholder table - Update context.md to remove kind-based casing references - Regenerate surface-checks.md from current annotations (6 checks) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 54fb668c-64c3-41e5-8933-c4300c884974 --- .../eng/scripts/client-criteria/README.md | 69 ++++++++++-------- .../eng/scripts/client-criteria/context.md | 22 +++--- .../scripts/client-criteria/verifiers.json | 27 +------ packages/http-specs/surface-checks.md | 21 +++--- .../src/actions/verify-surface-checks.ts | 35 ++++----- .../test/verify-surface-checks.test.ts | 72 +++++-------------- 6 files changed, 89 insertions(+), 157 deletions(-) diff --git a/packages/http-client-python/eng/scripts/client-criteria/README.md b/packages/http-client-python/eng/scripts/client-criteria/README.md index dad5454d0b2..a5bb58b37c9 100644 --- a/packages/http-client-python/eng/scripts/client-criteria/README.md +++ b/packages/http-client-python/eng/scripts/client-criteria/README.md @@ -14,13 +14,25 @@ renamed, a method relocated, a model flattened. Today those are checked by hand, per language. **Client surface checks** close that gap. On a spec element that already carries -`@scenarioDoc`, you add `@surfaceDoc(category, subject, expected, doc?)` — an -explicit, language-agnostic assertion: the `category` of surface property, the -`subject` type/member it's about, and the `expected` client-facing output. -`expected` is normally a single canonical string that each emitter **recasts** -to its idiomatic form; when a name can't be derived by casing, it can instead be -a `scope → value` dict (e.g. `#{ python: "io_thing", csharp: "IOThing" }`) whose +`@scenarioDoc`, you add a `@surfaceDoc` — an explicit, language-agnostic +assertion about the generated client surface: + +```typespec +@surfaceDoc(#{ + category: "naming", + expected: "defaultName", + subject: "Widget", // optional — defaults to target's name + doc: "Explanation...", // optional +}) +``` + +The `category` names what kind of surface property is asserted, `expected` is the +expected client-facing output, and `subject` identifies what's being checked +(defaults to the decorated element's name when omitted). `expected` is normally a +single canonical string; when a name can't be derived by casing, it can be a +`scope → value` dict (e.g. `#{ python: "io_thing", csharp: "IOThing" }`) whose values are matched **verbatim** for their language. + Nothing is inferred from other decorators, so a check is deterministic to read and grounded in a real scenario. A precompute step collects every `@surfaceDoc` into a shared **checks doc**. Each emitter supplies a **context.md** (what those @@ -59,25 +71,25 @@ and `expected` say about-what and to-what. `SurfaceCategory` is an **extensible* union — the named members are the canonical, tight set, but any string is accepted. -| category | `subject` is… | `expected` is… | -| ---------------- | ---------------------------- | --------------------------------------------------------------------------------------------- | -| `naming` | renamed type/member/op/param | canonical client identifier (recast per language, or a `scope → value` dict matched verbatim) | -| `access` | type/operation | `"internal"` or `"public"` | -| `clientLocation` | operation | client/group name it should be surfaced on | -| `hierarchy` | derived model | base type's client name | -| `flatten` | the flattened property | promoted property name(s); blank = just inlined | -| `paging` | operation | item/element type the pager yields | -| `other` | anything | free-form; verified against the prose (AI) | - -The precompute step derives one fixed, category-agnostic set of placeholders for -**every** category — so the core never needs to know what a category means: - -| placeholder | derived from | -| ------------ | ----------------------------------- | -| `{target}` | the `subject`'s name | -| `{expected}` | the `expected` argument | -| `{kind}` | the `subject`'s symbol kind | -| `{origin}` | the `subject`'s declaring container | +| category | `subject` is… | `expected` is… | +| ---------------- | ---------------------------- | ------------------------------------------------------- | +| `naming` | renamed type/member/op/param | expected client identifier (verbatim or with `:casing`) | +| `access` | type/operation | `"internal"` or `"public"` | +| `clientLocation` | operation | client/group name it should be surfaced on | +| `hierarchy` | derived model | base type's client name | +| `flatten` | the flattened property | promoted property name(s); blank = just inlined | +| `paging` | operation | item/element type the pager yields | +| `other` | anything | free-form; verified against the prose (AI) | + +The precompute step derives a category-agnostic set of placeholders for the +verifier engine: + +| placeholder | derived from | +| ------------ | --------------------------------------- | +| `{target}` | the `subject` string (or target's name) | +| `{expected}` | the `expected` value | + +Any other key in `details` is also accessible as `{key}` (e.g. `{internal}`). ### Adding a category @@ -87,7 +99,7 @@ core spector: 1. Optionally add a member to the `SurfaceCategory` union in spector for autocomplete/discoverability (a one-line, backward-compatible change). 2. In each emitter that wants it deterministic, add a `verifiers.json` entry - (using `{target}`/`{expected}`/`{kind}`/`{origin}`) and a `context.md` blurb. + (using `{target}`/`{expected}` + casing modifiers) and a `context.md` blurb. An emitter that adds **nothing** for a category still works: it falls back to AI-verifying the prose — exactly how `other` behaves. So promoting a check from @@ -142,8 +154,3 @@ never reaches AI; every check in the category resolves to `not-applicable`: - **How items mark N/A for a language:** per **category** in the emitter's `verifiers.json` (`{ "na": true }`), mirroring wire `unsupportedScenarios`; the runner reports `not-applicable` (see above). - -## Open questions - -- Should the manifest record the **resolved per-language symbol name** (so - renamed-and-access-controlled types stay mechanically checkable)? diff --git a/packages/http-client-python/eng/scripts/client-criteria/context.md b/packages/http-client-python/eng/scripts/client-criteria/context.md index 0022df8e582..4d8a5a96141 100644 --- a/packages/http-client-python/eng/scripts/client-criteria/context.md +++ b/packages/http-client-python/eng/scripts/client-criteria/context.md @@ -8,6 +8,7 @@ _The shared `/check-surface` prompt and `tsp-spector verify-surface-checks` read - **generated-root:** `packages/http-client-python/tests/generated/` - **flavors:** `azure`, `unbranded` - **checks-doc:** `packages/http-specs/surface-checks.md` _(generated from `@surfaceDoc`, ships with the specs)_ + ### What differs by flavor (branded = azure, unbranded) @@ -37,20 +38,17 @@ regex patterns + casing) consumed by `tsp-spector verify-surface-checks`. ### naming -- the generated identifier equals the `expected` client name **recast to - Python's idiomatic casing for the subject's kind** (case-sensitive): - `enum`/`model`/`type` → `PascalCase`, `property`/`parameter`/`operation` → - `snake_case`, enum value → `UPPER_SNAKE`. So an enum whose `expected` is - `ClientExtensibleEnum` must appear exactly as `ClientExtensibleEnum` (not - `client_extensible_enum`). The per-kind casing map lives in `verifiers.json` - (`{expected:byKind}`); the manifest carries the subject's `kind` under - `details`. +- the generated identifier equals the `expected` value from `@surfaceDoc` + (case-sensitive match). The spec author provides the expected client name + already in the correct casing. For Python: `enum`/`model`/`type` → + `PascalCase`, `property`/`parameter`/`operation` → `snake_case`, enum value → + `UPPER_SNAKE`. - **exact per-language names:** when a name can't be derived by casing (a reserved-word escape, an acronym, a hard rename), the spec author supplies a `scope → value` dict on `@surfaceDoc` (e.g. `#{ python: "io_thing" }`). Those checks carry a `scope` column; the engine only runs the ones scoped to - `python` and matches `{expected}` **verbatim** (the `:byKind` recast is - skipped). The same `naming` routine handles both — no `verifiers.json` change. + `python` and matches `{expected}` **verbatim**. The same `naming` routine + handles both — no `verifiers.json` change. ### flatten @@ -62,8 +60,8 @@ regex patterns + casing) consumed by `tsp-spector verify-surface-checks`. - an operation group is a class under `operations/_operations.py`; a method is a public `def` on it; the **root client** is the main `*Client` class. An operation is "on" a client/group when its snake_case name is a method of the - matching class. Deterministic: assert the method is present on the `expected` - client/group and absent from its `origin` (the one it moved from). + matching class. The `expected` value names the client/group where the method + should appear. ### hierarchy diff --git a/packages/http-client-python/eng/scripts/client-criteria/verifiers.json b/packages/http-client-python/eng/scripts/client-criteria/verifiers.json index 7702f1d5b05..a3214d9a3ca 100644 --- a/packages/http-client-python/eng/scripts/client-criteria/verifiers.json +++ b/packages/http-client-python/eng/scripts/client-criteria/verifiers.json @@ -1,20 +1,8 @@ { - "_comment": "Declarative routing table for the shared `tsp-spector verify-surface-checks` engine. AUTHORED once per emitter (by an agent or a human) — NOT generated by the emitter build, and NOT language-specific code: the engine is language-agnostic, all the Python specifics live here as file globs + regex templates + a casing map. A category listed here is decided without AI; one not listed is verified by AI against the prose. The engine substitutes a fixed, category-agnostic set of placeholders into `find`/`scope`: {target}=the subject symbol's name, {expected}=the author's expected client-surface output, {kind}=the subject's symbol kind, {origin}=the subject's declaring container. Add `:casing` to recase, e.g. {expected:byKind} (per-kind map below), {target:snake}. `expect`: present (default), absent, {absentWhen:}, or {whenExpected:{:present|absent}} (branch on {expected}). SCOPE: when a check is scoped to a language (authored as a `scope → value` dict on `@surfaceDoc`), the engine (1) only runs it for the matching language and (2) matches {expected} VERBATIM — casing modifiers like `:byKind` are ignored — so a per-language exact name (a reserved-word escape, an acronym, a hard rename) needs NO verifier change here; the same `naming` routine below handles both idiomatic and exact.", - "_promotable": "A category with a routine here is deterministic; leaving it OUT makes it AI-verified (against the prose) until added. Adding an entry (a pattern like `hierarchy` below) flips its checks from AI to deterministic with no change to the spec — and no other emitter has to do anything. An agent can author these entries from context.md.", - "_test_verifier_example": "Instead of a pattern, a category may defer to an existing test that already asserts this surface: { \"exactName\": { \"test\": \"emitter-tests/src/test/java/.../ExactNameTests.java#L34-L35\", \"note\": \"reflection asserts the exact generated name\" } }. Reported status 'deferred', how 'test' (no pattern, no AI).", - "_not_applicable_example": "If a surface concept has no equivalent in this language, mark the whole category N/A (mirrors wire `unsupportedScenarios`): { \"hierarchy\": { \"na\": true, \"note\": \"no class inheritance in this language\" } }. Every check in the category is reported status 'not-applicable', how 'not-applicable' (no pattern, no AI).", + "_comment": "Declarative routing table for the shared `tsp-spector verify-surface-checks` engine. AUTHORED once per emitter — NOT generated. The engine substitutes placeholders into `find`/`scope`: {target}=the subject name (from @surfaceDoc), {expected}=the expected client-surface output. Add `:casing` to recase, e.g. {expected:snake}, {target:pascal}. `expect`: present (default), absent, {absentWhen:}, or {whenExpected:{:present|absent}}. SCOPE: when a check is scoped to a language, the engine only runs it for the matching language and matches {expected} VERBATIM (casing modifiers are ignored).", "naming": { "files": ["**/*.py"], - "find": "\\b{expected:byKind}\\b", - "casing": { - "type": "pascal", - "model": "pascal", - "enum": "pascal", - "property": "snake", - "parameter": "snake", - "operation": "snake", - "enumvalue": "upper_snake" - } + "find": "\\b{expected}\\b" }, "access": { "files": ["**/models/__init__.py"], @@ -24,16 +12,5 @@ "hierarchy": { "files": ["**/models/_models.py"], "find": "class\\s+{target}\\s*\\([^)]*\\b{expected}\\b" - }, - "client-location": { - "files": ["**/operations/_operations.py"], - "scope": "class\\s+\\w*{expected}\\w*[\\s\\S]*?(?=\\nclass |$)", - "find": "def\\s+{target:snake}\\b", - "also": { - "files": ["**/operations/_operations.py"], - "scope": "class\\s+\\w*{origin}\\w*[\\s\\S]*?(?=\\nclass |$)", - "find": "def\\s+{target:snake}\\b", - "expect": "absent" - } } } diff --git a/packages/http-specs/surface-checks.md b/packages/http-specs/surface-checks.md index f64af13c792..fda9b4175c0 100644 --- a/packages/http-specs/surface-checks.md +++ b/packages/http-specs/surface-checks.md @@ -3,16 +3,11 @@ Generated from `@surfaceDoc` annotations. This table is both the human summary and the machine-readable checks doc parsed by the shared `verify-surface-checks` runner. -| id | scenario | category | target | scope | details | doc | -| ----------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | --------- | ------------------------ | ----- | ---------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | -| Payload_Pageable_PageSize_listWithoutContinuation_paging | Payload_Pageable_PageSize_listWithoutContinuation | paging | listWithoutContinuation | | expected=Pet; kind=operation; origin=PageSize | The client surfaces this single-page operation as a pageable collection of `Pet`. | -| Payload_Pageable_PageSize_listWithPageSize_paging | Payload_Pageable_PageSize_listWithPageSize | paging | listWithPageSize | | expected=Pet; kind=operation; origin=PageSize | The client surfaces this operation as a pageable collection of `Pet` with a page-size parameter. | -| Payload_Pageable_ServerDrivenPagination_ContinuationToken_requestQueryResponseBody_paging | Payload_Pageable_ServerDrivenPagination_ContinuationToken_requestQueryResponseBody | paging | requestQueryResponseBody | | expected=Pet; kind=operation; origin=ContinuationToken | The client surfaces this operation as a pageable collection of `Pet`, transparently threading the continuation token across pages. | -| Payload_Pageable_ServerDrivenPagination_link_paging | Payload_Pageable_ServerDrivenPagination_link | paging | link | | expected=Pet; kind=operation; origin=ServerDrivenPagination | The client surfaces this operation as a pageable collection that yields `Pet` items across pages. | -| Payload_Pageable_ServerDrivenPagination_linkString_paging | Payload_Pageable_ServerDrivenPagination_linkString | paging | linkString | | expected=Pet; kind=operation; origin=ServerDrivenPagination | The client surfaces this operation as a pageable collection that yields `Pet` items, following a string next-link. | -| Payload_Pageable_ServerDrivenPagination_nestedLink_paging | Payload_Pageable_ServerDrivenPagination_nestedLink | paging | nestedLink | | expected=Pet; kind=operation; origin=ServerDrivenPagination | The client surfaces this operation as a pageable collection of `Pet`, flattening the nested items/next-link envelope. | -| Payload_Pageable_XmlPagination_listWithNextLink_paging | Payload_Pageable_XmlPagination_listWithNextLink | paging | listWithNextLink | | expected=XmlPet; kind=operation; origin=XmlPagination | The client surfaces this XML operation as a pageable collection that yields `XmlPet` items across pages. | -| Serialization_EncodedName_Json_Property_send_naming | Serialization_EncodedName_Json_Property_send | naming | defaultName | | expected=defaultName; kind=property; origin=JsonEncodedNameModel | The client property is named `defaultName` even though the wire name is `wireName`. | -| SpecialWords_ExtensibleStrings_putExtensibleStringValue_naming | SpecialWords_ExtensibleStrings_putExtensibleStringValue | naming | and | | expected=and; kind=enumvalue; origin=ExtensibleString | The reserved-word enum member surfaces on the client with an idiomatic, non-conflicting identifier. | -| Type_Model_Inheritance_NotDiscriminated_getValid_hierarchy | Type_Model_Inheritance_NotDiscriminated_getValid | hierarchy | Siamese | | expected=Cat; kind=model; origin=NotDiscriminated | The `Siamese` model extends `Cat` as its base type on the client surface. | -| Type_Model_Inheritance_SingleDiscriminator_getModel_hierarchy | Type_Model_Inheritance_SingleDiscriminator_getModel | hierarchy | Sparrow | | expected=Bird; kind=model; origin=SingleDiscriminator | The `Sparrow` model extends `Bird` as its base type on the client surface. | +| id | scenario | category | target | scope | details | doc | +| ------------------------------------------------------------- | --------------------------------------------------- | --------- | -------------------------------- | ----- | -------------------- | ----------------------------------------------------------------------------------- | +| Payload_Pageable_PageSize_listWithoutContinuation_paging | Payload_Pageable_PageSize_listWithoutContinuation | paging | listWithoutContinuation | | expected=Pet | Single-page pageable operation yielding `Pet` elements, no continuation. | +| Payload_Pageable_PageSize_listWithPageSize_paging | Payload_Pageable_PageSize_listWithPageSize | paging | listWithPageSize | | expected=Pet | Page-size-driven pageable operation yielding `Pet` elements. | +| Payload_Pageable_ServerDrivenPagination_link_paging | Payload_Pageable_ServerDrivenPagination_link | paging | link | | expected=Pet | The paginated operation yields `Pet` elements. | +| Serialization_EncodedName_Json_Property_send_naming | Serialization_EncodedName_Json_Property_send | naming | JsonEncodedNameModel.defaultName | | expected=defaultName | The client property is named `defaultName` even though the wire name is `wireName`. | +| Type_Model_Inheritance_NotDiscriminated_getValid_hierarchy | Type_Model_Inheritance_NotDiscriminated_getValid | hierarchy | Siamese | | expected=Cat | The `Siamese` model extends `Cat` as its base type on the client surface. | +| Type_Model_Inheritance_SingleDiscriminator_getModel_hierarchy | Type_Model_Inheritance_SingleDiscriminator_getModel | hierarchy | Sparrow | | expected=Bird | The `Sparrow` model extends `Bird` as its base type on the client surface. | diff --git a/packages/spector/src/actions/verify-surface-checks.ts b/packages/spector/src/actions/verify-surface-checks.ts index 9c1133e8551..311c435a9f6 100644 --- a/packages/spector/src/actions/verify-surface-checks.ts +++ b/packages/spector/src/actions/verify-surface-checks.ts @@ -33,8 +33,6 @@ export interface CheckItem { scope?: string; details: Record; doc: string; - /** Optional per-language resolved names, e.g. `{ "python": "…" }`. */ - client_names?: Record; } /** Parse the checks doc — a Markdown table (the shipped format) or JSON. */ @@ -138,8 +136,6 @@ export interface Rule { find: string; /** `present` (default), `absent`, `{ absentWhen: }`, or `{ whenExpected: { : present|absent } }`. */ expect?: Expect; - /** Per-kind casing map for the `{name:byKind}` modifier. */ - casing?: Record; /** A second rule AND-ed with this one (e.g. present here AND absent there). */ also?: Rule; } @@ -210,21 +206,23 @@ function escapeRegExp(s: string): string { /** * Resolve a `{var[:casing]}` placeholder to a concrete value. - * `target` → item.target; `name` → resolved client name; anything else → a - * `details` field. `:byKind` looks up the casing map at `details.kind`. + * `target` → item.target (the subject string); `expected` → details.expected; + * anything else → a `details` field. `:casing` applies explicit casing (snake, + * pascal, etc.). A scoped check carries a verbatim per-language value — never + * recase it. */ function resolveVar( name: string, mod: string | undefined, item: CheckItem, - language: string, - casingMap: Record | undefined, + _language: string, ): string | undefined { let val: string | undefined; if (name === "target") { val = item.target; - } else if (name === "name") { - val = item.client_names?.[language] ?? (item.details?.name as string | undefined); + } else if (name === "expected") { + const d = item.details?.["expected"]; + val = d == null ? undefined : String(d); } else { const d = item.details?.[name]; val = d == null ? undefined : String(d); @@ -232,23 +230,16 @@ function resolveVar( if (val == null) return undefined; // A scoped check carries a verbatim per-language value — never recase it. if (mod && !item.scope) { - const casing = mod === "byKind" ? casingMap?.[String(item.details?.kind)] : mod; - if (!casing) return undefined; // no casing rule for this kind → N/A - val = applyCasing(val, casing); + val = applyCasing(val, mod); } return val; } /** Compile a regex template; returns null if any placeholder is unresolvable (N/A). */ -function buildRegex( - template: string, - item: CheckItem, - language: string, - casingMap: Record | undefined, -): RegExp | null { +function buildRegex(template: string, item: CheckItem, language: string): RegExp | null { let missing = false; const source = template.replace(/\{(\w+)(?::(\w+))?\}/g, (_m, v: string, mod?: string) => { - const resolved = resolveVar(v, mod, item, language, casingMap); + const resolved = resolveVar(v, mod, item, language); if (resolved == null) { missing = true; return ""; @@ -296,13 +287,13 @@ async function runRule( pkgDir: string, language: string, ): Promise { - const findRe = buildRegex(rule.find, item, language, rule.casing); + const findRe = buildRegex(rule.find, item, language); if (findRe === null) return null; const text = await readGlobbed(pkgDir, rule.files); let haystacks: string[] = [text]; if (rule.scope) { - const scopeRe = buildRegex(rule.scope, item, language, rule.casing); + const scopeRe = buildRegex(rule.scope, item, language); if (scopeRe === null) return null; haystacks = [...text.matchAll(new RegExp(scopeRe.source, "g"))].map((m) => m[0]); } diff --git a/packages/spector/test/verify-surface-checks.test.ts b/packages/spector/test/verify-surface-checks.test.ts index 9affc4997d3..64e69978dc6 100644 --- a/packages/spector/test/verify-surface-checks.test.ts +++ b/packages/spector/test/verify-surface-checks.test.ts @@ -21,23 +21,11 @@ const verifiers = { }, naming: { files: ["**/*.py"], - find: "\\b{name:byKind}\\b", - casing: { enum: "pascal", property: "snake" }, + find: "\\b{expected}\\b", }, hierarchy: { files: ["**/models/_models.py"], - find: "class\\s+{target}\\s*\\([^)]*\\b{base}\\b[^)]*\\)", - }, - "client-location": { - files: ["**/operations/_operations.py"], - scope: "class\\s+\\w*{client}\\w*[\\s\\S]*?(?=\\nclass |$)", - find: "def\\s+{target:snake}\\b", - also: { - files: ["**/operations/_operations.py"], - scope: "class\\s+\\w*{absentFrom}\\w*[\\s\\S]*?(?=\\nclass |$)", - find: "def\\s+{target:snake}\\b", - expect: "absent", - }, + find: "class\\s+{target}\\s*\\([^)]*\\b{expected}\\b[^)]*\\)", }, }; @@ -83,14 +71,14 @@ async function run(items: object[], scenario?: string): Promise { - it("naming: idiomatic casing via {name:byKind} passes on the recast identifier", async () => { + it("naming: idiomatic casing via {expected} passes on the expected identifier", async () => { const out = await run([ { id: "n1", scenario: "surfacedemo", category: "naming", target: "ServerExtensibleEnum", - details: { name: "ClientExtensibleEnum", kind: "enum" }, + details: { expected: "ClientExtensibleEnum" }, doc: "", }, ]); @@ -104,7 +92,7 @@ describe("verify-surface-checks (shared engine)", () => { scenario: "surfacedemo", category: "naming", target: "X", - details: { name: "TotallyMadeUp", kind: "enum" }, + details: { expected: "TotallyMadeUp" }, doc: "", }, ]); @@ -143,7 +131,7 @@ describe("verify-surface-checks (shared engine)", () => { scenario: "surfacedemo", category: "hierarchy", target: "Dog", - details: { base: "Pet" }, + details: { expected: "Pet" }, doc: "", }, ]); @@ -154,36 +142,26 @@ describe("verify-surface-checks (shared engine)", () => { scenario: "surfacedemo", category: "hierarchy", target: "Dog", - details: { base: "Animal" }, + details: { expected: "Animal" }, doc: "", }, ]); expect(bad.results[0].status).toBe("fail"); }); - it("client-location: present on expected client AND absent from the other", async () => { - const good = await run([ + it("client-location: no verifier for this category → needs_ai", async () => { + const out = await run([ { id: "c1", scenario: "surfacedemo", category: "client-location", target: "getWidget", - details: { client: "Widgets", absentFrom: "Gadgets" }, - doc: "", - }, - ]); - expect(good.results[0].status).toBe("pass"); - const bad = await run([ - { - id: "c2", - scenario: "surfacedemo", - category: "client-location", - target: "getWidget", - details: { client: "Gadgets", absentFrom: "Widgets" }, - doc: "", + details: { expected: "Widgets" }, + doc: "operation should live on the Widgets client", }, ]); - expect(bad.results[0].status).toBe("fail"); // getWidget is on Widgets, not Gadgets + expect(out.results).toHaveLength(0); + expect(out.needs_ai[0]).toMatchObject({ id: "c1", category: "client-location" }); }); it("scope: a check scoped to another language is skipped entirely", async () => { @@ -194,7 +172,7 @@ describe("verify-surface-checks (shared engine)", () => { category: "naming", target: "IOThing", scope: "csharp", - details: { name: "IOThing", kind: "enum" }, + details: { expected: "IOThing" }, doc: "", }, ]); @@ -202,7 +180,7 @@ describe("verify-surface-checks (shared engine)", () => { expect(out.needs_ai).toHaveLength(0); }); - it("scope: a scoped value is matched verbatim, bypassing idiomatic casing", async () => { + it("scope: a scoped value is matched verbatim", async () => { const out = await run([ { id: "s2", @@ -210,27 +188,13 @@ describe("verify-surface-checks (shared engine)", () => { category: "naming", target: "IOThing", scope: "python", - details: { name: "IOThing", kind: "enum" }, + details: { expected: "IOThing" }, doc: "", }, ]); expect(out.results[0]).toMatchObject({ status: "pass", how: "deterministic" }); }); - it("scope: an unscoped value is still recast (pascal mangles IOThing → no match)", async () => { - const out = await run([ - { - id: "s3", - scenario: "surfacedemo", - category: "naming", - target: "IOThing", - details: { name: "IOThing", kind: "enum" }, - doc: "", - }, - ]); - expect(out.results[0].status).toBe("fail"); - }); - it("no verifier for a category → needs_ai", async () => { const out = await run([ { @@ -264,7 +228,7 @@ describe("verify-surface-checks (shared engine)", () => { scenario: "surfacedemo", category: "hierarchy", target: "Dog", - details: { base: "Pet" }, + details: { expected: "Pet" }, doc: "", }, ], @@ -299,7 +263,7 @@ describe("verify-surface-checks (shared engine)", () => { scenario: "surfacedemo", category: "hierarchy", target: "Dog", - details: { base: "Pet" }, + details: { expected: "Pet" }, doc: "", }, ], From 6c4ec2e3894e3a151063d23f9e2021044ee295e2 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 30 Jul 2026 15:28:52 -0400 Subject: [PATCH 7/9] feat: add kind field to @surfaceDoc and re-add byKind casing modifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add optional `kind` field to the @surfaceDoc decorator (e.g. 'property', 'model', 'enum') so each language verifier can apply per-kind conventions without encoding language-specific info in the spec. - Add `kind?: string` to surfaceDoc check model in main.tsp - Thread kind through StoredSurfaceDoc, SurfaceDoc, SurfaceDetails - Re-add `casing` map to Rule interface and `byKind` modifier to resolveVar - Update Python verifiers.json with per-kind casing map - Add kind='property' to encoded-name naming spec annotation - Add byKind recasing test (property → snake_case) - Update README.md and context.md - Regenerate surface-checks.md (now shows kind in details) E2E verified: spec says expected='defaultName' + kind='property', Python verifier recases to 'default_name' via byKind, finds it in generated code. Wrong names correctly fail. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 54fb668c-64c3-41e5-8933-c4300c884974 --- .../eng/scripts/client-criteria/README.md | 4 +++ .../eng/scripts/client-criteria/context.md | 9 +++--- .../scripts/client-criteria/verifiers.json | 13 ++++++-- .../serialization/encoded-name/json/main.tsp | 1 + packages/http-specs/surface-checks.md | 16 +++++----- .../generated-defs/TypeSpec.Spector.ts | 2 ++ packages/spector/lib/main.tsp | 4 +++ .../src/actions/verify-surface-checks.ts | 28 ++++++++++++++--- packages/spector/src/lib/decorators.ts | 11 ++++++- .../test/verify-surface-checks.test.ts | 31 ++++++++++++++----- 10 files changed, 92 insertions(+), 27 deletions(-) diff --git a/packages/http-client-python/eng/scripts/client-criteria/README.md b/packages/http-client-python/eng/scripts/client-criteria/README.md index a5bb58b37c9..ee65efdc875 100644 --- a/packages/http-client-python/eng/scripts/client-criteria/README.md +++ b/packages/http-client-python/eng/scripts/client-criteria/README.md @@ -88,8 +88,12 @@ verifier engine: | ------------ | --------------------------------------- | | `{target}` | the `subject` string (or target's name) | | `{expected}` | the `expected` value | +| `{kind}` | the symbol kind (e.g. `property`) | Any other key in `details` is also accessible as `{key}` (e.g. `{internal}`). +Use the `:byKind` modifier (e.g. `{expected:byKind}`) with a `casing` map in +the verifier to apply per-kind conventions (Python: property → snake_case, +model → PascalCase, etc.). ### Adding a category diff --git a/packages/http-client-python/eng/scripts/client-criteria/context.md b/packages/http-client-python/eng/scripts/client-criteria/context.md index 4d8a5a96141..d3e9c58b1d3 100644 --- a/packages/http-client-python/eng/scripts/client-criteria/context.md +++ b/packages/http-client-python/eng/scripts/client-criteria/context.md @@ -39,10 +39,11 @@ regex patterns + casing) consumed by `tsp-spector verify-surface-checks`. ### naming - the generated identifier equals the `expected` value from `@surfaceDoc` - (case-sensitive match). The spec author provides the expected client name - already in the correct casing. For Python: `enum`/`model`/`type` → - `PascalCase`, `property`/`parameter`/`operation` → `snake_case`, enum value → - `UPPER_SNAKE`. + **recast to Python's idiomatic casing for the subject's `kind`** + (case-sensitive): `enum`/`model`/`type` → `PascalCase`, + `property`/`parameter`/`operation` → `snake_case`, enum value → + `UPPER_SNAKE`. The per-kind casing map lives in `verifiers.json` + (`{expected:byKind}`); the `@surfaceDoc` carries the subject's `kind`. - **exact per-language names:** when a name can't be derived by casing (a reserved-word escape, an acronym, a hard rename), the spec author supplies a `scope → value` dict on `@surfaceDoc` (e.g. `#{ python: "io_thing" }`). Those diff --git a/packages/http-client-python/eng/scripts/client-criteria/verifiers.json b/packages/http-client-python/eng/scripts/client-criteria/verifiers.json index a3214d9a3ca..ed474786304 100644 --- a/packages/http-client-python/eng/scripts/client-criteria/verifiers.json +++ b/packages/http-client-python/eng/scripts/client-criteria/verifiers.json @@ -1,8 +1,17 @@ { - "_comment": "Declarative routing table for the shared `tsp-spector verify-surface-checks` engine. AUTHORED once per emitter — NOT generated. The engine substitutes placeholders into `find`/`scope`: {target}=the subject name (from @surfaceDoc), {expected}=the expected client-surface output. Add `:casing` to recase, e.g. {expected:snake}, {target:pascal}. `expect`: present (default), absent, {absentWhen:}, or {whenExpected:{:present|absent}}. SCOPE: when a check is scoped to a language, the engine only runs it for the matching language and matches {expected} VERBATIM (casing modifiers are ignored).", + "_comment": "Declarative routing table for the shared `tsp-spector verify-surface-checks` engine. AUTHORED once per emitter — NOT generated. The engine substitutes placeholders into `find`/`scope`: {target}=the subject name (from @surfaceDoc), {expected}=the expected client-surface output. Add `:casing` to recase, e.g. {expected:snake}, {target:pascal}. Use `:byKind` to apply per-kind casing from the `casing` map (keyed by `details.kind`). `expect`: present (default), absent, {absentWhen:}, or {whenExpected:{:present|absent}}. SCOPE: when a check is scoped to a language, the engine only runs it for the matching language and matches {expected} VERBATIM (casing modifiers are ignored).", "naming": { "files": ["**/*.py"], - "find": "\\b{expected}\\b" + "find": "\\b{expected:byKind}\\b", + "casing": { + "type": "pascal", + "model": "pascal", + "enum": "pascal", + "property": "snake", + "parameter": "snake", + "operation": "snake", + "enumvalue": "upper_snake" + } }, "access": { "files": ["**/models/__init__.py"], diff --git a/packages/http-specs/specs/serialization/encoded-name/json/main.tsp b/packages/http-specs/specs/serialization/encoded-name/json/main.tsp index 0a94c5edd24..9c880129dd4 100644 --- a/packages/http-specs/specs/serialization/encoded-name/json/main.tsp +++ b/packages/http-specs/specs/serialization/encoded-name/json/main.tsp @@ -30,6 +30,7 @@ namespace Property { category: "naming", subject: "JsonEncodedNameModel.defaultName", expected: "defaultName", + kind: "property", doc: "The client property is named `defaultName` even though the wire name is `wireName`.", }) @post diff --git a/packages/http-specs/surface-checks.md b/packages/http-specs/surface-checks.md index fda9b4175c0..8d59e766064 100644 --- a/packages/http-specs/surface-checks.md +++ b/packages/http-specs/surface-checks.md @@ -3,11 +3,11 @@ Generated from `@surfaceDoc` annotations. This table is both the human summary and the machine-readable checks doc parsed by the shared `verify-surface-checks` runner. -| id | scenario | category | target | scope | details | doc | -| ------------------------------------------------------------- | --------------------------------------------------- | --------- | -------------------------------- | ----- | -------------------- | ----------------------------------------------------------------------------------- | -| Payload_Pageable_PageSize_listWithoutContinuation_paging | Payload_Pageable_PageSize_listWithoutContinuation | paging | listWithoutContinuation | | expected=Pet | Single-page pageable operation yielding `Pet` elements, no continuation. | -| Payload_Pageable_PageSize_listWithPageSize_paging | Payload_Pageable_PageSize_listWithPageSize | paging | listWithPageSize | | expected=Pet | Page-size-driven pageable operation yielding `Pet` elements. | -| Payload_Pageable_ServerDrivenPagination_link_paging | Payload_Pageable_ServerDrivenPagination_link | paging | link | | expected=Pet | The paginated operation yields `Pet` elements. | -| Serialization_EncodedName_Json_Property_send_naming | Serialization_EncodedName_Json_Property_send | naming | JsonEncodedNameModel.defaultName | | expected=defaultName | The client property is named `defaultName` even though the wire name is `wireName`. | -| Type_Model_Inheritance_NotDiscriminated_getValid_hierarchy | Type_Model_Inheritance_NotDiscriminated_getValid | hierarchy | Siamese | | expected=Cat | The `Siamese` model extends `Cat` as its base type on the client surface. | -| Type_Model_Inheritance_SingleDiscriminator_getModel_hierarchy | Type_Model_Inheritance_SingleDiscriminator_getModel | hierarchy | Sparrow | | expected=Bird | The `Sparrow` model extends `Bird` as its base type on the client surface. | +| id | scenario | category | target | scope | details | doc | +| ------------------------------------------------------------- | --------------------------------------------------- | --------- | -------------------------------- | ----- | ----------------------------------- | ----------------------------------------------------------------------------------- | +| Payload_Pageable_PageSize_listWithoutContinuation_paging | Payload_Pageable_PageSize_listWithoutContinuation | paging | listWithoutContinuation | | expected=Pet | Single-page pageable operation yielding `Pet` elements, no continuation. | +| Payload_Pageable_PageSize_listWithPageSize_paging | Payload_Pageable_PageSize_listWithPageSize | paging | listWithPageSize | | expected=Pet | Page-size-driven pageable operation yielding `Pet` elements. | +| Payload_Pageable_ServerDrivenPagination_link_paging | Payload_Pageable_ServerDrivenPagination_link | paging | link | | expected=Pet | The paginated operation yields `Pet` elements. | +| Serialization_EncodedName_Json_Property_send_naming | Serialization_EncodedName_Json_Property_send | naming | JsonEncodedNameModel.defaultName | | expected=defaultName; kind=property | The client property is named `defaultName` even though the wire name is `wireName`. | +| Type_Model_Inheritance_NotDiscriminated_getValid_hierarchy | Type_Model_Inheritance_NotDiscriminated_getValid | hierarchy | Siamese | | expected=Cat | The `Siamese` model extends `Cat` as its base type on the client surface. | +| Type_Model_Inheritance_SingleDiscriminator_getModel_hierarchy | Type_Model_Inheritance_SingleDiscriminator_getModel | hierarchy | Sparrow | | expected=Bird | The `Sparrow` model extends `Bird` as its base type on the client surface. | diff --git a/packages/spector/generated-defs/TypeSpec.Spector.ts b/packages/spector/generated-defs/TypeSpec.Spector.ts index 96f24d0c5b0..8ae119704b4 100644 --- a/packages/spector/generated-defs/TypeSpec.Spector.ts +++ b/packages/spector/generated-defs/TypeSpec.Spector.ts @@ -65,6 +65,7 @@ export function getSurfaceDoc( | "other"; readonly expected: string | Record; readonly subject?: string; + readonly kind?: string; readonly doc?: string; } | undefined { @@ -86,6 +87,7 @@ export function setSurfaceDoc( | "other"; readonly expected: string | Record; readonly subject?: string; + readonly kind?: string; readonly doc?: string; }, ): void { diff --git a/packages/spector/lib/main.tsp b/packages/spector/lib/main.tsp index be117ab84c6..b940002a5f7 100644 --- a/packages/spector/lib/main.tsp +++ b/packages/spector/lib/main.tsp @@ -96,10 +96,13 @@ union SurfaceCategory { * category: "naming", * expected: "widgetName", * subject: "Widget.defaultName", + * kind: "property", * }) * ``` * * When `subject` is omitted it defaults to the decorated target's name. + * When `kind` is provided, emitter verifiers can apply per-kind conventions + * (e.g. Python recases properties to snake_case, types to PascalCase). * * `expected` is a single canonical, language-agnostic string by default, recast * by each emitter. For exact per-language names, pass a `scope → value` dict: @@ -117,6 +120,7 @@ auto dec surfaceDoc( category: SurfaceCategory, expected: string | Record, subject?: string, + kind?: string, doc?: string, } ); diff --git a/packages/spector/src/actions/verify-surface-checks.ts b/packages/spector/src/actions/verify-surface-checks.ts index 311c435a9f6..c5e1616a615 100644 --- a/packages/spector/src/actions/verify-surface-checks.ts +++ b/packages/spector/src/actions/verify-surface-checks.ts @@ -136,6 +136,8 @@ export interface Rule { find: string; /** `present` (default), `absent`, `{ absentWhen: }`, or `{ whenExpected: { : present|absent } }`. */ expect?: Expect; + /** Per-kind casing map for the `{var:byKind}` modifier; keyed by `details.kind`. */ + casing?: Record; /** A second rule AND-ed with this one (e.g. present here AND absent there). */ also?: Rule; } @@ -216,6 +218,7 @@ function resolveVar( mod: string | undefined, item: CheckItem, _language: string, + casingMap: Record | undefined, ): string | undefined { let val: string | undefined; if (name === "target") { @@ -230,16 +233,31 @@ function resolveVar( if (val == null) return undefined; // A scoped check carries a verbatim per-language value — never recase it. if (mod && !item.scope) { - val = applyCasing(val, mod); + if (mod === "byKind") { + const kind = item.details?.["kind"]; + if (kind && casingMap) { + const casingName = casingMap[String(kind)]; + if (casingName) { + val = applyCasing(val, casingName); + } + } + } else { + val = applyCasing(val, mod); + } } return val; } /** Compile a regex template; returns null if any placeholder is unresolvable (N/A). */ -function buildRegex(template: string, item: CheckItem, language: string): RegExp | null { +function buildRegex( + template: string, + item: CheckItem, + language: string, + casingMap: Record | undefined, +): RegExp | null { let missing = false; const source = template.replace(/\{(\w+)(?::(\w+))?\}/g, (_m, v: string, mod?: string) => { - const resolved = resolveVar(v, mod, item, language); + const resolved = resolveVar(v, mod, item, language, casingMap); if (resolved == null) { missing = true; return ""; @@ -287,13 +305,13 @@ async function runRule( pkgDir: string, language: string, ): Promise { - const findRe = buildRegex(rule.find, item, language); + const findRe = buildRegex(rule.find, item, language, rule.casing); if (findRe === null) return null; const text = await readGlobbed(pkgDir, rule.files); let haystacks: string[] = [text]; if (rule.scope) { - const scopeRe = buildRegex(rule.scope, item, language); + const scopeRe = buildRegex(rule.scope, item, language, rule.casing); if (scopeRe === null) return null; haystacks = [...text.matchAll(new RegExp(scopeRe.source, "g"))].map((m) => m[0]); } diff --git a/packages/spector/src/lib/decorators.ts b/packages/spector/src/lib/decorators.ts index a211cfe509d..7ae8dbb750c 100644 --- a/packages/spector/src/lib/decorators.ts +++ b/packages/spector/src/lib/decorators.ts @@ -241,6 +241,8 @@ export type SurfaceDocTarget = Namespace | Interface | Operation; export interface SurfaceDetails { /** The author's `expected` client-surface output for this check. */ expected?: string; + /** The symbol kind (e.g. "property", "model", "enum", "operation"). */ + kind?: string; } /** A resolved `@surfaceDoc` annotation. */ @@ -262,6 +264,8 @@ export interface SurfaceDoc { category: string; /** The expected client-surface output for this category. */ expected: string; + /** The symbol kind (e.g. "property", "model", "enum", "operation") — used by verifiers for per-kind conventions. */ + kind?: string; /** * The language scope this check applies to, e.g. `"python"`, `"python,csharp"`, * or `"!java"`. Set only when `expected` came from a `scope → value` dict; in @@ -318,6 +322,7 @@ interface StoredSurfaceDoc { category: string; expected: string | Record; subject?: string; + kind?: string; doc?: string; }; } @@ -338,7 +343,7 @@ export function listSurfaceDocs(program: Program): SurfaceDoc[] { if (!stored) continue; const docTarget = target as SurfaceDocTarget; const scenario = getEnclosingScenarioName(program, docTarget); - const { category, expected, subject, doc } = stored.check; + const { category, expected, subject, kind, doc } = stored.check; const resolvedSubject = subject ?? docTarget.name ?? ""; for (const { expected: exp, scope } of expandExpected(expected)) { result.push({ @@ -347,6 +352,7 @@ export function listSurfaceDocs(program: Program): SurfaceDoc[] { subject: resolvedSubject, category, expected: exp, + kind, scope, doc: doc ?? synthesizeDoc(category, resolvedSubject, exp), }); @@ -406,5 +412,8 @@ export function buildSurfaceDetails(doc: SurfaceDoc): SurfaceDetails { if (doc.expected !== "") { details.expected = doc.expected; } + if (doc.kind) { + details.kind = doc.kind; + } return details; } diff --git a/packages/spector/test/verify-surface-checks.test.ts b/packages/spector/test/verify-surface-checks.test.ts index 64e69978dc6..56f9db5df28 100644 --- a/packages/spector/test/verify-surface-checks.test.ts +++ b/packages/spector/test/verify-surface-checks.test.ts @@ -21,7 +21,8 @@ const verifiers = { }, naming: { files: ["**/*.py"], - find: "\\b{expected}\\b", + find: "\\b{expected:byKind}\\b", + casing: { enum: "pascal", property: "snake" }, }, hierarchy: { files: ["**/models/_models.py"], @@ -52,6 +53,7 @@ beforeAll(async () => { `class WidgetsOperations:\n def get_widget(self):\n pass\n\nclass GadgetsOperations:\n def list_gadgets(self):\n pass\n`, ); await writeFile(join(pkg, "models", "_scoped.py"), `class IOThing:\n pass\n`); + await writeFile(join(pkg, "models", "_props.py"), `class SomeModel:\n default_name: bool\n`); verifiersPath = join(root, "verifiers.json"); await writeFile(verifiersPath, JSON.stringify(verifiers)); @@ -71,14 +73,14 @@ async function run(items: object[], scenario?: string): Promise { - it("naming: idiomatic casing via {expected} passes on the expected identifier", async () => { + it("naming: idiomatic casing via {expected:byKind} passes on the recast identifier", async () => { const out = await run([ { id: "n1", scenario: "surfacedemo", category: "naming", target: "ServerExtensibleEnum", - details: { expected: "ClientExtensibleEnum" }, + details: { expected: "ClientExtensibleEnum", kind: "enum" }, doc: "", }, ]); @@ -92,13 +94,28 @@ describe("verify-surface-checks (shared engine)", () => { scenario: "surfacedemo", category: "naming", target: "X", - details: { expected: "TotallyMadeUp" }, + details: { expected: "TotallyMadeUp", kind: "enum" }, doc: "", }, ]); expect(out.results[0].status).toBe("fail"); }); + it("naming: byKind recases a property to snake_case", async () => { + const out = await run([ + { + id: "n3", + scenario: "surfacedemo", + category: "naming", + target: "SomeModel.defaultName", + details: { expected: "defaultName", kind: "property" }, + doc: "", + }, + ]); + // expected "defaultName" + kind "property" → snake → "default_name" → found in _props.py + expect(out.results[0]).toMatchObject({ status: "pass", how: "deterministic" }); + }); + it("access: public export present passes; internal expectation flips it", async () => { const pass = await run([ { @@ -172,7 +189,7 @@ describe("verify-surface-checks (shared engine)", () => { category: "naming", target: "IOThing", scope: "csharp", - details: { expected: "IOThing" }, + details: { expected: "IOThing", kind: "enum" }, doc: "", }, ]); @@ -180,7 +197,7 @@ describe("verify-surface-checks (shared engine)", () => { expect(out.needs_ai).toHaveLength(0); }); - it("scope: a scoped value is matched verbatim", async () => { + it("scope: a scoped value is matched verbatim, bypassing byKind casing", async () => { const out = await run([ { id: "s2", @@ -188,7 +205,7 @@ describe("verify-surface-checks (shared engine)", () => { category: "naming", target: "IOThing", scope: "python", - details: { expected: "IOThing" }, + details: { expected: "IOThing", kind: "enum" }, doc: "", }, ]); From 9d65909a00b36e877201447969728eb0a0d8cc8f Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 31 Jul 2026 15:01:17 -0400 Subject: [PATCH 8/9] refactor: move shared surface-check assets out of http-client-python Move check-surface.prompt.md to .github/prompts/ (it's language-agnostic, parameterized via ${input:emitter}). Move design README.md to packages/spector/docs/surface-checks.md (it documents the @surfaceDoc system which lives in spector). Python emitter now only has the 2 files a language needs: - eng/scripts/client-criteria/context.md (emitter facts + concepts) - eng/scripts/client-criteria/verifiers.json (declarative routing) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 54fb668c-64c3-41e5-8933-c4300c884974 --- .../.github => .github}/prompts/check-surface.prompt.md | 4 ++-- .../README.md => spector/docs/surface-checks.md} | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) rename {packages/http-client-python/.github => .github}/prompts/check-surface.prompt.md (96%) rename packages/{http-client-python/eng/scripts/client-criteria/README.md => spector/docs/surface-checks.md} (99%) diff --git a/packages/http-client-python/.github/prompts/check-surface.prompt.md b/.github/prompts/check-surface.prompt.md similarity index 96% rename from packages/http-client-python/.github/prompts/check-surface.prompt.md rename to .github/prompts/check-surface.prompt.md index 27b7f63a809..fc82846948c 100644 --- a/packages/http-client-python/.github/prompts/check-surface.prompt.md +++ b/.github/prompts/check-surface.prompt.md @@ -43,7 +43,7 @@ verifier — in parallel, no AI — and prints `{"results": [...], "needs_ai": [ ``` npx tsp-spector verify-surface-checks \ - --checks packages/${input:emitter}/eng/scripts/client-criteria/ \ + --checks \ --verifiers packages/${input:emitter}/eng/scripts/client-criteria/verifiers.json \ --generated-root \ --flavor ${input:flavor} --language @@ -99,6 +99,6 @@ changes, and is never generated by the emitter — so **only write the file if t caller explicitly asks**; otherwise leave the suggestions for review. A verify run is read-only by default. -The point: the bulk is decided without AI — a py routine or an existing test — +The point: the bulk is decided without AI — a deterministic routine or an existing test — and AI is spent only on the checks that actually need judgment, shrinking further as verifiers.json is filled in. diff --git a/packages/http-client-python/eng/scripts/client-criteria/README.md b/packages/spector/docs/surface-checks.md similarity index 99% rename from packages/http-client-python/eng/scripts/client-criteria/README.md rename to packages/spector/docs/surface-checks.md index ee65efdc875..69d082215d7 100644 --- a/packages/http-client-python/eng/scripts/client-criteria/README.md +++ b/packages/spector/docs/surface-checks.md @@ -22,6 +22,7 @@ assertion about the generated client surface: category: "naming", expected: "defaultName", subject: "Widget", // optional — defaults to target's name + kind: "property", // optional — enables per-kind casing in verifiers doc: "Explanation...", // optional }) ``` From 0f13267029dd92088219ab3092809780cad88639 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 31 Jul 2026 15:45:03 -0400 Subject: [PATCH 9/9] refactor: switch surface-checks doc from Markdown to JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JSON is strictly better for the checks doc: trivially consumed by CI, scripts, and the verify engine without fragile pipe-escaping and semicolon-delimited detail parsing. - Replace createSurfaceChecksSummary with JSON output (remove prettier dep) - Remove Markdown table parser from parseChecksDoc (JSON-only now) - Remove renderDetails, escapeCell, parseDetails helper functions - Rename surface-checks.md → surface-checks.json in http-specs - Update CLI default output filename to .json - Update all doc/prompt references - Update tests to assert JSON structure Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 54fb668c-64c3-41e5-8933-c4300c884974 --- .github/prompts/check-surface.prompt.md | 2 +- .../eng/scripts/client-criteria/context.md | 2 +- packages/http-specs/surface-checks.json | 65 +++++++++++++++++++ packages/http-specs/surface-checks.md | 13 ---- packages/spector/docs/surface-checks.md | 2 +- .../src/actions/generate-surface-checks.ts | 2 +- .../src/actions/verify-surface-checks.ts | 54 +-------------- packages/spector/src/cli/cli.ts | 6 +- .../src/coverage/surface-checks-manifest.ts | 61 ++++------------- packages/spector/test/surface-docs.test.ts | 29 +++++---- .../test/verify-surface-checks.test.ts | 21 ++++-- 11 files changed, 118 insertions(+), 139 deletions(-) create mode 100644 packages/http-specs/surface-checks.json delete mode 100644 packages/http-specs/surface-checks.md diff --git a/.github/prompts/check-surface.prompt.md b/.github/prompts/check-surface.prompt.md index fc82846948c..c68101431d6 100644 --- a/.github/prompts/check-surface.prompt.md +++ b/.github/prompts/check-surface.prompt.md @@ -17,7 +17,7 @@ Read: - `packages/${input:emitter}/eng/scripts/client-criteria/context.md` — Emitter facts (language, generated-root, flavors, checks-doc path) and concept prose. - The **checks doc** — the `checks-doc` Markdown table named in `context.md` - (default `surface-checks.md`): the batch runner reads it for deterministic + (default `surface-checks.json`): the batch runner reads it for deterministic checks and you read its rows' `doc` prose for the AI ones. Use this path unless the caller explicitly overrides it. - `packages/${input:emitter}/eng/scripts/client-criteria/verifiers.json` — the diff --git a/packages/http-client-python/eng/scripts/client-criteria/context.md b/packages/http-client-python/eng/scripts/client-criteria/context.md index d3e9c58b1d3..a854aa7292e 100644 --- a/packages/http-client-python/eng/scripts/client-criteria/context.md +++ b/packages/http-client-python/eng/scripts/client-criteria/context.md @@ -7,7 +7,7 @@ _The shared `/check-surface` prompt and `tsp-spector verify-surface-checks` read - **language:** python _(selects which per-language client name applies when a check is language-scoped)_ - **generated-root:** `packages/http-client-python/tests/generated/` - **flavors:** `azure`, `unbranded` -- **checks-doc:** `packages/http-specs/surface-checks.md` _(generated from `@surfaceDoc`, ships with the specs)_ +- **checks-doc:** `packages/http-specs/surface-checks.json` _(generated from `@surfaceDoc`, ships with the specs)_ diff --git a/packages/http-specs/surface-checks.json b/packages/http-specs/surface-checks.json new file mode 100644 index 00000000000..523bbc116a9 --- /dev/null +++ b/packages/http-specs/surface-checks.json @@ -0,0 +1,65 @@ +{ + "items": [ + { + "id": "Payload_Pageable_PageSize_listWithoutContinuation_paging", + "scenario": "Payload_Pageable_PageSize_listWithoutContinuation", + "category": "paging", + "target": "listWithoutContinuation", + "details": { + "expected": "Pet" + }, + "doc": "Single-page pageable operation yielding `Pet` elements, no continuation." + }, + { + "id": "Payload_Pageable_PageSize_listWithPageSize_paging", + "scenario": "Payload_Pageable_PageSize_listWithPageSize", + "category": "paging", + "target": "listWithPageSize", + "details": { + "expected": "Pet" + }, + "doc": "Page-size-driven pageable operation yielding `Pet` elements." + }, + { + "id": "Payload_Pageable_ServerDrivenPagination_link_paging", + "scenario": "Payload_Pageable_ServerDrivenPagination_link", + "category": "paging", + "target": "link", + "details": { + "expected": "Pet" + }, + "doc": "The paginated operation yields `Pet` elements." + }, + { + "id": "Serialization_EncodedName_Json_Property_send_naming", + "scenario": "Serialization_EncodedName_Json_Property_send", + "category": "naming", + "target": "JsonEncodedNameModel.defaultName", + "details": { + "expected": "defaultName", + "kind": "property" + }, + "doc": "The client property is named `defaultName` even though the wire name is `wireName`." + }, + { + "id": "Type_Model_Inheritance_NotDiscriminated_getValid_hierarchy", + "scenario": "Type_Model_Inheritance_NotDiscriminated_getValid", + "category": "hierarchy", + "target": "Siamese", + "details": { + "expected": "Cat" + }, + "doc": "The `Siamese` model extends `Cat` as its base type on the client surface." + }, + { + "id": "Type_Model_Inheritance_SingleDiscriminator_getModel_hierarchy", + "scenario": "Type_Model_Inheritance_SingleDiscriminator_getModel", + "category": "hierarchy", + "target": "Sparrow", + "details": { + "expected": "Bird" + }, + "doc": "The `Sparrow` model extends `Bird` as its base type on the client surface." + } + ] +} diff --git a/packages/http-specs/surface-checks.md b/packages/http-specs/surface-checks.md deleted file mode 100644 index 8d59e766064..00000000000 --- a/packages/http-specs/surface-checks.md +++ /dev/null @@ -1,13 +0,0 @@ -# Surface checks - -Generated from `@surfaceDoc` annotations. This table is both the human summary -and the machine-readable checks doc parsed by the shared `verify-surface-checks` runner. - -| id | scenario | category | target | scope | details | doc | -| ------------------------------------------------------------- | --------------------------------------------------- | --------- | -------------------------------- | ----- | ----------------------------------- | ----------------------------------------------------------------------------------- | -| Payload_Pageable_PageSize_listWithoutContinuation_paging | Payload_Pageable_PageSize_listWithoutContinuation | paging | listWithoutContinuation | | expected=Pet | Single-page pageable operation yielding `Pet` elements, no continuation. | -| Payload_Pageable_PageSize_listWithPageSize_paging | Payload_Pageable_PageSize_listWithPageSize | paging | listWithPageSize | | expected=Pet | Page-size-driven pageable operation yielding `Pet` elements. | -| Payload_Pageable_ServerDrivenPagination_link_paging | Payload_Pageable_ServerDrivenPagination_link | paging | link | | expected=Pet | The paginated operation yields `Pet` elements. | -| Serialization_EncodedName_Json_Property_send_naming | Serialization_EncodedName_Json_Property_send | naming | JsonEncodedNameModel.defaultName | | expected=defaultName; kind=property | The client property is named `defaultName` even though the wire name is `wireName`. | -| Type_Model_Inheritance_NotDiscriminated_getValid_hierarchy | Type_Model_Inheritance_NotDiscriminated_getValid | hierarchy | Siamese | | expected=Cat | The `Siamese` model extends `Cat` as its base type on the client surface. | -| Type_Model_Inheritance_SingleDiscriminator_getModel_hierarchy | Type_Model_Inheritance_SingleDiscriminator_getModel | hierarchy | Sparrow | | expected=Bird | The `Sparrow` model extends `Bird` as its base type on the client surface. | diff --git a/packages/spector/docs/surface-checks.md b/packages/spector/docs/surface-checks.md index 69d082215d7..61d51a57031 100644 --- a/packages/spector/docs/surface-checks.md +++ b/packages/spector/docs/surface-checks.md @@ -154,7 +154,7 @@ never reaches AI; every check in the category resolves to `not-applicable`: - **Where the precomputed checks doc ships:** alongside the specs, exactly like the scenario summary. `packages/http-specs` runs `tsp-spector generate-surface-checks ./specs` (wired into `regen-docs`) to - produce a committed `surface-checks.md`, the same way + produce a committed `surface-checks.json`, the same way `generate-scenarios-summary` produces `spec-summary.md`. - **How items mark N/A for a language:** per **category** in the emitter's `verifiers.json` (`{ "na": true }`), mirroring wire `unsupportedScenarios`; diff --git a/packages/spector/src/actions/generate-surface-checks.ts b/packages/spector/src/actions/generate-surface-checks.ts index e1b7292f1d6..e65f64ac2bf 100644 --- a/packages/spector/src/actions/generate-surface-checks.ts +++ b/packages/spector/src/actions/generate-surface-checks.ts @@ -21,7 +21,7 @@ export async function generateSurfaceChecks({ process.exit(-1); } - const summary = await createSurfaceChecksSummary(manifest); + const summary = createSurfaceChecksSummary(manifest); await writeFile(outputFile, summary); logger.info( `${pc.green("✓")} Surface checks doc generated at ${outputFile} (${manifest.items.length} checks).`, diff --git a/packages/spector/src/actions/verify-surface-checks.ts b/packages/spector/src/actions/verify-surface-checks.ts index c5e1616a615..524e10bba30 100644 --- a/packages/spector/src/actions/verify-surface-checks.ts +++ b/packages/spector/src/actions/verify-surface-checks.ts @@ -35,58 +35,10 @@ export interface CheckItem { doc: string; } -/** Parse the checks doc — a Markdown table (the shipped format) or JSON. */ +/** Parse the checks doc (JSON format). */ export function parseChecksDoc(text: string): CheckItem[] { - const trimmed = text.trimStart(); - if (trimmed.startsWith("{") || trimmed.startsWith("[")) { - const data = JSON.parse(text); - return Array.isArray(data) ? data : data.items; - } - const rows: CheckItem[] = []; - let headerSeen = false; - for (const raw of text.split(/\r?\n/)) { - const line = raw.trim(); - if (!line.startsWith("|")) continue; - const cells = line - .replace(/^\|/, "") - .replace(/\|$/, "") - .split(/(? c.trim().replace(/\\\|/g, "|")); - if (!headerSeen) { - if (cells[0] === "id") headerSeen = true; - continue; - } - if (cells.every((c) => /^[- ]*$/.test(c))) continue; // separator row - if (cells.length < 7) continue; - const [id, scenario, category, target, scope, details, doc] = cells; - rows.push({ - id, - scenario: scenario || undefined, - category, - target, - scope: scope || undefined, - details: parseDetails(details), - doc, - }); - } - return rows; -} - -/** `key=value; key=value` → object; `true`/`false` become booleans. */ -function parseDetails(cell: string): Record { - const out: Record = {}; - for (const pair of cell.split(";")) { - const p = pair.trim(); - if (!p || !p.includes("=")) continue; - const i = p.indexOf("="); - const k = p.slice(0, i).trim(); - const v = p - .slice(i + 1) - .trim() - .replace(/\\\|/g, "|"); - out[k] = v === "true" ? true : v === "false" ? false : v; - } - return out; + const data = JSON.parse(text); + return Array.isArray(data) ? data : data.items; } /** diff --git a/packages/spector/src/cli/cli.ts b/packages/spector/src/cli/cli.ts index a81ec46e4bc..4e8648a2226 100644 --- a/packages/spector/src/cli/cli.ts +++ b/packages/spector/src/cli/cli.ts @@ -94,7 +94,7 @@ async function main() { ) .command( "generate-surface-checks ", - "Precompute the surface-checks doc (Markdown, from @surfaceDoc) for all specs.", + "Precompute the surface-checks doc (JSON, from @surfaceDoc) for all specs.", (cmd) => { return cmd .positional("scenariosPaths", { @@ -105,8 +105,8 @@ async function main() { }) .option("outputFile", { type: "string", - description: "Path to the generated surface-checks doc (Markdown).", - default: join(process.cwd(), "surface-checks.md"), + description: "Path to the generated surface-checks doc (JSON).", + default: join(process.cwd(), "surface-checks.json"), }); }, async (args) => { diff --git a/packages/spector/src/coverage/surface-checks-manifest.ts b/packages/spector/src/coverage/surface-checks-manifest.ts index 7978d2ecb22..82fcf8afe07 100644 --- a/packages/spector/src/coverage/surface-checks-manifest.ts +++ b/packages/spector/src/coverage/surface-checks-manifest.ts @@ -1,7 +1,6 @@ import { getSourceLocation, normalizePath } from "@typespec/compiler"; import { relative } from "path"; import pc from "picocolors"; -import prettier from "prettier"; import type { SurfaceDetails, SurfaceDoc, SurfaceDocTarget } from "../lib/decorators.js"; import { buildSurfaceDetails } from "../lib/decorators.js"; import { logger } from "../logger.js"; @@ -90,53 +89,21 @@ export function createSurfaceChecksManifest( } /** - * Render a {@link SurfaceChecksManifest} as a single Markdown document that is - * both human-readable and machine-readable: the table below is the source of - * truth consumed by the shared `verify-surface-checks` runner, and is also easy - * to read in review. The - * `details` column encodes category-specific expectations as `key=value` pairs - * separated by `; ` (booleans as `true`/`false`). + * Render a {@link SurfaceChecksManifest} as a JSON document consumed by the + * shared `verify-surface-checks` runner. Each item includes id, scenario, + * category, target, scope, details, and doc. */ -export function createSurfaceChecksSummary(manifest: SurfaceChecksManifest): Promise { - const lines = [ - `# Surface checks`, - ``, - `Generated from \`@surfaceDoc\` annotations. This table is both the human summary`, - `and the machine-readable checks doc parsed by the shared \`verify-surface-checks\` runner.`, - ``, - `| id | scenario | category | target | scope | details | doc |`, - `| --- | --- | --- | --- | --- | --- | --- |`, - ]; - for (const item of manifest.items) { - const cells = [ - item.id, - item.scenario ?? "", - item.category, - item.target, - item.scope ?? "", - renderDetails(item.details), - escapeCell(item.doc), - ]; - lines.push(`| ${cells.join(" | ")} |`); - } - return prettier.format(lines.join("\n"), { parser: "markdown" }); -} - -/** Encode `details` as `key=value; key=value` for the Markdown table cell. */ -function renderDetails(details: SurfaceDetails | undefined): string { - if (!details) { - return ""; - } - return escapeCell( - Object.entries(details) - .map(([k, v]) => `${k}=${v}`) - .join("; "), - ); -} - -/** Escape characters that would break a Markdown table cell. */ -function escapeCell(text: string): string { - return text.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\n/g, " "); +export function createSurfaceChecksSummary(manifest: SurfaceChecksManifest): string { + const items = manifest.items.map((item) => ({ + id: item.id, + scenario: item.scenario ?? "", + category: item.category, + target: item.target, + ...(item.scope ? { scope: item.scope } : {}), + details: item.details ?? {}, + doc: item.doc, + })); + return JSON.stringify({ items }, null, 2) + "\n"; } function uniqueId(used: Map, base: string): string { diff --git a/packages/spector/test/surface-docs.test.ts b/packages/spector/test/surface-docs.test.ts index f7c0f22e803..7e684820f18 100644 --- a/packages/spector/test/surface-docs.test.ts +++ b/packages/spector/test/surface-docs.test.ts @@ -206,7 +206,7 @@ describe("@surfaceDoc", () => { // --- rendering ---------------------------------------------------------- - it("renders a single Markdown table that carries every routable field", async () => { + it("renders a JSON checks doc that carries every routable field", async () => { const { program } = await Tester.compile( ` model Widget { @@ -220,21 +220,22 @@ describe("@surfaceDoc", () => { `, ); const manifest = createSurfaceChecksManifest(".", "1.0.0", "abc123", listSurfaceDocs(program)); - const md = await createSurfaceChecksSummary(manifest); + const json = createSurfaceChecksSummary(manifest); + const data = JSON.parse(json); // The rendered doc is idempotent: no volatile version/commit provenance. - expect(md).not.toContain("commit:"); - expect(md).toContain("Generated from `@surfaceDoc` annotations."); - // Table header with the routable columns (prettier pads cell widths). - const header = md.split("\n").find((l) => l.includes("| id")); - expect(header).toBeDefined(); - for (const col of ["id", "scenario", "category", "target", "scope", "details", "doc"]) { - expect(header).toContain(col); - } - // details encoded as key=value. - expect(md).toContain("expected=WidgetInternal"); - // Pipes inside prose are escaped so they don't break the table. - expect(md).toContain("Hidden \\| renamed to WidgetInternal."); + expect(json).not.toContain("commit:"); + // Structured items with all routable fields. + expect(data.items).toHaveLength(1); + const item = data.items[0]; + expect(item).toMatchObject({ + category: "naming", + target: "Widget", + details: { expected: "WidgetInternal" }, + doc: "Hidden | renamed to WidgetInternal.", + }); + expect(item.id).toBeDefined(); + expect(item.scenario).toBeDefined(); }); // --- per-language exact names (scope → value dict) ---------------------- diff --git a/packages/spector/test/verify-surface-checks.test.ts b/packages/spector/test/verify-surface-checks.test.ts index 56f9db5df28..f2c0346de0e 100644 --- a/packages/spector/test/verify-surface-checks.test.ts +++ b/packages/spector/test/verify-surface-checks.test.ts @@ -298,13 +298,20 @@ describe("verify-surface-checks (shared engine)", () => { expect(out.needs_ai).toHaveLength(0); }); - it("parseChecksDoc reads a Markdown table with details + escaped pipes", () => { - const md = [ - "| id | scenario | category | target | scope | details | doc |", - "| --- | --- | --- | --- | --- | --- | --- |", - "| x1 | S | access | Widget | | internal=true | Hidden \\| renamed. |", - ].join("\n"); - const items = parseChecksDoc(md); + it("parseChecksDoc reads a JSON checks doc", () => { + const json = JSON.stringify({ + items: [ + { + id: "x1", + scenario: "S", + category: "access", + target: "Widget", + details: { internal: true }, + doc: "Hidden | renamed.", + }, + ], + }); + const items = parseChecksDoc(json); expect(items[0]).toMatchObject({ id: "x1", scenario: "S",